From 26344c3ec7409713fcf52a9c41cd0dce537b3100 Mon Sep 17 00:00:00 2001 From: alanprot Date: Tue, 8 Apr 2025 11:51:43 -0700 Subject: [PATCH] First Parquet implementation draft --- go.mod | 10 + go.sum | 22 +- pkg/compactor/compaction_marker.go | 61 + pkg/compactor/parquet_compactor.go | 607 + .../parquet_blocks_finder_bucket_index.go | 139 + pkg/querier/parquet_queryable.go | 1041 + pkg/storage/parquet/cache.go | 80 + pkg/storage/parquet/enc.go | 71 + pkg/storage/parquet/enc_test.go | 42 + .../prometheus_parquet_chunks_encoder.go | 142 + pkg/storage/parquet/reader.go | 893 + pkg/storage/parquet/reader_test.go | 190 + pkg/storage/parquet/row.go | 16 + pkg/storage/parquet/stats.go | 247 + pkg/storage/parquet/stats_test.go | 21 + pkg/storage/parquet/util_test.go | 20 + pkg/storage/parquet/writer.go | 121 + pkg/storage/parquet/writer_test.go | 50 + .../tsdb/bucketindex/parquet_bucket_index.go | 93 + .../tsdb/bucketindex/parquet_loader.go | 301 + vendor/github.com/andybalholm/brotli/LICENSE | 19 + .../github.com/andybalholm/brotli/README.md | 14 + .../andybalholm/brotli/backward_references.go | 185 + .../brotli/backward_references_hq.go | 796 + .../github.com/andybalholm/brotli/bit_cost.go | 436 + .../andybalholm/brotli/bit_reader.go | 266 + .../andybalholm/brotli/bitwriter.go | 56 + .../andybalholm/brotli/block_splitter.go | 144 + .../brotli/block_splitter_command.go | 434 + .../brotli/block_splitter_distance.go | 433 + .../brotli/block_splitter_literal.go | 433 + .../andybalholm/brotli/brotli_bit_stream.go | 1539 + .../github.com/andybalholm/brotli/cluster.go | 30 + .../andybalholm/brotli/cluster_command.go | 164 + .../andybalholm/brotli/cluster_distance.go | 326 + .../andybalholm/brotli/cluster_literal.go | 326 + .../github.com/andybalholm/brotli/command.go | 254 + .../andybalholm/brotli/compress_fragment.go | 834 + .../brotli/compress_fragment_two_pass.go | 773 + .../andybalholm/brotli/constants.go | 77 + .../github.com/andybalholm/brotli/context.go | 2176 + .../github.com/andybalholm/brotli/decode.go | 2581 + .../andybalholm/brotli/dictionary.go | 122890 +++++++++++++++ .../andybalholm/brotli/dictionary_hash.go | 32779 ++++ .../github.com/andybalholm/brotli/encode.go | 1220 + .../github.com/andybalholm/brotli/encoder.go | 168 + .../andybalholm/brotli/encoder_dict.go | 22 + .../andybalholm/brotli/entropy_encode.go | 592 + .../brotli/entropy_encode_static.go | 4399 + .../github.com/andybalholm/brotli/fast_log.go | 290 + .../andybalholm/brotli/find_match_length.go | 45 + vendor/github.com/andybalholm/brotli/h10.go | 287 + vendor/github.com/andybalholm/brotli/h5.go | 214 + vendor/github.com/andybalholm/brotli/h6.go | 216 + vendor/github.com/andybalholm/brotli/hash.go | 342 + .../andybalholm/brotli/hash_composite.go | 93 + .../brotli/hash_forgetful_chain.go | 252 + .../brotli/hash_longest_match_quickly.go | 214 + .../andybalholm/brotli/hash_rolling.go | 168 + .../andybalholm/brotli/histogram.go | 226 + vendor/github.com/andybalholm/brotli/http.go | 184 + .../github.com/andybalholm/brotli/huffman.go | 653 + .../andybalholm/brotli/literal_cost.go | 182 + .../andybalholm/brotli/matchfinder/emitter.go | 45 + .../andybalholm/brotli/matchfinder/m0.go | 169 + .../andybalholm/brotli/matchfinder/m4.go | 297 + .../brotli/matchfinder/matchfinder.go | 103 + .../brotli/matchfinder/textencoder.go | 53 + .../github.com/andybalholm/brotli/memory.go | 66 + .../andybalholm/brotli/metablock.go | 574 + .../andybalholm/brotli/metablock_command.go | 165 + .../andybalholm/brotli/metablock_distance.go | 165 + .../andybalholm/brotli/metablock_literal.go | 165 + .../github.com/andybalholm/brotli/params.go | 37 + .../github.com/andybalholm/brotli/platform.go | 103 + .../github.com/andybalholm/brotli/prefix.go | 30 + .../andybalholm/brotli/prefix_dec.go | 723 + .../github.com/andybalholm/brotli/quality.go | 196 + .../github.com/andybalholm/brotli/reader.go | 108 + .../andybalholm/brotli/ringbuffer.go | 134 + vendor/github.com/andybalholm/brotli/state.go | 294 + .../andybalholm/brotli/static_dict.go | 662 + .../andybalholm/brotli/static_dict_lut.go | 75094 +++++++++ .../andybalholm/brotli/symbol_list.go | 22 + .../andybalholm/brotli/transform.go | 641 + .../andybalholm/brotli/utf8_util.go | 70 + vendor/github.com/andybalholm/brotli/util.go | 7 + .../andybalholm/brotli/write_bits.go | 52 + .../github.com/andybalholm/brotli/writer.go | 162 + .../github.com/axiomhq/hyperloglog/.gitignore | 14 + .../axiomhq/hyperloglog/Contributing.md | 41 + vendor/github.com/axiomhq/hyperloglog/LICENSE | 19 + .../github.com/axiomhq/hyperloglog/README.md | 51 + vendor/github.com/axiomhq/hyperloglog/beta.go | 273 + .../axiomhq/hyperloglog/compressed.go | 168 + .../axiomhq/hyperloglog/hyperloglog.go | 354 + .../github.com/axiomhq/hyperloglog/sparse.go | 112 + .../github.com/axiomhq/hyperloglog/utils.go | 46 + vendor/github.com/kamstrup/intmap/.gitignore | 1 + vendor/github.com/kamstrup/intmap/LICENSE | 23 + vendor/github.com/kamstrup/intmap/README.md | 52 + vendor/github.com/kamstrup/intmap/map64.go | 442 + vendor/github.com/kamstrup/intmap/set.go | 59 + vendor/github.com/mattn/go-runewidth/LICENSE | 21 + .../github.com/mattn/go-runewidth/README.md | 27 + .../mattn/go-runewidth/runewidth.go | 358 + .../mattn/go-runewidth/runewidth_appengine.go | 9 + .../mattn/go-runewidth/runewidth_js.go | 9 + .../mattn/go-runewidth/runewidth_posix.go | 81 + .../mattn/go-runewidth/runewidth_table.go | 439 + .../mattn/go-runewidth/runewidth_windows.go | 28 + .../olekukonko/tablewriter/.gitignore | 15 + .../olekukonko/tablewriter/.travis.yml | 22 + .../olekukonko/tablewriter/LICENSE.md | 19 + .../olekukonko/tablewriter/README.md | 431 + .../github.com/olekukonko/tablewriter/csv.go | 52 + .../olekukonko/tablewriter/table.go | 967 + .../tablewriter/table_with_color.go | 136 + .../github.com/olekukonko/tablewriter/util.go | 93 + .../github.com/olekukonko/tablewriter/wrap.go | 99 + .../parquet-go/parquet-go/.gitattributes | 1 + .../parquet-go/parquet-go/.gitignore | 21 + .../github.com/parquet-go/parquet-go/.mailmap | 2 + .../github.com/parquet-go/parquet-go/.words | 27 + .../parquet-go/parquet-go/AUTHORS.txt | 5 + .../parquet-go/parquet-go/CHANGELOG.md | 16 + .../parquet-go/parquet-go/CODEOWNERS | 1 + .../parquet-go/parquet-go/CODE_OF_CONDUCT.md | 73 + .../parquet-go/parquet-go/CONTRIBUTING.md | 52 + .../github.com/parquet-go/parquet-go/LICENSE | 213 + .../github.com/parquet-go/parquet-go/Makefile | 15 + .../parquet-go/parquet-go/README.md | 581 + .../parquet-go/parquet-go/allocator.go | 64 + .../github.com/parquet-go/parquet-go/array.go | 50 + .../parquet-go/parquet-go/bitmap.go | 43 + .../github.com/parquet-go/parquet-go/bloom.go | 280 + .../parquet-go/parquet-go/bloom/block.go | 28 + .../parquet-go/bloom/block_amd64.go | 39 + .../parquet-go/parquet-go/bloom/block_amd64.s | 129 + .../parquet-go/bloom/block_default.go | 65 + .../parquet-go/bloom/block_optimized.go | 53 + .../parquet-go/parquet-go/bloom/bloom.go | 13 + .../parquet-go/parquet-go/bloom/filter.go | 97 + .../parquet-go/bloom/filter_amd64.go | 33 + .../parquet-go/bloom/filter_amd64.s | 214 + .../parquet-go/bloom/filter_default.go | 17 + .../parquet-go/parquet-go/bloom/hash.go | 77 + .../parquet-go/bloom/xxhash/LICENSE | 27 + .../parquet-go/bloom/xxhash/sum64uint.go | 37 + .../bloom/xxhash/sum64uint_amd64.go | 49 + .../parquet-go/bloom/xxhash/sum64uint_amd64.s | 755 + .../bloom/xxhash/sum64uint_purego.go | 53 + .../parquet-go/bloom/xxhash/xxhash.go | 55 + .../parquet-go/bloom/xxhash/xxhash_amd64.go | 6 + .../parquet-go/bloom/xxhash/xxhash_amd64.s | 180 + .../parquet-go/bloom/xxhash/xxhash_purego.go | 50 + .../parquet-go/parquet-go/bloom_be.go | 19 + .../parquet-go/parquet-go/bloom_le.go | 12 + .../parquet-go/parquet-go/buffer.go | 716 + .../parquet-go/parquet-go/buffer_pool.go | 377 + .../parquet-go/parquet-go/column.go | 818 + .../parquet-go/parquet-go/column_buffer.go | 2472 + .../parquet-go/column_buffer_amd64.go | 30 + .../parquet-go/column_buffer_amd64.s | 67 + .../parquet-go/column_buffer_purego.go | 30 + .../parquet-go/parquet-go/column_chunk.go | 339 + .../parquet-go/parquet-go/column_index.go | 754 + .../parquet-go/parquet-go/column_index_be.go | 84 + .../parquet-go/parquet-go/column_index_le.go | 38 + .../parquet-go/parquet-go/column_mapping.go | 88 + .../parquet-go/parquet-go/column_path.go | 108 + .../parquet-go/parquet-go/compare.go | 351 + .../parquet-go/parquet-go/compress.go | 96 + .../parquet-go/compress/brotli/brotli.go | 54 + .../parquet-go/compress/compress.go | 142 + .../parquet-go/compress/gzip/gzip.go | 67 + .../parquet-go/parquet-go/compress/lz4/lz4.go | 87 + .../parquet-go/compress/snappy/snappy.go | 31 + .../compress/uncompressed/uncompressed.go | 27 + .../parquet-go/compress/zstd/zstd.go | 102 + .../parquet-go/parquet-go/config.go | 879 + .../parquet-go/parquet-go/convert.go | 1068 + .../parquet-go/parquet-go/dedupe.go | 107 + .../parquet-go/parquet-go/deprecated/int96.go | 179 + .../parquet-go/deprecated/parquet.go | 112 + .../parquet-go/parquet-go/dictionary.go | 1483 + .../parquet-go/parquet-go/dictionary_amd64.go | 168 + .../parquet-go/parquet-go/dictionary_amd64.s | 941 + .../parquet-go/dictionary_purego.go | 210 + .../parquet-go/parquet-go/encoding.go | 160 + .../encoding/bitpacked/bitpacked.go | 119 + .../bytestreamsplit/bytestreamsplit.go | 60 + .../bytestreamsplit/bytestreamsplit_amd64.go | 35 + .../bytestreamsplit/bytestreamsplit_amd64.s | 426 + .../bytestreamsplit/bytestreamsplit_purego.go | 83 + .../encoding/delta/binary_packed.go | 488 + .../encoding/delta/binary_packed_amd64.go | 256 + .../encoding/delta/binary_packed_amd64.s | 920 + .../encoding/delta/binary_packed_purego.go | 105 + .../parquet-go/encoding/delta/byte_array.go | 212 + .../encoding/delta/byte_array_amd64.go | 164 + .../encoding/delta/byte_array_amd64.s | 243 + .../encoding/delta/byte_array_purego.go | 63 + .../parquet-go/encoding/delta/delta.go | 100 + .../parquet-go/encoding/delta/delta_amd64.go | 16 + .../parquet-go/encoding/delta/delta_amd64.s | 13 + .../encoding/delta/length_byte_array.go | 81 + .../encoding/delta/length_byte_array_amd64.go | 9 + .../encoding/delta/length_byte_array_amd64.s | 122 + .../delta/length_byte_array_purego.go | 24 + .../parquet-go/encoding/encoding.go | 72 + .../parquet-go/encoding/notsupported.go | 213 + .../parquet-go/encoding/plain/dictionary.go | 27 + .../parquet-go/encoding/plain/plain.go | 246 + .../parquet-go/encoding/plain/plain_be.go | 113 + .../parquet-go/encoding/plain/plain_le.go | 52 + .../parquet-go/encoding/rle/dictionary.go | 59 + .../parquet-go/parquet-go/encoding/rle/rle.go | 570 + .../parquet-go/encoding/rle/rle_amd64.go | 60 + .../parquet-go/encoding/rle/rle_amd64.s | 174 + .../parquet-go/encoding/rle/rle_purego.go | 22 + .../parquet-go/encoding/thrift/LICENSE | 21 + .../parquet-go/encoding/thrift/binary.go | 369 + .../parquet-go/encoding/thrift/compact.go | 348 + .../parquet-go/encoding/thrift/debug.go | 230 + .../parquet-go/encoding/thrift/decode.go | 689 + .../parquet-go/encoding/thrift/encode.go | 400 + .../parquet-go/encoding/thrift/error.go | 111 + .../parquet-go/encoding/thrift/protocol.go | 73 + .../parquet-go/encoding/thrift/struct.go | 140 + .../parquet-go/encoding/thrift/thrift.go | 164 + .../parquet-go/encoding/thrift/unsafe.go | 20 + .../parquet-go/parquet-go/encoding/values.go | 276 + .../parquet-go/parquet-go/errors.go | 87 + .../github.com/parquet-go/parquet-go/file.go | 1078 + .../parquet-go/parquet-go/filter.go | 82 + .../parquet-go/parquet-go/format/parquet.go | 1230 + .../parquet-go/parquet-go/go.tools.mod | 26 + .../parquet-go/parquet-go/go.tools.sum | 30 + .../parquet-go/hashprobe/aeshash/aeshash.go | 21 + .../hashprobe/aeshash/aeshash_amd64.go | 60 + .../hashprobe/aeshash/aeshash_amd64.s | 155 + .../hashprobe/aeshash/aeshash_purego.go | 29 + .../parquet-go/hashprobe/hashprobe.go | 783 + .../parquet-go/hashprobe/hashprobe_amd64.go | 38 + .../parquet-go/hashprobe/hashprobe_amd64.s | 197 + .../parquet-go/hashprobe/hashprobe_purego.go | 19 + .../parquet-go/hashprobe/wyhash/wyhash.go | 49 + .../hashprobe/wyhash/wyhash_amd64.go | 14 + .../hashprobe/wyhash/wyhash_amd64.s | 118 + .../hashprobe/wyhash/wyhash_purego.go | 23 + .../parquet-go/internal/bitpack/bitpack.go | 8 + .../internal/bitpack/masks_int32_amd64.s | 1288 + .../parquet-go/internal/bitpack/pack.go | 83 + .../parquet-go/internal/bitpack/unpack.go | 27 + .../internal/bitpack/unpack_int32_amd64.go | 36 + .../internal/bitpack/unpack_int32_amd64.s | 352 + .../internal/bitpack/unpack_int32_be.go | 15 + .../internal/bitpack/unpack_int32_le.go | 9 + .../internal/bitpack/unpack_int32_purego.go | 21 + .../internal/bitpack/unpack_int64_amd64.go | 26 + .../internal/bitpack/unpack_int64_amd64.s | 198 + .../internal/bitpack/unpack_int64_purego.go | 25 + .../internal/bytealg/broadcast_amd64.go | 17 + .../internal/bytealg/broadcast_amd64.s | 51 + .../internal/bytealg/broadcast_purego.go | 9 + .../parquet-go/internal/bytealg/bytealg.go | 2 + .../internal/bytealg/bytealg_amd64.go | 17 + .../internal/bytealg/count_amd64.go | 26 + .../parquet-go/internal/bytealg/count_amd64.s | 100 + .../internal/bytealg/count_purego.go | 9 + .../parquet-go/internal/debug/debug.go | 95 + .../internal/debug/finalizer_off.go | 6 + .../parquet-go/internal/debug/finalizer_on.go | 7 + .../internal/unsafecast/unsafecast.go | 54 + .../github.com/parquet-go/parquet-go/level.go | 27 + .../parquet-go/parquet-go/limits.go | 64 + .../github.com/parquet-go/parquet-go/merge.go | 501 + .../parquet-go/parquet-go/multi_row_group.go | 283 + .../github.com/parquet-go/parquet-go/node.go | 546 + .../github.com/parquet-go/parquet-go/null.go | 113 + .../parquet-go/parquet-go/null_amd64.go | 74 + .../parquet-go/parquet-go/null_amd64.s | 227 + .../parquet-go/parquet-go/null_purego.go | 64 + .../parquet-go/parquet-go/offset_index.go | 128 + .../github.com/parquet-go/parquet-go/order.go | 102 + .../parquet-go/parquet-go/order_amd64.go | 21 + .../parquet-go/parquet-go/order_amd64.s | 547 + .../parquet-go/parquet-go/order_purego.go | 42 + .../github.com/parquet-go/parquet-go/page.go | 1453 + .../parquet-go/parquet-go/page_bounds.go | 25 + .../parquet-go/page_bounds_amd64.go | 129 + .../parquet-go/parquet-go/page_bounds_amd64.s | 551 + .../parquet-go/page_bounds_purego.go | 143 + .../parquet-go/parquet-go/page_header.go | 221 + .../parquet-go/parquet-go/page_max.go | 23 + .../parquet-go/parquet-go/page_max_amd64.go | 24 + .../parquet-go/parquet-go/page_max_amd64.s | 598 + .../parquet-go/parquet-go/page_max_purego.go | 110 + .../parquet-go/parquet-go/page_min.go | 23 + .../parquet-go/parquet-go/page_min_amd64.go | 24 + .../parquet-go/parquet-go/page_min_amd64.s | 592 + .../parquet-go/parquet-go/page_min_purego.go | 110 + .../parquet-go/parquet-go/page_values.go | 487 + .../parquet-go/parquet-go/parquet.go | 122 + .../parquet-go/parquet-go/parquet_amd64.go | 18 + .../github.com/parquet-go/parquet-go/print.go | 339 + .../parquet-go/parquet-go/reader.go | 671 + .../github.com/parquet-go/parquet-go/row.go | 848 + .../parquet-go/parquet-go/row_buffer.go | 463 + .../parquet-go/parquet-go/row_builder.go | 202 + .../parquet-go/parquet-go/row_group.go | 498 + .../github.com/parquet-go/parquet-go/scan.go | 33 + .../parquet-go/parquet-go/schema.go | 1082 + .../parquet-go/parquet-go/search.go | 116 + .../parquet-go/parquet-go/sorting.go | 224 + .../parquet-go/parquet-go/sparse/array.go | 320 + .../parquet-go/parquet-go/sparse/gather.go | 37 + .../parquet-go/sparse/gather_amd64.go | 85 + .../parquet-go/sparse/gather_amd64.s | 193 + .../parquet-go/sparse/gather_purego.go | 72 + .../parquet-go/parquet-go/sparse/sparse.go | 20 + .../parquet-go/parquet-go/transform.go | 140 + .../github.com/parquet-go/parquet-go/type.go | 2436 + .../github.com/parquet-go/parquet-go/value.go | 1072 + .../parquet-go/parquet-go/value_amd64.go | 18 + .../parquet-go/parquet-go/value_amd64.s | 59 + .../parquet-go/parquet-go/value_be.go | 19 + .../parquet-go/parquet-go/value_le.go | 8 + .../parquet-go/parquet-go/values_purego.go | 9 + .../parquet-go/parquet-go/writer.go | 1845 + vendor/github.com/pierrec/lz4/v4/.gitignore | 36 + vendor/github.com/pierrec/lz4/v4/LICENSE | 28 + vendor/github.com/pierrec/lz4/v4/README.md | 92 + .../pierrec/lz4/v4/compressing_reader.go | 222 + .../pierrec/lz4/v4/internal/lz4block/block.go | 481 + .../lz4/v4/internal/lz4block/blocks.go | 87 + .../lz4/v4/internal/lz4block/decode_amd64.s | 448 + .../lz4/v4/internal/lz4block/decode_arm.s | 231 + .../lz4/v4/internal/lz4block/decode_arm64.s | 241 + .../lz4/v4/internal/lz4block/decode_asm.go | 10 + .../lz4/v4/internal/lz4block/decode_other.go | 139 + .../lz4/v4/internal/lz4errors/errors.go | 19 + .../lz4/v4/internal/lz4stream/block.go | 348 + .../lz4/v4/internal/lz4stream/frame.go | 204 + .../lz4/v4/internal/lz4stream/frame_gen.go | 103 + .../lz4/v4/internal/xxh32/xxh32zero.go | 212 + .../lz4/v4/internal/xxh32/xxh32zero_arm.go | 11 + .../lz4/v4/internal/xxh32/xxh32zero_arm.s | 251 + .../lz4/v4/internal/xxh32/xxh32zero_other.go | 10 + vendor/github.com/pierrec/lz4/v4/lz4.go | 157 + vendor/github.com/pierrec/lz4/v4/options.go | 242 + .../github.com/pierrec/lz4/v4/options_gen.go | 92 + vendor/github.com/pierrec/lz4/v4/reader.go | 275 + vendor/github.com/pierrec/lz4/v4/state.go | 75 + vendor/github.com/pierrec/lz4/v4/state_gen.go | 28 + vendor/github.com/pierrec/lz4/v4/writer.go | 242 + vendor/github.com/rivo/uniseg/LICENSE.txt | 21 + vendor/github.com/rivo/uniseg/README.md | 137 + vendor/github.com/rivo/uniseg/doc.go | 108 + .../github.com/rivo/uniseg/eastasianwidth.go | 2588 + .../rivo/uniseg/emojipresentation.go | 295 + .../github.com/rivo/uniseg/gen_breaktest.go | 215 + .../github.com/rivo/uniseg/gen_properties.go | 261 + vendor/github.com/rivo/uniseg/grapheme.go | 331 + .../rivo/uniseg/graphemeproperties.go | 1915 + .../github.com/rivo/uniseg/graphemerules.go | 176 + vendor/github.com/rivo/uniseg/line.go | 134 + .../github.com/rivo/uniseg/lineproperties.go | 3554 + vendor/github.com/rivo/uniseg/linerules.go | 626 + vendor/github.com/rivo/uniseg/properties.go | 208 + vendor/github.com/rivo/uniseg/sentence.go | 90 + .../rivo/uniseg/sentenceproperties.go | 2845 + .../github.com/rivo/uniseg/sentencerules.go | 276 + vendor/github.com/rivo/uniseg/step.go | 242 + vendor/github.com/rivo/uniseg/width.go | 61 + vendor/github.com/rivo/uniseg/word.go | 89 + .../github.com/rivo/uniseg/wordproperties.go | 1883 + vendor/github.com/rivo/uniseg/wordrules.go | 282 + vendor/modules.txt | 56 + 380 files changed, 335519 insertions(+), 1 deletion(-) create mode 100644 pkg/compactor/compaction_marker.go create mode 100644 pkg/compactor/parquet_compactor.go create mode 100644 pkg/querier/parquet_blocks_finder_bucket_index.go create mode 100644 pkg/querier/parquet_queryable.go create mode 100644 pkg/storage/parquet/cache.go create mode 100644 pkg/storage/parquet/enc.go create mode 100644 pkg/storage/parquet/enc_test.go create mode 100644 pkg/storage/parquet/prometheus_parquet_chunks_encoder.go create mode 100644 pkg/storage/parquet/reader.go create mode 100644 pkg/storage/parquet/reader_test.go create mode 100644 pkg/storage/parquet/row.go create mode 100644 pkg/storage/parquet/stats.go create mode 100644 pkg/storage/parquet/stats_test.go create mode 100644 pkg/storage/parquet/util_test.go create mode 100644 pkg/storage/parquet/writer.go create mode 100644 pkg/storage/parquet/writer_test.go create mode 100644 pkg/storage/tsdb/bucketindex/parquet_bucket_index.go create mode 100644 pkg/storage/tsdb/bucketindex/parquet_loader.go create mode 100644 vendor/github.com/andybalholm/brotli/LICENSE create mode 100644 vendor/github.com/andybalholm/brotli/README.md create mode 100644 vendor/github.com/andybalholm/brotli/backward_references.go create mode 100644 vendor/github.com/andybalholm/brotli/backward_references_hq.go create mode 100644 vendor/github.com/andybalholm/brotli/bit_cost.go create mode 100644 vendor/github.com/andybalholm/brotli/bit_reader.go create mode 100644 vendor/github.com/andybalholm/brotli/bitwriter.go create mode 100644 vendor/github.com/andybalholm/brotli/block_splitter.go create mode 100644 vendor/github.com/andybalholm/brotli/block_splitter_command.go create mode 100644 vendor/github.com/andybalholm/brotli/block_splitter_distance.go create mode 100644 vendor/github.com/andybalholm/brotli/block_splitter_literal.go create mode 100644 vendor/github.com/andybalholm/brotli/brotli_bit_stream.go create mode 100644 vendor/github.com/andybalholm/brotli/cluster.go create mode 100644 vendor/github.com/andybalholm/brotli/cluster_command.go create mode 100644 vendor/github.com/andybalholm/brotli/cluster_distance.go create mode 100644 vendor/github.com/andybalholm/brotli/cluster_literal.go create mode 100644 vendor/github.com/andybalholm/brotli/command.go create mode 100644 vendor/github.com/andybalholm/brotli/compress_fragment.go create mode 100644 vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go create mode 100644 vendor/github.com/andybalholm/brotli/constants.go create mode 100644 vendor/github.com/andybalholm/brotli/context.go create mode 100644 vendor/github.com/andybalholm/brotli/decode.go create mode 100644 vendor/github.com/andybalholm/brotli/dictionary.go create mode 100644 vendor/github.com/andybalholm/brotli/dictionary_hash.go create mode 100644 vendor/github.com/andybalholm/brotli/encode.go create mode 100644 vendor/github.com/andybalholm/brotli/encoder.go create mode 100644 vendor/github.com/andybalholm/brotli/encoder_dict.go create mode 100644 vendor/github.com/andybalholm/brotli/entropy_encode.go create mode 100644 vendor/github.com/andybalholm/brotli/entropy_encode_static.go create mode 100644 vendor/github.com/andybalholm/brotli/fast_log.go create mode 100644 vendor/github.com/andybalholm/brotli/find_match_length.go create mode 100644 vendor/github.com/andybalholm/brotli/h10.go create mode 100644 vendor/github.com/andybalholm/brotli/h5.go create mode 100644 vendor/github.com/andybalholm/brotli/h6.go create mode 100644 vendor/github.com/andybalholm/brotli/hash.go create mode 100644 vendor/github.com/andybalholm/brotli/hash_composite.go create mode 100644 vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go create mode 100644 vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go create mode 100644 vendor/github.com/andybalholm/brotli/hash_rolling.go create mode 100644 vendor/github.com/andybalholm/brotli/histogram.go create mode 100644 vendor/github.com/andybalholm/brotli/http.go create mode 100644 vendor/github.com/andybalholm/brotli/huffman.go create mode 100644 vendor/github.com/andybalholm/brotli/literal_cost.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/emitter.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/m0.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/m4.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go create mode 100644 vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go create mode 100644 vendor/github.com/andybalholm/brotli/memory.go create mode 100644 vendor/github.com/andybalholm/brotli/metablock.go create mode 100644 vendor/github.com/andybalholm/brotli/metablock_command.go create mode 100644 vendor/github.com/andybalholm/brotli/metablock_distance.go create mode 100644 vendor/github.com/andybalholm/brotli/metablock_literal.go create mode 100644 vendor/github.com/andybalholm/brotli/params.go create mode 100644 vendor/github.com/andybalholm/brotli/platform.go create mode 100644 vendor/github.com/andybalholm/brotli/prefix.go create mode 100644 vendor/github.com/andybalholm/brotli/prefix_dec.go create mode 100644 vendor/github.com/andybalholm/brotli/quality.go create mode 100644 vendor/github.com/andybalholm/brotli/reader.go create mode 100644 vendor/github.com/andybalholm/brotli/ringbuffer.go create mode 100644 vendor/github.com/andybalholm/brotli/state.go create mode 100644 vendor/github.com/andybalholm/brotli/static_dict.go create mode 100644 vendor/github.com/andybalholm/brotli/static_dict_lut.go create mode 100644 vendor/github.com/andybalholm/brotli/symbol_list.go create mode 100644 vendor/github.com/andybalholm/brotli/transform.go create mode 100644 vendor/github.com/andybalholm/brotli/utf8_util.go create mode 100644 vendor/github.com/andybalholm/brotli/util.go create mode 100644 vendor/github.com/andybalholm/brotli/write_bits.go create mode 100644 vendor/github.com/andybalholm/brotli/writer.go create mode 100644 vendor/github.com/axiomhq/hyperloglog/.gitignore create mode 100644 vendor/github.com/axiomhq/hyperloglog/Contributing.md create mode 100644 vendor/github.com/axiomhq/hyperloglog/LICENSE create mode 100644 vendor/github.com/axiomhq/hyperloglog/README.md create mode 100644 vendor/github.com/axiomhq/hyperloglog/beta.go create mode 100644 vendor/github.com/axiomhq/hyperloglog/compressed.go create mode 100644 vendor/github.com/axiomhq/hyperloglog/hyperloglog.go create mode 100644 vendor/github.com/axiomhq/hyperloglog/sparse.go create mode 100644 vendor/github.com/axiomhq/hyperloglog/utils.go create mode 100644 vendor/github.com/kamstrup/intmap/.gitignore create mode 100644 vendor/github.com/kamstrup/intmap/LICENSE create mode 100644 vendor/github.com/kamstrup/intmap/README.md create mode 100644 vendor/github.com/kamstrup/intmap/map64.go create mode 100644 vendor/github.com/kamstrup/intmap/set.go create mode 100644 vendor/github.com/mattn/go-runewidth/LICENSE create mode 100644 vendor/github.com/mattn/go-runewidth/README.md create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_appengine.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_js.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_posix.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_table.go create mode 100644 vendor/github.com/mattn/go-runewidth/runewidth_windows.go create mode 100644 vendor/github.com/olekukonko/tablewriter/.gitignore create mode 100644 vendor/github.com/olekukonko/tablewriter/.travis.yml create mode 100644 vendor/github.com/olekukonko/tablewriter/LICENSE.md create mode 100644 vendor/github.com/olekukonko/tablewriter/README.md create mode 100644 vendor/github.com/olekukonko/tablewriter/csv.go create mode 100644 vendor/github.com/olekukonko/tablewriter/table.go create mode 100644 vendor/github.com/olekukonko/tablewriter/table_with_color.go create mode 100644 vendor/github.com/olekukonko/tablewriter/util.go create mode 100644 vendor/github.com/olekukonko/tablewriter/wrap.go create mode 100644 vendor/github.com/parquet-go/parquet-go/.gitattributes create mode 100644 vendor/github.com/parquet-go/parquet-go/.gitignore create mode 100644 vendor/github.com/parquet-go/parquet-go/.mailmap create mode 100644 vendor/github.com/parquet-go/parquet-go/.words create mode 100644 vendor/github.com/parquet-go/parquet-go/AUTHORS.txt create mode 100644 vendor/github.com/parquet-go/parquet-go/CHANGELOG.md create mode 100644 vendor/github.com/parquet-go/parquet-go/CODEOWNERS create mode 100644 vendor/github.com/parquet-go/parquet-go/CODE_OF_CONDUCT.md create mode 100644 vendor/github.com/parquet-go/parquet-go/CONTRIBUTING.md create mode 100644 vendor/github.com/parquet-go/parquet-go/LICENSE create mode 100644 vendor/github.com/parquet-go/parquet-go/Makefile create mode 100644 vendor/github.com/parquet-go/parquet-go/README.md create mode 100644 vendor/github.com/parquet-go/parquet-go/allocator.go create mode 100644 vendor/github.com/parquet-go/parquet-go/array.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bitmap.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/block.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/block_default.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/block_optimized.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/bloom.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/filter.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/filter_default.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/hash.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/LICENSE create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom_be.go create mode 100644 vendor/github.com/parquet-go/parquet-go/bloom_le.go create mode 100644 vendor/github.com/parquet-go/parquet-go/buffer.go create mode 100644 vendor/github.com/parquet-go/parquet-go/buffer_pool.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_buffer.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/column_buffer_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_chunk.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_index.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_index_be.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_index_le.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_mapping.go create mode 100644 vendor/github.com/parquet-go/parquet-go/column_path.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compare.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/brotli/brotli.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/compress.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/gzip/gzip.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/lz4/lz4.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/snappy/snappy.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/uncompressed/uncompressed.go create mode 100644 vendor/github.com/parquet-go/parquet-go/compress/zstd/zstd.go create mode 100644 vendor/github.com/parquet-go/parquet-go/config.go create mode 100644 vendor/github.com/parquet-go/parquet-go/convert.go create mode 100644 vendor/github.com/parquet-go/parquet-go/dedupe.go create mode 100644 vendor/github.com/parquet-go/parquet-go/deprecated/int96.go create mode 100644 vendor/github.com/parquet-go/parquet-go/deprecated/parquet.go create mode 100644 vendor/github.com/parquet-go/parquet-go/dictionary.go create mode 100644 vendor/github.com/parquet-go/parquet-go/dictionary_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/dictionary_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/dictionary_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/bitpacked/bitpacked.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/delta.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/encoding.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/notsupported.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/plain/dictionary.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/plain/plain.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_be.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_le.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/rle/dictionary.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/rle/rle.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/LICENSE create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/binary.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/compact.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/debug.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/decode.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/encode.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/error.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/protocol.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/struct.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/thrift.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/thrift/unsafe.go create mode 100644 vendor/github.com/parquet-go/parquet-go/encoding/values.go create mode 100644 vendor/github.com/parquet-go/parquet-go/errors.go create mode 100644 vendor/github.com/parquet-go/parquet-go/file.go create mode 100644 vendor/github.com/parquet-go/parquet-go/filter.go create mode 100644 vendor/github.com/parquet-go/parquet-go/format/parquet.go create mode 100644 vendor/github.com/parquet-go/parquet-go/go.tools.mod create mode 100644 vendor/github.com/parquet-go/parquet-go/go.tools.sum create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/bitpack.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/masks_int32_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/pack.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_be.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_le.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/debug/debug.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_off.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_on.go create mode 100644 vendor/github.com/parquet-go/parquet-go/internal/unsafecast/unsafecast.go create mode 100644 vendor/github.com/parquet-go/parquet-go/level.go create mode 100644 vendor/github.com/parquet-go/parquet-go/limits.go create mode 100644 vendor/github.com/parquet-go/parquet-go/merge.go create mode 100644 vendor/github.com/parquet-go/parquet-go/multi_row_group.go create mode 100644 vendor/github.com/parquet-go/parquet-go/node.go create mode 100644 vendor/github.com/parquet-go/parquet-go/null.go create mode 100644 vendor/github.com/parquet-go/parquet-go/null_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/null_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/null_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/offset_index.go create mode 100644 vendor/github.com/parquet-go/parquet-go/order.go create mode 100644 vendor/github.com/parquet-go/parquet-go/order_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/order_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/order_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_bounds.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/page_bounds_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_header.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_max.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_max_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_max_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/page_max_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_min.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_min_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_min_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/page_min_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/page_values.go create mode 100644 vendor/github.com/parquet-go/parquet-go/parquet.go create mode 100644 vendor/github.com/parquet-go/parquet-go/parquet_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/print.go create mode 100644 vendor/github.com/parquet-go/parquet-go/reader.go create mode 100644 vendor/github.com/parquet-go/parquet-go/row.go create mode 100644 vendor/github.com/parquet-go/parquet-go/row_buffer.go create mode 100644 vendor/github.com/parquet-go/parquet-go/row_builder.go create mode 100644 vendor/github.com/parquet-go/parquet-go/row_group.go create mode 100644 vendor/github.com/parquet-go/parquet-go/scan.go create mode 100644 vendor/github.com/parquet-go/parquet-go/schema.go create mode 100644 vendor/github.com/parquet-go/parquet-go/search.go create mode 100644 vendor/github.com/parquet-go/parquet-go/sorting.go create mode 100644 vendor/github.com/parquet-go/parquet-go/sparse/array.go create mode 100644 vendor/github.com/parquet-go/parquet-go/sparse/gather.go create mode 100644 vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/sparse/gather_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/sparse/sparse.go create mode 100644 vendor/github.com/parquet-go/parquet-go/transform.go create mode 100644 vendor/github.com/parquet-go/parquet-go/type.go create mode 100644 vendor/github.com/parquet-go/parquet-go/value.go create mode 100644 vendor/github.com/parquet-go/parquet-go/value_amd64.go create mode 100644 vendor/github.com/parquet-go/parquet-go/value_amd64.s create mode 100644 vendor/github.com/parquet-go/parquet-go/value_be.go create mode 100644 vendor/github.com/parquet-go/parquet-go/value_le.go create mode 100644 vendor/github.com/parquet-go/parquet-go/values_purego.go create mode 100644 vendor/github.com/parquet-go/parquet-go/writer.go create mode 100644 vendor/github.com/pierrec/lz4/v4/.gitignore create mode 100644 vendor/github.com/pierrec/lz4/v4/LICENSE create mode 100644 vendor/github.com/pierrec/lz4/v4/README.md create mode 100644 vendor/github.com/pierrec/lz4/v4/compressing_reader.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm64.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s create mode 100644 vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go create mode 100644 vendor/github.com/pierrec/lz4/v4/lz4.go create mode 100644 vendor/github.com/pierrec/lz4/v4/options.go create mode 100644 vendor/github.com/pierrec/lz4/v4/options_gen.go create mode 100644 vendor/github.com/pierrec/lz4/v4/reader.go create mode 100644 vendor/github.com/pierrec/lz4/v4/state.go create mode 100644 vendor/github.com/pierrec/lz4/v4/state_gen.go create mode 100644 vendor/github.com/pierrec/lz4/v4/writer.go create mode 100644 vendor/github.com/rivo/uniseg/LICENSE.txt create mode 100644 vendor/github.com/rivo/uniseg/README.md create mode 100644 vendor/github.com/rivo/uniseg/doc.go create mode 100644 vendor/github.com/rivo/uniseg/eastasianwidth.go create mode 100644 vendor/github.com/rivo/uniseg/emojipresentation.go create mode 100644 vendor/github.com/rivo/uniseg/gen_breaktest.go create mode 100644 vendor/github.com/rivo/uniseg/gen_properties.go create mode 100644 vendor/github.com/rivo/uniseg/grapheme.go create mode 100644 vendor/github.com/rivo/uniseg/graphemeproperties.go create mode 100644 vendor/github.com/rivo/uniseg/graphemerules.go create mode 100644 vendor/github.com/rivo/uniseg/line.go create mode 100644 vendor/github.com/rivo/uniseg/lineproperties.go create mode 100644 vendor/github.com/rivo/uniseg/linerules.go create mode 100644 vendor/github.com/rivo/uniseg/properties.go create mode 100644 vendor/github.com/rivo/uniseg/sentence.go create mode 100644 vendor/github.com/rivo/uniseg/sentenceproperties.go create mode 100644 vendor/github.com/rivo/uniseg/sentencerules.go create mode 100644 vendor/github.com/rivo/uniseg/step.go create mode 100644 vendor/github.com/rivo/uniseg/width.go create mode 100644 vendor/github.com/rivo/uniseg/word.go create mode 100644 vendor/github.com/rivo/uniseg/wordproperties.go create mode 100644 vendor/github.com/rivo/uniseg/wordrules.go diff --git a/go.mod b/go.mod index 1ca92aab88..02108d2438 100644 --- a/go.mod +++ b/go.mod @@ -76,11 +76,13 @@ require ( require ( github.com/VictoriaMetrics/fastcache v1.12.2 + github.com/axiomhq/hyperloglog v0.2.5 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 github.com/cespare/xxhash/v2 v2.3.0 github.com/google/go-cmp v0.7.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 + github.com/parquet-go/parquet-go v0.25.0 github.com/sercand/kuberesolver/v5 v5.1.1 github.com/tjhop/slog-gokit v0.1.3 go.opentelemetry.io/collector/pdata v1.24.0 @@ -101,6 +103,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2 v1.16.16 // indirect github.com/aws/aws-sdk-go-v2/config v1.15.1 // indirect @@ -171,6 +174,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/kamstrup/intmap v0.5.1 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -178,6 +182,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/socket v0.4.1 // indirect github.com/mdlayher/vsock v1.2.1 // indirect @@ -192,9 +197,11 @@ require ( github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/ncw/swift v1.0.53 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.116.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.116.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus-community/prom-label-proxy v0.8.1-0.20240127162815-c1195f9aabc0 // indirect @@ -202,6 +209,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/sigv4 v0.1.1 // indirect github.com/redis/rueidis v1.0.45-alpha.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/cors v1.11.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect @@ -289,3 +297,5 @@ replace gopkg.in/alecthomas/kingpin.v2 => github.com/alecthomas/kingpin v1.3.8-0 // gRPC 1.66 introduced memory pooling which breaks Cortex queries. Pin 1.65.0 until we have a fix. replace google.golang.org/grpc => google.golang.org/grpc v1.65.0 + +replace github.com/parquet-go/parquet-go => github.com/alanprot/parquet-go v0.25.0-rc.1 diff --git a/go.sum b/go.sum index f4760d779b..7ee901a12e 100644 --- a/go.sum +++ b/go.sum @@ -808,6 +808,8 @@ github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/alanprot/parquet-go v0.25.0-rc.1 h1:dnElcQFd6uksLIIEisC8WOs6TOc8lx/O0pUayZtr2q8= +github.com/alanprot/parquet-go v0.25.0-rc.1/go.mod h1:OqBBRGBl7+llplCvDMql8dEKaDqjaFA/VAPw+OJiNiw= github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497 h1:aDITxVUQ/3KBhpVWX57Vo9ntGTxoRw1F0T6/x/tRzNU= github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497/go.mod h1:b6br6/pDFSfMkBgC96TbpOji05q5pa+v5rIlS0Y6XtI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -828,6 +830,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= @@ -877,6 +881,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9d github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/axiomhq/hyperloglog v0.2.5 h1:Hefy3i8nAs8zAI/tDp+wE7N+Ltr8JnwiW3875pvl0N8= +github.com/axiomhq/hyperloglog v0.2.5/go.mod h1:DLUK9yIzpU5B6YFLjxTIcbHu1g4Y1WQb1m5RH3radaM= github.com/baidubce/bce-sdk-go v0.9.111 h1:yGgtPpZYUZW4uoVorQ4xnuEgVeddACydlcJKW87MDV4= github.com/baidubce/bce-sdk-go v0.9.111/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= @@ -1328,6 +1334,8 @@ github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= github.com/hetznercloud/hcloud-go/v2 v2.18.0 h1:BemrVGeWI8Kn/pvaC1jBsHZxQMnRqOydS7Ju4BERB4Q= github.com/hetznercloud/hcloud-go/v2 v2.18.0/go.mod h1:r5RTzv+qi8IbLcDIskTzxkFIji7Ovc8yNgepQR9M+UA= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible h1:tKTaPHNVwikS3I1rdyf1INNvgJXWSf/+TzqsiGbrgnQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.3+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -1362,6 +1370,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kamstrup/intmap v0.5.1 h1:ENGAowczZA+PJPYYlreoqJvWgQVtAmX1l899WfYFVK0= +github.com/kamstrup/intmap v0.5.1/go.mod h1:gWUVWHKzWj8xpJVFf5GC0O26bWmv3GqdnIX/LMT6Aq4= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= @@ -1442,6 +1452,9 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1509,6 +1522,8 @@ github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os= @@ -1527,7 +1542,7 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing-contrib/go-grpc v0.1.0 h1:9JHDtQXv6UL0tFF8KJB/4ApJgeOcaHp1h07d0PJjESc= github.com/opentracing-contrib/go-grpc v0.1.0/go.mod h1:i3/jx/TvJZ/HKidtT4XGIi/NosUEpzS9xjVJctbKZzI= -github.com/opentracing-contrib/go-stdlib v1.1.0 h1:cZBWc4pA4e65tqTJddbflK435S0tDImj6c9BMvkdUH0= +github.com/opentracing-contrib/go-stdlib v1.1.0 h1:hSJ8yYaiAO/k2YZUeWJWpQCPE2wRCDtxRnir0gU6wbA= github.com/opentracing-contrib/go-stdlib v1.1.0/go.mod h1:S0p+X9p6dcBkoMTL+Qq2VOvxKs9ys5PpYWXWqlCS0bQ= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= @@ -1546,6 +1561,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -1605,6 +1622,9 @@ github.com/redis/rueidis v1.0.45-alpha.1 h1:69Bu1l7gVC/qDYuGGwMwGg2rjOjSyxESol/Z github.com/redis/rueidis v1.0.45-alpha.1/go.mod h1:q7BfhDaPt7xxwy2nv2RqQO12/mmHflDjebpcNwWFjms= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= diff --git a/pkg/compactor/compaction_marker.go b/pkg/compactor/compaction_marker.go new file mode 100644 index 0000000000..1120126660 --- /dev/null +++ b/pkg/compactor/compaction_marker.go @@ -0,0 +1,61 @@ +package compactor + +import ( + "bytes" + "context" + "encoding/json" + "io" + "path" + + "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/efficientgo/core/errors" + "github.com/go-kit/log" + "github.com/oklog/ulid" + "github.com/thanos-io/objstore" + "github.com/thanos-io/thanos/pkg/runutil" +) + +const ( + ParquetCompactionMakerFileName = "parquet-compaction-mark.json" + CurrentVersion = 6 +) + +type CompactionMark struct { + Version int `json:"version"` +} + +func (m *CompactionMark) markerFilename() string { return ParquetCompactionMakerFileName } + +func ReadCompactMark(ctx context.Context, id ulid.ULID, userBkt objstore.InstrumentedBucket, logger log.Logger) (*CompactionMark, error) { + markerPath := path.Join(id.String(), ParquetCompactionMakerFileName) + reader, err := userBkt.WithExpectedErrs(tsdb.IsOneOfTheExpectedErrors(userBkt.IsAccessDeniedErr, userBkt.IsObjNotFoundErr)).Get(ctx, markerPath) + if err != nil { + if userBkt.IsObjNotFoundErr(err) { + return &CompactionMark{}, nil + } + + return &CompactionMark{}, err + } + defer runutil.CloseWithLogOnErr(logger, reader, "close bucket index reader") + + metaContent, err := io.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "read file: %s", ParquetCompactionMakerFileName) + } + + marker := CompactionMark{} + err = json.Unmarshal(metaContent, &marker) + return &marker, err +} + +func WriteCompactMark(ctx context.Context, id ulid.ULID, userBkt objstore.Bucket) error { + marker := CompactionMark{ + Version: CurrentVersion, + } + markerPath := path.Join(id.String(), ParquetCompactionMakerFileName) + b, err := json.Marshal(marker) + if err != nil { + return err + } + return userBkt.Upload(ctx, markerPath, bytes.NewReader(b)) +} diff --git a/pkg/compactor/parquet_compactor.go b/pkg/compactor/parquet_compactor.go new file mode 100644 index 0000000000..3e6683df41 --- /dev/null +++ b/pkg/compactor/parquet_compactor.go @@ -0,0 +1,607 @@ +package compactor + +import ( + "bytes" + "context" + "fmt" + "hash/fnv" + "io" + "os" + "path" + "path/filepath" + "runtime" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/thanos-io/thanos/pkg/block/metadata" + + "github.com/cortexproject/cortex/pkg/ring" + "github.com/cortexproject/cortex/pkg/storage/bucket" + storage "github.com/cortexproject/cortex/pkg/storage/parquet" + cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex" + util_log "github.com/cortexproject/cortex/pkg/util/log" + "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/efficientgo/core/errors" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/oklog/ulid" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/index" + "github.com/thanos-io/objstore" + "github.com/thanos-io/thanos/pkg/block" + "github.com/thanos-io/thanos/pkg/logutil" + "golang.org/x/sync/errgroup" +) + +const ( + batchSize = 50000 + parquetFileName = "block.parquet" + compactorRingKey = "compactor-parquet" + maxParquetIndexSizeLimit = 100 +) + +type ParquetCompactorService struct { + services.Service + + bucket objstore.InstrumentedBucket + loader *bucketindex.Loader + Cfg Config + registerer prometheus.Registerer + logger log.Logger + limits *validation.Overrides + blockRanges []int64 + + // Ring used for sharding compactions. + ringLifecycler *ring.Lifecycler + ring *ring.Ring + ringSubservices *services.Manager + ringSubservicesWatcher *services.FailureWatcher +} + +func NewParquetCompactor(compactorCfg Config, storageCfg cortex_tsdb.BlocksStorageConfig, logger log.Logger, registerer prometheus.Registerer, limits *validation.Overrides, ingestionReplicationFactor int) (*ParquetCompactorService, error) { + bucketClient, err := bucket.NewClient(context.Background(), storageCfg.Bucket, nil, "parquet-compactor", util_log.Logger, prometheus.DefaultRegisterer) + + if err != nil { + return nil, err + } + indexLoaderConfig := bucketindex.LoaderConfig{ + CheckInterval: time.Minute, + UpdateOnStaleInterval: storageCfg.BucketStore.SyncInterval, + UpdateOnErrorInterval: storageCfg.BucketStore.BucketIndex.UpdateOnErrorInterval, + IdleTimeout: storageCfg.BucketStore.BucketIndex.IdleTimeout, + } + + loader := bucketindex.NewLoader(indexLoaderConfig, bucketClient, limits, util_log.Logger, prometheus.DefaultRegisterer) + + c := &ParquetCompactorService{ + Cfg: compactorCfg, + bucket: bucketClient, + loader: loader, + logger: log.With(logger, "component", "compactor"), + registerer: registerer, + blockRanges: compactorCfg.BlockRanges.ToMilliseconds(), + limits: limits, + } + + c.Service = services.NewBasicService(c.starting, c.run, nil) + return c, nil +} + +func (c *ParquetCompactorService) starting(ctx context.Context) error { + var err error + if c.Cfg.ShardingEnabled { + lifecyclerCfg := c.Cfg.ShardingRing.ToLifecyclerConfig() + c.ringLifecycler, err = ring.NewLifecycler(lifecyclerCfg, ring.NewNoopFlushTransferer(), "compactor-parquet", compactorRingKey, true, false, c.logger, prometheus.WrapRegistererWithPrefix("cortex_", c.registerer)) + if err != nil { + return errors.Wrap(err, "unable to initialize compactor ring lifecycler") + } + + c.ring, err = ring.New(lifecyclerCfg.RingConfig, "compactor", compactorRingKey, c.logger, prometheus.WrapRegistererWithPrefix("cortex_", c.registerer)) + if err != nil { + return errors.Wrap(err, "unable to initialize compactor ring") + } + + c.ringSubservices, err = services.NewManager(c.ringLifecycler, c.ring) + if err == nil { + c.ringSubservicesWatcher = services.NewFailureWatcher() + c.ringSubservicesWatcher.WatchManager(c.ringSubservices) + + err = services.StartManagerAndAwaitHealthy(ctx, c.ringSubservices) + } + } + + ctxWithTimeout, cancel := context.WithTimeout(ctx, c.Cfg.ShardingRing.WaitActiveInstanceTimeout) + defer cancel() + if err := ring.WaitInstanceState(ctxWithTimeout, c.ring, c.ringLifecycler.ID, ring.ACTIVE); err != nil { + level.Error(c.logger).Log("msg", "compactor failed to become ACTIVE in the ring", "err", err) + return err + } + level.Info(c.logger).Log("msg", "compactor is ACTIVE in the ring") + if err := c.loader.StartAsync(context.Background()); err != nil { + return errors.Wrap(err, "failed to start loader") + } + + if err := c.loader.AwaitRunning(ctx); err != nil { + return errors.Wrap(err, "failed to start loader") + } + return nil +} + +func (c *ParquetCompactorService) run(ctx context.Context) error { + + go func() { + updateIndexTicker := time.NewTicker(time.Second * 60) + for { + select { + case <-ctx.Done(): + return + case <-updateIndexTicker.C: + users, err := c.discoverUsers(ctx) + if err != nil { + level.Error(util_log.Logger).Log("msg", "Error scanning users", "err", err) + break + } + for _, u := range users { + if ok, _ := c.ownBlock(u); ok { + err := c.updateParquetIndex(ctx, u) + if err != nil { + level.Error(util_log.Logger).Log("msg", "Error updating index", "err", err) + } + } + } + } + } + }() + + t := time.NewTicker(time.Second * 10) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + u, err := c.discoverUsers(ctx) + if err != nil { + level.Error(util_log.Logger).Log("msg", "Error scanning users", "err", err) + return err + } + + for _, u := range u { + uBucket := bucket.NewUserBucketClient(u, c.bucket, c.limits) + + pIdx, err := bucketindex.ReadParquetIndex(ctx, uBucket, util_log.Logger) + if err != nil { + level.Error(util_log.Logger).Log("msg", "Error loading index", "err", err) + break + } + + for { + if ctx.Err() != nil { + return ctx.Err() + } + + level.Info(util_log.Logger).Log("msg", "Scanning User", "user", u) + + idx, _, err := c.loader.GetIndex(ctx, u) + if err != nil { + level.Error(util_log.Logger).Log("msg", "Error loading index", "err", err) + break + } + + level.Info(util_log.Logger).Log("msg", "Loaded index", "user", u, "totalBlocks", len(idx.Blocks), "deleteBlocks", len(idx.BlockDeletionMarks)) + + level.Info(util_log.Logger).Log("msg", "Loaded Parquet index", "user", u, "totalBlocks", len(pIdx.Blocks)) + + ownedBlocks, totalBlocks := c.findNextBlockToCompact(ctx, uBucket, idx, pIdx) + if len(ownedBlocks) == 0 { + level.Info(util_log.Logger).Log("msg", "No owned blocks to compact", "numBlocks", len(pIdx.Blocks), "totalBlocks", totalBlocks) + break + } + + b := ownedBlocks[0] + + if err := os.RemoveAll(c.compactRootDir()); err != nil { + level.Error(util_log.Logger).Log("msg", "failed to remove compaction work directory", "path", c.compactRootDir(), "err", err) + } + bdir := filepath.Join(c.compactDirForUser(u), b.ID.String()) + level.Info(util_log.Logger).Log("msg", "Downloading block", "block", b.ID.String(), "maxTime", b.MaxTime, "dir", bdir, "ownedBlocks", len(ownedBlocks), "totalBlocks", totalBlocks) + if err := block.Download(ctx, util_log.Logger, uBucket, b.ID, bdir, objstore.WithFetchConcurrency(10)); err != nil { + level.Error(util_log.Logger).Log("msg", "Error downloading block", "err", err) + continue + } + err = c.doImport(ctx, b.ID, uBucket, bdir) + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to import block", "block", b.String(), "err", err) + } + // Add the blocks + pIdx.Blocks[b.ID] = bucketindex.BlockWithExtension{Block: b} + } + } + } + } +} + +func (c *ParquetCompactorService) updateParquetIndex(ctx context.Context, u string) error { + level.Info(util_log.Logger).Log("msg", "Updating index", "user", u) + uBucket := bucket.NewUserBucketClient(u, c.bucket, c.limits) + deleted := map[ulid.ULID]struct{}{} + idx, _, err := c.loader.GetIndex(ctx, u) + + if err != nil { + return err + } + + for _, b := range idx.BlockDeletionMarks { + deleted[b.ID] = struct{}{} + } + + pIdx, err := bucketindex.ReadParquetIndex(ctx, uBucket, util_log.Logger) + if err != nil { + return errors.Wrap(err, "failed to read parquet index") + } + + for _, b := range idx.Blocks { + if _, ok := deleted[b.ID]; ok { + continue + } + + if _, ok := pIdx.Blocks[b.ID]; ok { + continue + } + + marker, err := ReadCompactMark(ctx, b.ID, uBucket, c.logger) + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to check if file exists", "err", err) + continue + } + + if marker.Version == CurrentVersion { + m, err := block.DownloadMeta(ctx, util_log.Logger, uBucket, b.ID) + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to download block", "block", b.ID, "err", err) + } + extensions := bucketindex.Extensions{} + _, err = metadata.ConvertExtensions(m.Thanos.Extensions, &extensions) + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to convert extensions", "err", err) + } + pIdx.Blocks[b.ID] = bucketindex.BlockWithExtension{Block: b, Extensions: extensions} + } + } + c.removeDeletedBlocks(idx, pIdx) + //// Remove block from bucket index if marker version is outdated. + //c.removeOutdatedBlocks(ctx, uBucket, pIdx) + return bucketindex.WriteParquetIndex(ctx, uBucket, pIdx) +} + +func (c *ParquetCompactorService) removeDeletedBlocks(idx *bucketindex.Index, pIdx *bucketindex.ParquetIndex) { + blocks := map[ulid.ULID]struct{}{} + deleted := map[ulid.ULID]struct{}{} + + for _, b := range idx.BlockDeletionMarks { + deleted[b.ID] = struct{}{} + } + + for _, b := range idx.Blocks { + if _, ok := deleted[b.ID]; !ok { + blocks[b.ID] = struct{}{} + } + } + + for _, b := range pIdx.Blocks { + if _, ok := blocks[b.ID]; !ok { + delete(pIdx.Blocks, b.ID) + } + } +} + +func (c *ParquetCompactorService) removeOutdatedBlocks(ctx context.Context, uBucket objstore.InstrumentedBucket, pIdx *bucketindex.ParquetIndex) { + for _, b := range pIdx.Blocks { + marker, err := ReadCompactMark(ctx, b.ID, uBucket, c.logger) + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to check if file exists", "err", err) + continue + } + + if marker.Version < CurrentVersion { + delete(pIdx.Blocks, b.ID) + } + } +} + +func (c *ParquetCompactorService) findNextBlockToCompact(ctx context.Context, uBucket objstore.InstrumentedBucket, idx *bucketindex.Index, pIdx *bucketindex.ParquetIndex) ([]*bucketindex.Block, int) { + deleted := map[ulid.ULID]struct{}{} + result := make([]*bucketindex.Block, 0, len(idx.Blocks)) + totalBlocks := 0 + + for _, b := range idx.BlockDeletionMarks { + deleted[b.ID] = struct{}{} + } + + for _, b := range idx.Blocks { + // Do not compact if deleted + if _, ok := deleted[b.ID]; ok { + continue + } + + // Do not compact if is already compacted + if _, ok := pIdx.Blocks[b.ID]; ok { + continue + } + + // Don't try to compact 2h block. Compact 12h and 24h. + if getBlockTimeRange(b, c.blockRanges) == c.blockRanges[0] { + continue + } + + totalBlocks++ + + // Do not compact block if is not owned + if ok, err := c.ownBlock(b.ID.String()); err != nil || !ok { + continue + } + + marker, err := ReadCompactMark(ctx, b.ID, uBucket, c.logger) + + if err == nil && marker.Version == CurrentVersion { + continue + } + + result = append(result, b) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].MinTime > result[j].MinTime + }) + + return result, totalBlocks +} + +func (c *ParquetCompactorService) discoverUsers(ctx context.Context) ([]string, error) { + var users []string + + err := c.bucket.Iter(ctx, "", func(entry string) error { + u := strings.TrimSuffix(entry, "/") + users = append(users, u) + return nil + }) + + return users, err +} + +// compactDirForUser returns the directory to be used to download and compact the blocks for a user +func (c *ParquetCompactorService) compactDirForUser(userID string) string { + return filepath.Join(c.compactRootDir(), userID) +} + +func (c *ParquetCompactorService) compactRootDir() string { + return filepath.Join(c.Cfg.DataDir, "compact") +} + +func (c *ParquetCompactorService) ownBlock(blockId string) (bool, error) { + // Hash the user ID. + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(blockId)) + userHash := hasher.Sum32() + + // Check whether this compactor instance owns the user. + rs, err := c.ring.Get(userHash, RingOp, nil, nil, nil) + if err != nil { + return false, err + } + + if len(rs.Instances) != 1 { + return false, fmt.Errorf("unexpected number of compactors in the shard (expected 1, got %d)", len(rs.Instances)) + } + + return rs.Instances[0].Addr == c.ringLifecycler.Addr, nil +} + +func (c *ParquetCompactorService) doImport(ctx context.Context, id ulid.ULID, bucket objstore.Bucket, bDir string) (err error) { + r, w := io.Pipe() + + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + defer wg.Done() + defer r.Close() + err = bucket.Upload(context.Background(), path.Join(id.String(), parquetFileName), r) + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to upload file", "err", err) + } + if err == nil { + err = WriteCompactMark(ctx, id, bucket) + } + }() + + sc, ln, totalMetrics, err := c.readTsdb(ctx, bDir) + if err != nil { + return err + } + + writer := storage.NewParquetWriter(w, 1e6, maxParquetIndexSizeLimit, storage.ChunkColumnsPerDay, ln, labels.MetricName) + if err != nil { + return err + } + + total := 0 + for b := range sc { + if ctx.Err() != nil { + err = ctx.Err() + } + fmt.Printf("Writing Metrics [%v] [%v]\n", 100*(float64(total)/float64(totalMetrics)), b[0].Columns[labels.MetricName]) + writer.WriteRows(b) + total += len(b) + } + + if e := writer.Close(); err == nil { + err = e + } + + if e := w.Close(); err == nil { + err = e + } + + wg.Wait() + return err +} + +func (c *ParquetCompactorService) readTsdb(ctx context.Context, path string) (chan []storage.ParquetRow, []string, int, error) { + b, err := tsdb.OpenBlock(logutil.GoKitLogToSlog(c.logger), path, nil, tsdb.DefaultPostingsDecoderFactory) + if err != nil { + return nil, nil, 0, err + } + idx, err := b.Index() + if err != nil { + return nil, nil, 0, err + } + + cReader, err := b.Chunks() + if err != nil { + return nil, nil, 0, err + } + + metricNames, err := idx.LabelValues(ctx, labels.MetricName) + if err != nil { + return nil, nil, 0, err + } + k, v := index.AllPostingsKey() + all, err := idx.Postings(ctx, k, v) + if err != nil { + return nil, nil, 0, err + } + + total := 0 + for all.Next() { + total++ + } + + labelNames, err := idx.LabelNames(ctx) + if err != nil { + return nil, nil, 0, err + } + + slices.SortFunc(metricNames, func(a, b string) int { + return bytes.Compare( + truncateByteArray([]byte(a), maxParquetIndexSizeLimit), + truncateByteArray([]byte(b), maxParquetIndexSizeLimit), + ) + }) + rc := make(chan []storage.ParquetRow, 10) + chunksEncoder := storage.NewPrometheusParquetChunksEncoder() + + go func() { + defer close(rc) + batch := make([]storage.ParquetRow, 0, batchSize) + batchMutex := &sync.Mutex{} + for _, metricName := range metricNames { + if ctx.Err() != nil { + return + } + p, err := idx.Postings(ctx, labels.MetricName, metricName) + if err != nil { + return + } + eg := &errgroup.Group{} + eg.SetLimit(runtime.GOMAXPROCS(0)) + + for p.Next() { + chks := []chunks.Meta{} + builder := labels.ScratchBuilder{} + + at := p.At() + err = idx.Series(at, &builder, &chks) + if err != nil { + return + } + eg.Go(func() error { + for i := range chks { + chks[i].Chunk, _, err = cReader.ChunkOrIterable(chks[i]) + if err != nil { + return err + } + } + + data, err := chunksEncoder.Encode(chks) + if err != nil { + return err + } + + promLbls := builder.Labels() + lbsls := make(map[string]string) + + promLbls.Range(func(l labels.Label) { + lbsls[l.Name] = l.Value + }) + + row := storage.ParquetRow{ + Hash: promLbls.Hash(), + Columns: lbsls, + Data: data, + } + + batchMutex.Lock() + batch = append(batch, row) + if len(batch) >= batchSize { + rc <- batch + batch = make([]storage.ParquetRow, 0, batchSize) + } + batchMutex.Unlock() + return nil + }) + } + + err = eg.Wait() + if err != nil { + level.Error(util_log.Logger).Log("msg", "failed to process chunk", "err", err) + return + } + } + if len(batch) > 0 { + rc <- batch + } + }() + + return rc, labelNames, total, nil +} + +func getBlockTimeRange(b *bucketindex.Block, timeRanges []int64) int64 { + timeRange := int64(0) + // fallback logic to guess block time range based + // on MaxTime and MinTime + blockRange := b.MaxTime - b.MinTime + for _, tr := range timeRanges { + rangeStart := getStart(b.MinTime, tr) + rangeEnd := rangeStart + tr + if tr >= blockRange && rangeEnd >= b.MaxTime { + timeRange = tr + break + } + } + return timeRange +} + +func getStart(mint int64, tr int64) int64 { + // Compute start of aligned time range of size tr closest to the current block's start. + // This code has been copied from TSDB. + if mint >= 0 { + return tr * (mint / tr) + } + + return tr * ((mint - tr + 1) / tr) +} + +func truncateByteArray(value []byte, sizeLimit int) []byte { + if len(value) > sizeLimit { + value = value[:sizeLimit] + } + return value +} diff --git a/pkg/querier/parquet_blocks_finder_bucket_index.go b/pkg/querier/parquet_blocks_finder_bucket_index.go new file mode 100644 index 0000000000..189bcce2fb --- /dev/null +++ b/pkg/querier/parquet_blocks_finder_bucket_index.go @@ -0,0 +1,139 @@ +package querier + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/go-kit/log" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/thanos-io/objstore" + + "github.com/cespare/xxhash/v2" + "github.com/cortexproject/cortex/pkg/storage/bucket" + "github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex" + "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/validation" + "github.com/prometheus/prometheus/model/labels" +) + +type finderMetrics struct { + blocksSkipped prometheus.Counter +} + +func newFinderMetrics(reg prometheus.Registerer) *finderMetrics { + return &finderMetrics{ + blocksSkipped: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "cortex_parquet_blocks_finder_skipped_total", + Help: "total number of blocks skipped", + }), + } +} + +type ParquetBucketIndexBlocksFinderConfig struct { + IndexLoader bucketindex.LoaderConfig + MaxStalePeriod time.Duration + IgnoreDeletionMarksDelay time.Duration + IgnoreBlocksWithin time.Duration +} + +// ParquetBucketIndexBlocksFinder implements BlocksFinder interface and find blocks in the bucket +// looking up the bucket index. +type ParquetBucketIndexBlocksFinder struct { + services.Service + + cfg BucketIndexBlocksFinderConfig + loader *bucketindex.ParquetLoader + m *finderMetrics +} + +func NewParquetBucketIndexBlocksFinder(cfg BucketIndexBlocksFinderConfig, bkt objstore.Bucket, cfgProvider bucket.TenantConfigProvider, logger log.Logger, reg prometheus.Registerer) *ParquetBucketIndexBlocksFinder { + loader := bucketindex.NewLParquetLoader(cfg.IndexLoader, bkt, cfgProvider, logger, reg) + + return &ParquetBucketIndexBlocksFinder{ + cfg: cfg, + Service: loader, + loader: loader, + m: newFinderMetrics(reg), + } +} + +// GetBlocks implements BlocksFinder. +func (f *ParquetBucketIndexBlocksFinder) GetBlocks(ctx context.Context, userID string, minT, maxT int64, matchers []*labels.Matcher) (bucketindex.Blocks, map[ulid.ULID]*bucketindex.BlockDeletionMark, error) { + if f.State() != services.Running { + return nil, nil, errBucketIndexBlocksFinderNotRunning + } + if maxT < minT { + return nil, nil, errInvalidBlocksRange + } + + // Get the bucket index for this user. + idx, ss, err := f.loader.GetIndex(ctx, userID) + if errors.Is(err, bucketindex.ErrIndexNotFound) { + // This is a legit edge case, happening when a new tenant has not shipped blocks to the storage yet + // so the bucket index hasn't been created yet. + return nil, nil, nil + } else if errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) { + return nil, nil, validation.AccessDeniedError(err.Error()) + } + + // Short circuit when bucket failed to be updated due CMK errors recently + if time.Since(ss.GetNonQueryableUntil()) < 0 && ss.NonQueryableReason == bucketindex.CustomerManagedKeyError { + return nil, nil, validation.AccessDeniedError(bucket.ErrCustomerManagedKeyAccessDenied.Error()) + } + + if err != nil { + return nil, nil, err + } + + //// Ensure the bucket index is not too old. + //if time.Since(idx.GetUpdatedAt()) > f.cfg.MaxStalePeriod { + // return nil, nil, errBucketIndexTooOld + //} + + var ( + matchingBlocks = map[ulid.ULID]*bucketindex.Block{} + matchingDeletionMarks = map[ulid.ULID]*bucketindex.BlockDeletionMark{} + ) + + // Filter blocks containing samples within the range. + for _, block := range idx.Blocks { + if !block.Within(minT, maxT) { + continue + } + + matchingBlocks[block.ID] = block.Block + } + + //for _, mark := range idx.BlockDeletionMarks { + // // Filter deletion marks by matching blocks only. + // if _, ok := matchingBlocks[mark.ID]; !ok { + // continue + // } + // + // // Exclude blocks marked for deletion. This is the same logic as Thanos IgnoreDeletionMarkFilter. + // if time.Since(time.Unix(mark.DeletionTime, 0)).Seconds() > f.cfg.IgnoreDeletionMarksDelay.Seconds() { + // delete(matchingBlocks, mark.ID) + // continue + // } + // + // matchingDeletionMarks[mark.ID] = mark + //} + + // Convert matching blocks into a list. + blocks := make(bucketindex.Blocks, 0, len(matchingBlocks)) + for _, b := range matchingBlocks { + blocks = append(blocks, b) + } + + return blocks, matchingDeletionMarks, nil +} + +func hash(s string) uint64 { + h := xxhash.New() + _, _ = h.Write([]byte(s)) + return h.Sum64() +} diff --git a/pkg/querier/parquet_queryable.go b/pkg/querier/parquet_queryable.go new file mode 100644 index 0000000000..27eb76b9a2 --- /dev/null +++ b/pkg/querier/parquet_queryable.go @@ -0,0 +1,1041 @@ +package querier + +import ( + "context" + "io" + "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/opentracing/opentracing-go" + "github.com/parquet-go/parquet-go" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/util/annotations" + "github.com/thanos-io/objstore" + "github.com/thanos-io/thanos/pkg/dedup" + "github.com/thanos-io/thanos/pkg/strutil" + "github.com/weaveworks/common/instrument" + "golang.org/x/sync/errgroup" + + "github.com/cortexproject/cortex/pkg/cortexpb" + "github.com/cortexproject/cortex/pkg/querier/stats" + "github.com/cortexproject/cortex/pkg/storage/bucket" + cortex_storage "github.com/cortexproject/cortex/pkg/storage/parquet" + cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex" + "github.com/cortexproject/cortex/pkg/tenant" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/limiter" + util_log "github.com/cortexproject/cortex/pkg/util/log" + "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/spanlogger" + "github.com/cortexproject/cortex/pkg/util/validation" +) + +const ( + parquetFileName = "block.parquet" + ModuleName = "parquet-queryable" + seriesHash = "__series_hash__" +) + +type ParquetQueryable struct { + services.Service + + finder *ParquetBucketIndexBlocksFinder + logger log.Logger + queryStoreAfter time.Duration + queryIngestersWithin time.Duration + limits BlocksStoreLimits + bucket objstore.InstrumentedBucket + chunkDecoder *cortex_storage.PrometheusParquetChunksDecoder + asyncRead bool + dictionaryCacheSize int + projectionPushdown bool + + metrics *metrics + + // Subservices manager. + subservices *services.Manager + subservicesWatcher *services.FailureWatcher + + readerCache *cortex_storage.Cache[*cortex_storage.ParquetReader] + cacheMetrics *cortex_storage.CacheMetrics +} + +type metrics struct { + parquetFileOpenDuration prometheus.Histogram + pagesSkippedPageIndex *prometheus.CounterVec + projectionPushdown prometheus.Counter + dataFetchDuration *prometheus.HistogramVec +} + +func newMetrics(reg prometheus.Registerer) *metrics { + return &metrics{ + parquetFileOpenDuration: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_parquet_querier_parquet_file_open_duration_seconds", + Help: "native histogram for parquet file open latency", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }), + pagesSkippedPageIndex: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_parquet_querier_pages_skipped_page_index_total", + Help: "total number of pages skipped using page index for each column", + }, []string{"column"}), + projectionPushdown: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "cortex_parquet_querier_projection_pushdowns_total", + Help: "total number of projection pushdowns", + }), + dataFetchDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "cortex_parquet_querier_data_fetch_duration", + Help: "Time spent to fetch data", + Buckets: instrument.DefBuckets, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: time.Hour, + }, []string{"method"}), + } +} + +func NewParquetStoreQueryable( + limits BlocksStoreLimits, + config Config, + storageCfg cortex_tsdb.BlocksStorageConfig, + logger log.Logger, + reg prometheus.Registerer, +) (*ParquetQueryable, error) { + bucketClient, err := bucket.NewClient(context.Background(), storageCfg.Bucket, nil, ModuleName, logger, reg) + if err != nil { + return nil, err + } + + cacheMetrics := cortex_storage.NewCacheMetrics(reg) + cache, err := cortex_storage.NewCache[*cortex_storage.ParquetReader]("files", 1000, cacheMetrics) + if err != nil { + return nil, err + } + + indexLoaderConfig := bucketindex.LoaderConfig{ + CheckInterval: time.Minute, + UpdateOnStaleInterval: storageCfg.BucketStore.SyncInterval, + UpdateOnErrorInterval: storageCfg.BucketStore.BucketIndex.UpdateOnErrorInterval, + IdleTimeout: storageCfg.BucketStore.BucketIndex.IdleTimeout, + } + bucketIndexBlocksFinder := BucketIndexBlocksFinderConfig{ + IndexLoader: indexLoaderConfig, + } + finder := NewParquetBucketIndexBlocksFinder(bucketIndexBlocksFinder, bucketClient, limits, util_log.Logger, prometheus.DefaultRegisterer) + manager, err := services.NewManager(finder) + + q := &ParquetQueryable{ + bucket: bucketClient, + finder: finder, + queryStoreAfter: config.QueryStoreAfter, + queryIngestersWithin: config.QueryIngestersWithin, + logger: logger, + subservices: manager, + subservicesWatcher: services.NewFailureWatcher(), + limits: limits, + chunkDecoder: cortex_storage.NewPrometheusParquetChunksDecoder(), + asyncRead: true, + dictionaryCacheSize: 1024, + metrics: newMetrics(reg), + readerCache: cache, + cacheMetrics: cacheMetrics, + projectionPushdown: false, + } + + q.Service = services.NewBasicService(q.starting, q.running, q.stopping) + + return q, nil +} + +func (q *ParquetQueryable) starting(ctx context.Context) error { + q.subservicesWatcher.WatchManager(q.subservices) + + if err := services.StartManagerAndAwaitHealthy(ctx, q.subservices); err != nil { + return errors.Wrap(err, "unable to start blocks storage queryable subservices") + } + + return nil +} + +func (q *ParquetQueryable) running(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + case err := <-q.subservicesWatcher.Chan(): + return errors.Wrap(err, "block storage queryable subservice failed") + } + } +} + +func (q *ParquetQueryable) stopping(_ error) error { + return services.StopManagerAndAwaitStopped(context.Background(), q.subservices) +} + +// Querier returns a new Querier on the storage. +func (q *ParquetQueryable) Querier(mint, maxt int64) (storage.Querier, error) { + return &parquetQuerier{ + minT: mint, + maxT: maxt, + finder: q.finder, + limits: q.limits, + logger: q.logger, + queryStoreAfter: q.queryStoreAfter, + queryIngestersWithin: q.queryIngestersWithin, + bucket: q.bucket, + chunkDecoder: q.chunkDecoder, + asyncRead: q.asyncRead, + dictionaryCacheSize: q.dictionaryCacheSize, + metrics: q.metrics, + readerCache: q.readerCache, + cacheMetrics: q.cacheMetrics, + projectionPushdown: q.projectionPushdown, + }, nil +} + +type parquetQuerier struct { + minT, maxT int64 + finder *ParquetBucketIndexBlocksFinder + limits BlocksStoreLimits + logger log.Logger + bucket objstore.InstrumentedBucket + + metrics *metrics + + // If set, the querier manipulates the max time to not be greater than + // "now - queryStoreAfter" so that most recent blocks are not queried. + queryStoreAfter time.Duration + queryIngestersWithin time.Duration + asyncRead bool + dictionaryCacheSize int + projectionPushdown bool + + readerCache *cortex_storage.Cache[*cortex_storage.ParquetReader] + cacheMetrics *cortex_storage.CacheMetrics + + chunkDecoder *cortex_storage.PrometheusParquetChunksDecoder +} + +func (q *parquetQuerier) findBlocks(ctx context.Context, userID string, logger log.Logger, minT, maxT int64, matchers []*labels.Matcher) (bucketindex.Blocks, error) { + // If queryStoreAfter is enabled, we do manipulate the query maxt to query samples up until + // now - queryStoreAfter, because the most recent time range is covered by ingesters. This + // optimization is particularly important for the blocks storage because can be used to skip + // querying most recent not-compacted-yet blocks from the storage. + if q.queryStoreAfter > 0 { + now := time.Now() + origMaxT := maxT + maxT = min(maxT, util.TimeToMillis(now.Add(-q.queryStoreAfter))) + + if origMaxT != maxT { + level.Debug(logger).Log("msg", "the max time of the query to blocks storage has been manipulated", "original", origMaxT, "updated", maxT) + } + + if maxT < minT { + return nil, nil + } + } + // No deleted block for now. + blocks, _, err := q.finder.GetBlocks(ctx, userID, minT, maxT, matchers) + return blocks, err +} + +// Select implements storage.Querier interface. +// The bool passed is ignored because the series is always sorted. +func (q *parquetQuerier) Select(ctx context.Context, _ bool, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + n := time.Now() + ss := q.selectSorted(ctx, sp, matchers...) + q.metrics.dataFetchDuration.WithLabelValues("select").Observe(time.Since(n).Seconds()) + return ss +} + +func (q *parquetQuerier) searchWithMatchers(ctx context.Context, reader *cortex_storage.ParquetReader, sts *cortex_storage.Stats, matchers ...*labels.Matcher) ([][]int64, error) { + start := time.Now() + defer func() { + sts.AddSearchWallTime(time.Since(start).Milliseconds()) + }() + span, ctx := opentracing.StartSpanFromContext(ctx, "parquetQuerier.searchWithMatchers") + defer span.Finish() + nameMatcher, matchers := equalNameMatcher(matchers...) + var ( + rows [][]int64 + err error + fullScanned bool + ) + if nameMatcher != nil { + rows, err = reader.SearchRows(ctx, nameMatcher.Name, nameMatcher.Value, sts) + if err != nil { + return nil, err + } + sts.AddColumnSearched(nameMatcher.Name, len(rows)) + + if len(rows) == 0 { + return nil, nil + } + fullScanned = true + } + + for _, matcher := range matchers { + // .* regexp matches any string. + if matcher.Type == labels.MatchRegexp && matcher.Value == ".*" { + sts.AddColumnSearched(matcher.Name, len(rows)) + continue + } + + // .* not regexp doesn't match any string. + // TODO: skip such matcher earlier. + if matcher.Type == labels.MatchNotRegexp && matcher.Value == ".*" { + sts.AddColumnSearched(matcher.Name, 0) + return nil, nil + } + + // Only full scan if we never full scan before. + rows, err = reader.ScanRows(ctx, rows, !fullScanned, matcher, sts) + if err != nil { + return nil, err + } + sts.AddColumnSearched(matcher.Name, len(rows)) + if len(rows) == 0 { + return nil, nil + } + + if !fullScanned { + fullScanned = true + } + } + return rows, nil +} + +func (q *parquetQuerier) selectSorted(ctx context.Context, sp *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { + userID, err := tenant.TenantID(ctx) + if err != nil { + return storage.ErrSeriesSet(err) + } + + if len(matchers) == 0 { + return storage.ErrSeriesSet(errors.New("no matchers")) + } + + spanLog, ctx := spanlogger.New(ctx, "ParquetQuerier.selectSorted") + defer spanLog.Span.Finish() + requestStart := time.Now() + + var ( + queryLimiter = limiter.QueryLimiterFromContextWithFallback(ctx) + reqStats = stats.FromContext(ctx) + ) + + minT, maxT, limit := q.minT, q.maxT, int64(0) + if sp != nil { + minT, maxT, limit = sp.Start, sp.End, int64(sp.Limit) + } + spanLog.SetTag("minT", minT) + spanLog.SetTag("maxT", maxT) + skipChunks := sp != nil && sp.Func == "series" + // Only pushdown if we know in the we are not querying ingesters. Add 1 hour buffer to avoid accidental overalp. + queryIngesters := q.queryIngestersWithin == 0 || maxT >= util.TimeToMillis(time.Now().Add(-q.queryIngestersWithin).Add(-time.Hour)) + + var ( + grouping []string + by bool + ) + // For now, we simplify things and only think about pruning columns if not querying ingesters + projectionPushdown := false + if q.projectionPushdown && !queryIngesters && sp != nil { + projectionPushdown = true + grouping = sp.Grouping + by = sp.By + } + + var ( + resSeriesSets = []storage.SeriesSet(nil) + resultMtx sync.Mutex + ) + + sts := cortex_storage.NewStats() + + blocks, err := q.findBlocks(ctx, userID, q.logger, minT, maxT, matchers) + spanLog.SetTag("blocks", len(blocks)) + + if err != nil { + return storage.ErrSeriesSet(err) + } + if len(blocks) == 0 { + return storage.EmptySeriesSet() + } + prs, err := q.blocksToParquetReader(ctx, userID, blocks, sts) + if err != nil { + return storage.ErrSeriesSet(err) + } + + defer func() { + sts.AddRequestDuration(time.Since(requestStart).Milliseconds()) + extraFields := []interface{}{ + "mint", minT, + "maxt", maxT, + "matchers", util.LabelMatchersToString(matchers), + "projectionPushdown", projectionPushdown, + "by", by, + "step", sp.Step, + "range", sp.Range, + "func", sp.Func, + "fetchedSeries", reqStats.LoadFetchedSeries(), + "fetchedChunks", reqStats.LoadFetchedChunks(), + "fetchedChunkBytes", reqStats.LoadFetchedChunkBytes(), + "fetchedSamples", reqStats.LoadFetchedSamples(), + "fetchedDataBytes", reqStats.LoadFetchedDataBytes(), + } + if len(grouping) > 0 { + extraFields = append(extraFields, "grouping", strings.Join(grouping, ",")) + } + sts.Display(spanLog, extraFields...) + }() + + eg := &errgroup.Group{} + for i, reader := range prs { + if reader == nil { + continue + } + sts.AddFilesScanned(1) + b := blocks[i] + reader := reader + eg.Go(func() error { + start := time.Now() + defer func() { + sts.AddStorageWallTime(time.Since(start).Milliseconds()) + }() + rows, err := q.searchWithMatchers(ctx, reader, sts, matchers...) + if err != nil { + level.Error(q.logger).Log("msg", "error querying series", "err", err, "block", b.ID) + return errors.Wrapf(err, "block id %s", b.ID) + } + dataColsIdx := q.getDataColsIdx(skipChunks, reader.DataColsSize(), b) + prs, err := reader.Materialize(ctx, rows, dataColsIdx, grouping, by, sts) + if err != nil { + level.Error(q.logger).Log("msg", "error materializing series", "err", err, "block", b.ID) + return errors.Wrapf(err, "block id %s", b.ID) + } + sts.AddRowsFetched(len(prs)) + ss, err := newParquetRowsSeriesSet(true, prs, q.minT, q.maxT, skipChunks, q.chunkDecoder, projectionPushdown && by, queryLimiter, reqStats, sts) + if err != nil { + return err + } + resultMtx.Lock() + resSeriesSets = append(resSeriesSets, ss) + defer resultMtx.Unlock() + return nil + return nil + }) + } + if err := eg.Wait(); err != nil { + return storage.ErrSeriesSet(err) + } + + if len(resSeriesSets) == 0 { + return storage.EmptySeriesSet() + } + + return storage.NewMergeSeriesSet(resSeriesSets, int(limit), storage.ChainedSeriesMerge) +} + +func (q *parquetQuerier) getDataColsIdx(skipChunks bool, dataCols int, b *bucketindex.Block) []int { + if skipChunks { + return []int{} + } + idxInit := 0 + idxEnd := dataCols - 1 + dataColsIdx := make([]int, 0, dataCols) + dataColLength := time.Hour * 24 / time.Duration(dataCols) + + if q.minT > b.MinTime { + hour := time.UnixMilli(q.minT).UTC().Hour() + + idxInit = (hour / int(dataColLength.Hours())) % dataCols + } + + if q.maxT < b.MaxTime { + hour := time.UnixMilli(q.maxT).UTC().Hour() + idxEnd = (hour / int(dataColLength.Hours())) % dataCols + } + + for i := idxInit; i <= idxEnd; i++ { + dataColsIdx = append(dataColsIdx, i) + } + return dataColsIdx +} + +func equalNameMatcher(matchers ...*labels.Matcher) (*labels.Matcher, []*labels.Matcher) { + others := make([]*labels.Matcher, 0, len(matchers)) + var metricNameMatcher *labels.Matcher + for _, m := range matchers { + if m.Name == labels.MetricName && m.Type == labels.MatchEqual { + metricNameMatcher = m + } else { + others = append(others, m) + } + } + return metricNameMatcher, others +} + +func (q *parquetQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, nil, err + } + + spanLog, ctx := spanlogger.New(ctx, "ParquetQuerier.LabelNames") + defer spanLog.Span.Finish() + + minT, maxT, limit := q.minT, q.maxT, int64(0) + + if hints != nil { + limit = int64(hints.Limit) + } + + var ( + resMtx sync.Mutex + resNameSets = [][]string{} + resWarnings = annotations.Annotations(nil) + ) + sts := cortex_storage.NewStats() + defer func() { + extraFields := []interface{}{ + "mint", minT, + "maxt", maxT, + "matchers", util.LabelMatchersToString(matchers), + } + sts.Display(spanLog, extraFields...) + }() + + blocks, err := q.findBlocks(ctx, userID, q.logger, minT, maxT, matchers) + if err != nil { + return nil, nil, err + } + + prs, err := q.blocksToParquetReader(ctx, userID, blocks, sts) + if err != nil { + return nil, nil, err + } + eg := &errgroup.Group{} + // Happy path. + for _, reader := range prs { + if reader == nil { + continue + } + sts.AddFilesScanned(1) + eg.Go(func() error { + if len(matchers) == 0 { + resMtx.Lock() + resNameSets = append(resNameSets, reader.ColumnNames()) + defer resMtx.Unlock() + } else { + rows, err := q.searchWithMatchers(ctx, reader, sts, matchers...) + if err != nil { + return err + } + out, err := reader.MaterializeColumnNames(ctx, rows, sts) + if err != nil { + return err + } + resMtx.Lock() + resNameSets = append(resNameSets, out) + defer resMtx.Unlock() + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, nil, err + } + + return strutil.MergeSlices(int(limit), resNameSets...), resWarnings, nil +} + +func (q *parquetQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, nil, err + } + + spanLog, ctx := spanlogger.New(ctx, "ParquetQuerier.LabelValues") + defer spanLog.Span.Finish() + + minT, maxT, limit := q.minT, q.maxT, int64(0) + + if hints != nil { + limit = int64(hints.Limit) + } + + var ( + resValueSets = [][]string{} + resWarnings = annotations.Annotations(nil) + + resultMtx sync.Mutex + ) + + sts := cortex_storage.NewStats() + defer func() { + extraFields := []interface{}{ + "name", name, + "mint", minT, + "maxt", maxT, + "matchers", util.LabelMatchersToString(matchers), + } + sts.Display(spanLog, extraFields...) + }() + + blocks, err := q.findBlocks(ctx, userID, q.logger, minT, maxT, matchers) + if err != nil { + return nil, nil, err + } + + prs, err := q.blocksToParquetReader(ctx, userID, blocks, sts) + if err != nil { + return nil, nil, err + } + eg := &errgroup.Group{} + // Happy path. + for _, reader := range prs { + if reader == nil { + continue + } + sts.AddFilesScanned(1) + eg.Go(func() error { + if !reader.ColumnExists(name) { + return nil + } + + var ( + values []string + err error + ) + if len(matchers) == 0 { + values, err = reader.ColumnValues(ctx, name) + if err != nil { + return err + } + } else { + rows, err := q.searchWithMatchers(ctx, reader, sts, matchers...) + if err != nil { + return err + } + vals, err := reader.MaterializeColumn(ctx, rows, sts, name) + if err != nil { + return err + } + values = parquetValuesToStrings(vals[0]) + } + + resultMtx.Lock() + resValueSets = append(resValueSets, values) + defer resultMtx.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, nil, err + } + + return strutil.MergeSlices(int(limit), resValueSets...), resWarnings, nil +} + +func (q *parquetQuerier) Close() error { + return nil +} + +type bReadAt struct { + path string + obj objstore.InstrumentedBucketReader + ctx context.Context + + m *metrics +} + +func (b *bReadAt) ReadAt(p []byte, off int64) (n int, err error) { + rc, err := b.obj.GetRange(b.ctx, b.path, off, int64(len(p))) + if err != nil { + return 0, err + } + defer rc.Close() + n, err = rc.Read(p) + if err == io.EOF { + err = nil + } + return +} + +func (b *bReadAt) CreateReadAtWithContext(ctx context.Context) io.ReaderAt { + return &bReadAt{ + path: b.path, + obj: b.obj, + m: b.m, + ctx: ctx, + } +} + +func (q *parquetQuerier) blocksToParquetReader(ctx context.Context, uname string, blocks bucketindex.Blocks, sts *cortex_storage.Stats) ([]*cortex_storage.ParquetReader, error) { + start := time.Now() + defer func() { + sts.AddOpenParquetReaderDuration(time.Since(start).Milliseconds()) + }() + + span, _ := opentracing.StartSpanFromContext(ctx, "parquetQuerier.blocksToParquetReader") + defer span.Finish() + output := make([]*cortex_storage.ParquetReader, len(blocks)) + mtx := &sync.Mutex{} + eg := &errgroup.Group{} + readerCacheMiss := 0 + defer func() { + sts.AddParquetReaderCacheMiss(readerCacheMiss) + }() + for i, block := range blocks { + id := block.ID.String() + name := path.Join(uname, id, parquetFileName) + + // Try to get from cache. + if reader := q.readerCache.Get(name); reader != nil { + output[i] = reader + continue + } + readerCacheMiss++ + + eg.Go(func() error { + attr, err := q.bucket.Attributes(ctx, name) + if err != nil { + if q.bucket.IsObjNotFoundErr(err) { + return nil + } + return err + } + t := time.Now() + r := &bReadAt{path: name, obj: q.bucket, m: q.metrics} + pr, err := cortex_storage.NewParquetReader(r.CreateReadAtWithContext, attr.Size, q.asyncRead, q.cacheMetrics, q.dictionaryCacheSize, q.metrics.pagesSkippedPageIndex, q.metrics.projectionPushdown) + if err != nil { + return err + } + mtx.Lock() + output[i] = pr + mtx.Unlock() + q.readerCache.Set(name, pr) + q.metrics.parquetFileOpenDuration.Observe(time.Since(t).Seconds()) + return err + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + return output, nil +} + +func parquetValuesToStrings(values []parquet.Value) []string { + s := make(map[string]struct{}, len(values)) + for _, value := range values { + s[value.String()] = struct{}{} + } + result := make([]string, 0, len(s)) + for k := range s { + result = append(result, k) + } + sort.Strings(result) + return result +} + +type RemoveHashLabelSeriesSet struct { + s storage.SeriesSet + builder *labels.Builder +} + +func newRemoveHashLabelSeriesSet(s storage.SeriesSet) *RemoveHashLabelSeriesSet { + return &RemoveHashLabelSeriesSet{s: s, builder: labels.NewBuilder(labels.EmptyLabels())} +} + +func (ss *RemoveHashLabelSeriesSet) Next() bool { + return ss.s.Next() +} + +// At returns full series. Returned series should be iterable even after Next is called. +func (ss *RemoveHashLabelSeriesSet) At() storage.Series { + series := ss.s.At() + ss.builder.Reset(series.Labels()) + + return &storage.SeriesEntry{ + Lset: ss.builder.Del(seriesHash).Labels(), + SampleIteratorFn: series.Iterator, + } +} + +// The error that iteration as failed with. +// When an error occurs, set cannot continue to iterate. +func (ss *RemoveHashLabelSeriesSet) Err() error { + return ss.s.Err() +} + +// A collection of warnings for the whole set. +// Warnings could be return even iteration has not failed with error. +func (ss *RemoveHashLabelSeriesSet) Warnings() annotations.Annotations { + return ss.s.Warnings() +} + +type ParquetRowsSeriesSet struct { + series []Series + curr Series + err error +} + +type Series struct { + lbls labels.Labels + chks []chunks.Meta + mint, maxt int64 +} + +func (s Series) Labels() labels.Labels { + return s.lbls +} + +func (s Series) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { + iters := make([]chunkenc.Iterator, 0, len(s.chks)) + for _, chk := range s.chks { + iters = append(iters, chk.Chunk.Iterator(nil)) + } + return dedup.NewBoundedSeriesIterator(newChunkSeriesIterator(iters), s.mint, s.maxt) +} + +func newParquetRowsSeriesSet(sorted bool, rows []cortex_storage.ParquetRow, mint, maxt int64, skipChunks bool, chunksDecoder *cortex_storage.PrometheusParquetChunksDecoder, projectionPushdown bool, queryLimiter *limiter.QueryLimiter, reqStats *stats.QueryStats, sts *cortex_storage.Stats) (*ParquetRowsSeriesSet, error) { + start := time.Now() + defer func() { + sts.AddCreateSeriesSetWallTime(time.Since(start).Milliseconds()) + }() + series := make([]Series, 0, 1024) + lblsBuilder := labels.NewScratchBuilder(10) + var ( + chunksCount uint64 + chunksSizeBytes uint64 + labelsSizeBytes uint64 + samples uint64 + overfetchedRows int + ) + for i := range rows { + var ( + chks []chunks.Meta + err error + skip bool + ) + if !skipChunks { + chks, skip, err = getChunksFromParquetRow(chunksDecoder, rows[i], mint, maxt) + if err != nil { + return nil, err + } + if skip { + overfetchedRows++ + continue + } + } + + lblsBuilder.Reset() + for n, v := range rows[i].Columns { + lblsBuilder.Add(n, v) + } + // If we are not pruning columns then there is no need to add series hash as label. + if projectionPushdown { + lblsBuilder.Add(seriesHash, strconv.FormatUint(rows[i].Hash, 10)) + } + + lblsBuilder.Sort() + lbls := lblsBuilder.Labels() + if err := queryLimiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(lbls)); err != nil { + return nil, validation.LimitError(err.Error()) + } + + chksCount := 0 + chunksBytes := 0 + for _, meta := range chks { + chunksBytes += len(meta.Chunk.Bytes()) + samples += uint64(meta.Chunk.NumSamples()) + } + + if chunkBytesLimitErr := queryLimiter.AddChunkBytes(chunksBytes); chunkBytesLimitErr != nil { + return nil, validation.LimitError(chunkBytesLimitErr.Error()) + } + if chunkLimitErr := queryLimiter.AddChunks(chksCount); chunkLimitErr != nil { + return nil, validation.LimitError(chunkLimitErr.Error()) + } + + labelsBytes := 0 + lbls.Range(func(l labels.Label) { + labelsBytes += len(l.Name) + len(l.Value) + }) + + if dataBytesLimitErr := queryLimiter.AddDataBytes(chunksBytes + labelsBytes); dataBytesLimitErr != nil { + return nil, validation.LimitError(dataBytesLimitErr.Error()) + } + + chunksCount += uint64(chksCount) + chunksSizeBytes += uint64(chunksBytes) + labelsSizeBytes += uint64(labelsBytes) + series = append(series, Series{ + lbls: lbls, + chks: chks, + mint: mint, + maxt: maxt, + }) + } + + sts.AddOverfetchedRows(overfetchedRows) + + reqStats.AddFetchedSeries(uint64(len(series))) + reqStats.AddFetchedChunks(chunksCount) + reqStats.AddFetchedChunkBytes(chunksSizeBytes) + reqStats.AddFetchedSamples(samples) + reqStats.AddFetchedDataBytes(chunksSizeBytes + labelsSizeBytes) + + if sorted { + sort.Sort(byLabels(series)) + } + return &ParquetRowsSeriesSet{ + series: series, + }, nil +} + +func getChunksFromParquetRow(chunksDecoder *cortex_storage.PrometheusParquetChunksDecoder, row cortex_storage.ParquetRow, mint, maxt int64) ([]chunks.Meta, bool, error) { + chks, err := chunksDecoder.Decode(row.Data, mint, maxt) + if err != nil { + return nil, false, err + } + if len(chks) == 0 { + return nil, true, nil + } + return chks, false, nil +} + +type byLabels []Series + +func (b byLabels) Len() int { return len(b) } +func (b byLabels) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b byLabels) Less(i, j int) bool { return labels.Compare(b[i].Labels(), b[j].Labels()) < 0 } + +func (ss *ParquetRowsSeriesSet) Next() bool { + if ss.err != nil { + return false + } + if len(ss.series) == 0 { + return false + } + ss.curr, ss.series = ss.series[0], ss.series[1:] + return true +} + +// At returns full series. Returned series should be iterable even after Next is called. +func (ss *ParquetRowsSeriesSet) At() storage.Series { + return ss.curr +} + +// The error that iteration as failed with. +// When an error occurs, set cannot continue to iterate. +func (ss *ParquetRowsSeriesSet) Err() error { + return ss.err +} + +// A collection of warnings for the whole set. +// Warnings could be return even iteration has not failed with error. +func (ss *ParquetRowsSeriesSet) Warnings() annotations.Annotations { + return nil +} + +type chunkSeriesIterator struct { + chunks []chunkenc.Iterator + i int + lastVal chunkenc.ValueType +} + +func newChunkSeriesIterator(cs []chunkenc.Iterator) chunkenc.Iterator { + if len(cs) == 0 { + // This should not happen. StoreAPI implementations should not send empty results. + return errSeriesIterator{err: errors.Errorf("store returned an empty result")} + } + return &chunkSeriesIterator{chunks: cs} +} + +func (it *chunkSeriesIterator) Seek(t int64) chunkenc.ValueType { + // We generally expect the chunks already to be cut down + // to the range we are interested in. There's not much to be gained from + // hopping across chunks so we just call next until we reach t. + for { + ct := it.AtT() + if ct >= t { + return it.lastVal + } + it.lastVal = it.Next() + if it.lastVal == chunkenc.ValNone { + return chunkenc.ValNone + } + } +} + +func (it *chunkSeriesIterator) At() (t int64, v float64) { + return it.chunks[it.i].At() +} + +func (it *chunkSeriesIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram.Histogram) { + return it.chunks[it.i].AtHistogram(h) +} + +func (it *chunkSeriesIterator) AtFloatHistogram(fh *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return it.chunks[it.i].AtFloatHistogram(fh) +} + +func (it *chunkSeriesIterator) AtT() int64 { + return it.chunks[it.i].AtT() +} + +func (it *chunkSeriesIterator) Next() chunkenc.ValueType { + lastT := it.AtT() + + if valueType := it.chunks[it.i].Next(); valueType != chunkenc.ValNone { + it.lastVal = valueType + return valueType + } + if it.Err() != nil { + return chunkenc.ValNone + } + if it.i >= len(it.chunks)-1 { + return chunkenc.ValNone + } + // Chunks are guaranteed to be ordered but not generally guaranteed to not overlap. + // We must ensure to skip any overlapping range between adjacent chunks. + it.i++ + return it.Seek(lastT + 1) +} + +func (it *chunkSeriesIterator) Err() error { + return it.chunks[it.i].Err() +} + +type errSeriesIterator struct { + err error +} + +func (errSeriesIterator) Seek(int64) chunkenc.ValueType { return chunkenc.ValNone } +func (errSeriesIterator) Next() chunkenc.ValueType { return chunkenc.ValNone } +func (errSeriesIterator) At() (int64, float64) { return 0, 0 } +func (errSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) { + return 0, nil +} +func (errSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return 0, nil +} +func (errSeriesIterator) AtT() int64 { return 0 } +func (it errSeriesIterator) Err() error { return it.err } diff --git a/pkg/storage/parquet/cache.go b/pkg/storage/parquet/cache.go new file mode 100644 index 0000000000..cbdd6ecf5f --- /dev/null +++ b/pkg/storage/parquet/cache.go @@ -0,0 +1,80 @@ +package storage + +import ( + lru "github.com/hashicorp/golang-lru/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +type Cache[T any] struct { + cache *lru.Cache[string, T] + metrics *CacheMetrics + name string +} + +type CacheMetrics struct { + hits *prometheus.CounterVec + misses *prometheus.CounterVec + evictions *prometheus.CounterVec + size *prometheus.GaugeVec +} + +func NewCacheMetrics(reg prometheus.Registerer) *CacheMetrics { + return &CacheMetrics{ + hits: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_parquet_reader_cache_hits_total", + Help: "Total number of parquet reader cache hits", + }, []string{"name"}), + misses: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_parquet_reader_cache_misses_total", + Help: "Total number of parquet reader cache misses", + }, []string{"name"}), + evictions: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_parquet_reader_cache_evictions_total", + Help: "Total number of parquet reader cache evictions", + }, []string{"name"}), + size: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_parquet_reader_cache_size", + Help: "Current number of cached parquet readers", + }, []string{"name"}), + } +} + +func NewCache[T any](name string, size int, metrics *CacheMetrics) (*Cache[T], error) { + cache, err := lru.NewWithEvict(size, func(key string, value T) { + metrics.evictions.WithLabelValues(name).Inc() + metrics.size.WithLabelValues(name).Dec() + }) + if err != nil { + return nil, err + } + + return &Cache[T]{ + cache: cache, + name: name, + metrics: metrics, + }, nil +} + +func (c *Cache[T]) Get(path string) (r T) { + if reader, ok := c.cache.Get(path); ok { + c.metrics.hits.WithLabelValues(c.name).Inc() + return reader + } + c.metrics.misses.WithLabelValues(c.name).Inc() + return +} + +func (c *Cache[T]) Set(path string, reader T) { + if !c.cache.Contains(path) { + c.metrics.size.WithLabelValues(c.name).Inc() + } + c.cache.Add(path, reader) +} + +func (c *Cache[T]) Remove(path string) { + if c.cache.Contains(path) { + c.cache.Remove(path) + c.metrics.size.WithLabelValues(c.name).Dec() + } +} diff --git a/pkg/storage/parquet/enc.go b/pkg/storage/parquet/enc.go new file mode 100644 index 0000000000..f072b7f347 --- /dev/null +++ b/pkg/storage/parquet/enc.go @@ -0,0 +1,71 @@ +package storage + +import ( + "bytes" + "encoding/binary" +) + +func EncodeStringSlice(s []string) []byte { + b := new(bytes.Buffer) + for i := 0; i < len(s); i++ { + b.WriteString(s[i]) + b.WriteByte(0) + } + + return b.Bytes() +} + +func DecodeStringSlice(b []byte) ([]string, error) { + buffer := bytes.NewBuffer(b) + r := make([]string, 0, 10) + + for { + s, err := buffer.ReadString(0) + if err != nil { + break + } + r = append(r, s[:len(s)-1]) + } + + return r, nil +} + +func EncodeUintSlice(s []uint64) []byte { + l := make([]byte, binary.MaxVarintLen32) + r := make([]byte, 0, len(s)*binary.MaxVarintLen32) + + // size + n := binary.PutUvarint(l[:], uint64(len(s))) + r = append(r, l[:n]...) + + for i := 0; i < len(s); i++ { + n := binary.PutUvarint(l[:], s[i]) + r = append(r, l[:n]...) + } + + return r +} + +func DecodeUintSlice(b []byte) ([]uint64, error) { + buffer := bytes.NewBuffer(b) + + // size + s, err := binary.ReadUvarint(buffer) + + if err != nil { + return nil, err + } + + r := make([]uint64, 0, s) + + for i := uint64(0); i < s; i++ { + v, err := binary.ReadUvarint(buffer) + + if err != nil { + return nil, err + } + r = append(r, v) + } + + return r, nil +} diff --git a/pkg/storage/parquet/enc_test.go b/pkg/storage/parquet/enc_test.go new file mode 100644 index 0000000000..8b33fdf754 --- /dev/null +++ b/pkg/storage/parquet/enc_test.go @@ -0,0 +1,42 @@ +package storage + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncodeDecodeUsintSlicesSlices(t *testing.T) { + slices := [][]uint64{ + {}, + {1, 2, 3}, + } + + for _, slice := range slices { + t.Run(fmt.Sprint(slice), func(t *testing.T) { + encode := EncodeUintSlice(slice) + decode, err := DecodeUintSlice(encode) + require.NoError(t, err) + require.Equal(t, slice, decode) + }) + } + +} + +func TestEncodeDecodeStringSlicesSlices(t *testing.T) { + slices := [][]string{ + {}, + {"", "0", "foo", "bar"}, + } + + for _, slice := range slices { + t.Run(fmt.Sprint(slice), func(t *testing.T) { + encode := EncodeStringSlice(slice) + decode, err := DecodeStringSlice(encode) + require.NoError(t, err) + require.Equal(t, slice, decode) + }) + } + +} diff --git a/pkg/storage/parquet/prometheus_parquet_chunks_encoder.go b/pkg/storage/parquet/prometheus_parquet_chunks_encoder.go new file mode 100644 index 0000000000..82f02b32a8 --- /dev/null +++ b/pkg/storage/parquet/prometheus_parquet_chunks_encoder.go @@ -0,0 +1,142 @@ +package storage + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "sort" + "time" + + "github.com/thanos-io/thanos/pkg/compact/downsample" + + "github.com/dennwc/varint" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" +) + +type PrometheusParquetChunksEncoder struct { +} + +type PrometheusParquetChunksDecoder struct { + Pool chunkenc.Pool +} + +func NewPrometheusParquetChunksEncoder() *PrometheusParquetChunksEncoder { + return &PrometheusParquetChunksEncoder{} +} + +func NewPrometheusParquetChunksDecoder() *PrometheusParquetChunksDecoder { + return &PrometheusParquetChunksDecoder{ + Pool: downsample.NewPool(), + } +} + +func (e *PrometheusParquetChunksEncoder) Close() error { + return nil +} + +func (e *PrometheusParquetChunksEncoder) Encode(chks []chunks.Meta) ([][]byte, error) { + sort.Slice(chks, func(i, j int) bool { + return chks[i].MinTime < chks[j].MinTime + }) + + reEncodedChunks := make([]chunks.Meta, ChunkColumnsPerDay) + reEncodedChunksAppenders := make([]chunkenc.Appender, ChunkColumnsPerDay) + + for i := 0; i < ChunkColumnsPerDay; i++ { + reEncodedChunks[i] = chunks.Meta{ + Chunk: chunkenc.NewXORChunk(), + MinTime: math.MaxInt64, + } + app, err := reEncodedChunks[i].Chunk.Appender() + if err != nil { + return nil, err + } + reEncodedChunksAppenders[i] = app + } + + var it chunkenc.Iterator + for _, chk := range chks { + it = chk.Chunk.Iterator(it) + switch chk.Chunk.Encoding() { + case chunkenc.EncXOR: + for vt := it.Next(); vt != chunkenc.ValNone; vt = it.Next() { + if vt != chunkenc.ValFloat { + return nil, fmt.Errorf("found value type %v in float chunk", vt) + } + t, v := it.At() + hour := time.UnixMilli(t).UTC().Hour() + chkIdx := (hour / int(ChunkColumnLength.Hours())) % ChunkColumnsPerDay + reEncodedChunksAppenders[chkIdx].Append(t, v) + if t < reEncodedChunks[chkIdx].MinTime { + reEncodedChunks[chkIdx].MinTime = t + } + if t > reEncodedChunks[chkIdx].MaxTime { + reEncodedChunks[chkIdx].MaxTime = t + } + } + default: + continue + } + } + + result := make([][]byte, ChunkColumnsPerDay) + + for i, chk := range reEncodedChunks { + var b [varint.MaxLen64]byte + n := binary.PutUvarint(b[:], uint64(chk.MinTime)) + result[i] = append(result[i], b[:n]...) + n = binary.PutUvarint(b[:], uint64(chk.MaxTime)) + result[i] = append(result[i], b[:n]...) + n = binary.PutUvarint(b[:], uint64(chk.Chunk.Encoding())) + result[i] = append(result[i], b[:n]...) + n = binary.PutUvarint(b[:], uint64(len(chk.Chunk.Bytes()))) + result[i] = append(result[i], b[:n]...) + result[i] = append(result[i], chk.Chunk.Bytes()...) + } + return result, nil +} + +func (e *PrometheusParquetChunksDecoder) Decode(data [][]byte, mint, maxt int64) ([]chunks.Meta, error) { + result := make([]chunks.Meta, 0, len(data)) + + for _, dChunk := range data { + b := bytes.NewBuffer(dChunk) + c := chunks.Meta{} + minTime, err := binary.ReadUvarint(b) + if err != nil { + return nil, err + } + c.MinTime = int64(minTime) + maxTime, err := binary.ReadUvarint(b) + if err != nil { + return nil, err + } + c.MinTime = int64(maxTime) + chkEnc, err := binary.ReadUvarint(b) + if err != nil { + return nil, err + } + size, err := binary.ReadUvarint(b) + if err != nil { + return nil, err + } + cData := b.Bytes()[:size] + chk, err := e.Pool.Get(chunkenc.Encoding(chkEnc), cData) + c.Chunk = chk + if int64(minTime) > maxt { + continue + } + + if int64(maxTime) >= mint { + result = append(result, chunks.Meta{ + MinTime: int64(minTime), + MaxTime: int64(maxTime), + Chunk: chk, + }) + } + } + + return result, nil +} diff --git a/pkg/storage/parquet/reader.go b/pkg/storage/parquet/reader.go new file mode 100644 index 0000000000..1b9d4e33be --- /dev/null +++ b/pkg/storage/parquet/reader.go @@ -0,0 +1,893 @@ +package storage + +import ( + "context" + "fmt" + "io" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + "unsafe" + + "github.com/thanos-io/thanos/pkg/store" + + "github.com/efficientgo/core/errors" + + "github.com/opentracing/opentracing-go" + "github.com/parquet-go/parquet-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/model/labels" + "golang.org/x/sync/errgroup" +) + +const ( + defaultReadBufferSize = 100 * 1024 // 100k +) + +type CreateReadAtWithContext func(ctx context.Context) io.ReaderAt + +type bufferedReaderAt struct { + r io.ReaderAt + b []byte + offset int64 +} + +func (b bufferedReaderAt) ReadAt(p []byte, off int64) (n int, err error) { + if off >= b.offset && off < b.offset+int64(len(b.b)) { + diff := off - b.offset + n := copy(p, b.b[diff:]) + return n, nil + } + + return b.r.ReadAt(p, off) +} + +func newBufferedReaderAt(r io.ReaderAt, minOffset, maxOffset int64) io.ReaderAt { + if minOffset < maxOffset { + b := make([]byte, maxOffset-minOffset) + n, err := r.ReadAt(b, minOffset) + if err == nil { + return &bufferedReaderAt{r: r, b: b[:n], offset: minOffset} + } + } + return r +} + +type ParquetReader struct { + //r io.ReaderAt + + size int64 + Rows int64 + f *parquet.File + columnIndexMap map[string]int + sortedColumn string + dataColsSize int + asyncRead bool // Add this field + + pagesSkippedPageIndex *prometheus.CounterVec + projectionPushdown prometheus.Counter + + dictionaryCache *Cache[parquet.Dictionary] + readerFactory CreateReadAtWithContext +} + +func NewParquetReader(readerFactory CreateReadAtWithContext, size int64, asyncRead bool, cacheMetrics *CacheMetrics, dictionaryCacheSize int, pagesSkippedPageIndex *prometheus.CounterVec, projectionPushdown prometheus.Counter) (*ParquetReader, error) { + options := []parquet.FileOption{ + parquet.SkipBloomFilters(true), // we don't use bloom filters + parquet.ReadBufferSize(1024 * 512), + parquet.OptimisticRead(true), + parquet.SkipMagicBytes(true), + } + + // Set read mode based on the flag + if asyncRead { + options = append(options, parquet.FileReadMode(parquet.ReadModeAsync)) + } + + // Create a reader with initial file loading. + f, err := parquet.OpenFile(readerFactory(context.Background()), size, options...) + if err != nil { + return nil, err + } + columnIndexMap := make(map[string]int) + for i, c := range f.Schema().Columns() { + columnIndexMap[c[0]] = i + } + + sortedColumn := "" + numDataCols := 0 + + for _, md := range f.Metadata().KeyValueMetadata { + switch md.Key { + case SortedColMetadataKey: + sortedColumn = md.Value + case NumberOfDataColumnsMetadataKey: + numDataCols, _ = strconv.Atoi(md.Value) + } + } + + cache, err := NewCache[parquet.Dictionary]("dictionary", dictionaryCacheSize, cacheMetrics) + if err != nil { + return nil, err + } + + return &ParquetReader{ + readerFactory: readerFactory, + f: f, + size: size, + Rows: f.NumRows(), + columnIndexMap: columnIndexMap, + sortedColumn: sortedColumn, + dataColsSize: numDataCols, + asyncRead: asyncRead, + pagesSkippedPageIndex: pagesSkippedPageIndex, + projectionPushdown: projectionPushdown, + dictionaryCache: cache, + }, nil +} + +func (pr *ParquetReader) ColumnExists(name string) bool { + if _, ok := pr.columnIndexMap[name]; ok { + return true + } + return false +} + +func (pr *ParquetReader) ColumnNames() []string { + names := make([]string, 0, len(pr.columnIndexMap)) + for colName := range pr.columnIndexMap { + if colName != ColHash && colName != ColIndexes && !pr.isColData(colName) { + names = append(names, colName) + } + } + sort.Strings(names) + return names +} + +func (pr *ParquetReader) SearchRows(ctx context.Context, colName, colValue string, sts *Stats) ([][]int64, error) { + span, _ := opentracing.StartSpanFromContext(ctx, "ParquetReader.SearchRows") + defer span.Finish() + if colName != pr.sortedColumn { + return nil, fmt.Errorf("cannot search column %v as its not sorted. Sorted column %v", colName, pr.sortedColumn) + } + + ci, ok := pr.columnIndexMap[colName] + results := make([][]int64, len(pr.f.RowGroups())) + + if !ok { + return results, nil + } + + pagesRead := 0 + pagesSizeBytes := int64(0) +NextRG: + for rgi, rg := range pr.f.RowGroups() { + ch, err := rg.ColumnChunks()[ci].ColumnIndex() + if err != nil { + return nil, err + } + + found := parquet.Search(ch, parquet.ValueOf(colValue), parquet.ByteArrayType) + + pagesToRead := []int{} + for i := found; i < ch.NumPages(); i++ { + minValue := ch.MinValue(i).String() + if minValue > colValue { + break + } + pagesToRead = append(pagesToRead, i) + } + + if found < ch.NumPages() { + offset, err := rg.ColumnChunks()[ci].OffsetIndex() + if err != nil { + return results, err + } + pages := pr.getPage(ctx, rgi, colName, offset, pagesToRead) + pagesSizeBytes += getCompressedPageSizes(pagesToRead, offset) + row := offset.FirstRowIndex(found) + err = pages.SeekToRow(row) + if err != nil { + return results, err + } + currentRow := row + for { + page, err := pages.ReadPage() + if err != nil { + break + } + pagesRead++ + + vr := page.Values() + values := [100]parquet.Value{} + + for { + n, err := vr.ReadValues(values[:]) + for _, value := range values[:n] { + c := strings.Compare(colValue, yoloString(value.ByteArray())) + switch c { + case 0: + results[rgi] = append(results[rgi], currentRow) + case -1: + pr.releasePage(pages) + continue NextRG + } + + currentRow++ + } + if err != nil { + break + } + } + } + pr.releasePage(pages) + } + } + + sts.AddPagesSearchScanned(colName, pagesRead) + sts.AddPagesSearchSizeBytes(colName, pagesSizeBytes) + return results, nil +} + +func (pr *ParquetReader) DataColsSize() int { + return pr.dataColsSize +} + +func (pr *ParquetReader) ScanRows(ctx context.Context, rows [][]int64, full bool, m *labels.Matcher, sts *Stats) ([][]int64, error) { + span, _ := opentracing.StartSpanFromContext(ctx, "ParquetReader.ScanRows") + defer span.Finish() + + mtx := sync.Mutex{} + results := make([][]int64, len(pr.f.RowGroups())) + + ci, ok := pr.columnIndexMap[m.Name] + if !ok { + return results, nil + } + + var pagesSkippedCounter prometheus.Counter + if pr.pagesSkippedPageIndex != nil { + pagesSkippedCounter = pr.pagesSkippedPageIndex.WithLabelValues(m.Name) + } + + errGroup := &errgroup.Group{} + + loadPageIndex, matchNull, matchNotNull := usePageIndex(m) + + for rgi, rg := range pr.f.RowGroups() { + if !full && len(rows[rgi]) == 0 { + continue + } + + errGroup.Go(func() error { + var ( + pageSkipped int + ) + offsetIndex, err := rg.ColumnChunks()[ci].OffsetIndex() + if err != nil { + return err + } + // Only use page Index if it is a full scan and equal matcher. + var columnIndex parquet.ColumnIndex + if loadPageIndex { + columnIndex, err = rg.ColumnChunks()[ci].ColumnIndex() + if err != nil { + return err + } + } + + pagesToRowsMap := map[int][]int64{} + + if full { + for i := 0; i < offsetIndex.NumPages(); i++ { + if loadPageIndex { + if !filterPageWithPageIndex(m, columnIndex, i, matchNull, matchNotNull) { + pageSkipped++ + if pagesSkippedCounter != nil { + pagesSkippedCounter.Inc() + } + continue + } + } + + firstRow := offsetIndex.FirstRowIndex(i) + lastRow := rg.NumRows() + if i < offsetIndex.NumPages()-1 { + lastRow = offsetIndex.FirstRowIndex(i+1) - 1 + } + pagesToRowsMap[i] = append(pagesToRowsMap[i], firstRow) + pagesToRowsMap[i] = append(pagesToRowsMap[i], lastRow) + } + } else { + pagesSkipped := make(map[int]bool) + for _, v := range rows[rgi] { + idx := sort.Search(offsetIndex.NumPages(), func(i int) bool { + return offsetIndex.FirstRowIndex(i) > v + }) - 1 + if loadPageIndex { + if _, ok := pagesSkipped[idx]; !ok { + skipPage := !filterPageWithPageIndex(m, columnIndex, idx, matchNull, matchNotNull) + pagesSkipped[idx] = skipPage + } + if pagesSkipped[idx] { + pageSkipped++ + if pagesSkippedCounter != nil { + pagesSkippedCounter.Inc() + } + continue + } + } + pagesToRowsMap[idx] = append(pagesToRowsMap[idx], v) + } + } + sts.AddPagesSkipped(m.Name, pageSkipped) + for _, pagesToRead := range mergePagesIdx(pagesToRowsMap, offsetIndex) { + errGroup.Go(func() error { + var ( + pageScanned int + pagesSizeBytes int64 + ) + defer func() { + sts.AddPagesSearchScanned(m.Name, pageScanned) + sts.AddPagesSearchSizeBytes(m.Name, pagesSizeBytes) + }() + offset, err := rg.ColumnChunks()[ci].OffsetIndex() + if err != nil { + return err + } + pages := pr.getPage(ctx, rgi, m.Name, offset, pagesToRead.pages) + pagesSizeBytes += getCompressedPageSizes(pagesToRead.pages, offset) + defer pr.releasePage(pages) + + next := pagesToRead.rows[0] + maxRow := pagesToRead.rows[len(pagesToRead.rows)-1] + currentRow := next + err = pages.SeekToRow(next) + if err != nil { + return err + } + + for { + page, err := pages.ReadPage() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + pageScanned++ + vr := page.Values() + values := [100]parquet.Value{} + + for { + n, err := vr.ReadValues(values[:]) + for _, value := range values[:n] { + if currentRow == next { + // TODO: Lets run the match using dictionary or cache the values already matched, + // so we prevent running the same matcher multiple times. (Regex match can be quite resource intensive). + if m.Matches(yoloString(value.ByteArray())) { + mtx.Lock() + results[rgi] = append(results[rgi], currentRow) + mtx.Unlock() + } + if !full { + pagesToRead.rows = pagesToRead.rows[1:] + if len(pagesToRead.rows) == 0 { + parquet.Release(page) + return nil + } + next = pagesToRead.rows[0] + } else { + next = next + 1 + } + } + currentRow++ + if currentRow > maxRow { + parquet.Release(page) + return nil + } + } + if err != nil { + parquet.Release(page) + break + } + } + } + }) + } + return nil + }) + } + + err := errGroup.Wait() + + for _, result := range results { + slices.Sort(result) + } + + return results, err +} + +func (pr *ParquetReader) ColumnValues(ctx context.Context, c string) ([]string, error) { + errGroup := &errgroup.Group{} + + r := make(map[string]struct{}, 128) + rMtx := sync.Mutex{} + + for rgi := range pr.f.RowGroups() { + errGroup.Go(func() error { + pages := pr.getPage(ctx, rgi, c, nil, nil) + dic, err := pages.ReadDictionary() + if err != nil { + return err + } + for i := 0; i < dic.Len(); i++ { + rMtx.Lock() + r[dic.Index(int32(i)).String()] = struct{}{} + rMtx.Unlock() + } + return nil + }) + } + + err := errGroup.Wait() + + if err != nil { + return nil, err + } + + results := make([]string, 0, len(r)) + for v := range r { + results = append(results, v) + } + sort.Strings(results) + return results, nil +} + +func (pr *ParquetReader) Materialize(ctx context.Context, rows [][]int64, dataColsIdx []int, grouping []string, by bool, sts *Stats) ([]ParquetRow, error) { + start := time.Now() + defer func() { + sts.AddMaterializeWallTime(time.Since(start).Milliseconds()) + }() + span, _ := opentracing.StartSpanFromContext(ctx, "ParquetReader.Materialize") + defer span.Finish() + + if by { + if pr.projectionPushdown != nil { + pr.projectionPushdown.Inc() + } + return pr.materializeBy(ctx, rows, dataColsIdx, grouping, sts) + } + + // TODO: skip fetching hash if not needed. + columns := []string{ColHash, ColIndexes} + cValues, err := pr.MaterializeColumn(ctx, rows, sts, columns...) + if err != nil { + return nil, err + } + + results := make([]ParquetRow, len(cValues[0])) + colsMap := make(map[string]struct{}, len(cValues[0])) + + for i, v := range cValues[0] { + results[i] = ParquetRow{ + Hash: v.Uint64(), + Columns: map[string]string{}, + } + + if i < len(cValues[1]) { + colIndexes, err := DecodeUintSlice(cValues[1][i].Bytes()) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode column indexes: len %d", len(cValues[1][i].Bytes())) + } + for _, idx := range colIndexes { + if _, ok := colsMap[pr.f.Schema().Columns()[idx][0]]; !ok { + colsMap[pr.f.Schema().Columns()[idx][0]] = struct{}{} + } + } + } + } + + cols := make([]string, 0, len(colsMap)+len(dataColsIdx)) + for _, idx := range dataColsIdx { + c := fmt.Sprintf("%v_%v", ColData, idx) + cols = append(cols, c) + } + // Subtract not needed groups. + for _, group := range grouping { + delete(colsMap, group) + } + if len(grouping) > 0 && pr.projectionPushdown != nil { + pr.projectionPushdown.Inc() + } + for k := range colsMap { + cols = append(cols, k) + } + + cValues, err = pr.MaterializeColumn(ctx, rows, sts, cols...) + if err != nil { + return nil, errors.Wrapf(err, "failed to materialize column values") + } + + for i, colValues := range cValues { + for j, v := range colValues { + if !v.IsNull() { + if pr.isColData(cols[i]) { + if len(v.ByteArray()) > 0 { + results[j].Data = append(results[j].Data, v.ByteArray()) + } + } else { + results[j].Columns[cols[i]] = v.String() + } + } + } + } + + span.SetTag("cols", len(dataColsIdx)) + span.SetTag("by", by) + span.SetTag("results", len(results)) + + return results, nil +} + +func (pr *ParquetReader) materializeBy(ctx context.Context, rows [][]int64, dataColsIdx []int, grouping []string, sts *Stats) ([]ParquetRow, error) { + columns := []string{ColHash} + for _, group := range grouping { + if _, ok := pr.columnIndexMap[group]; ok { + columns = append(columns, group) + } + } + + for _, idx := range dataColsIdx { + c := fmt.Sprintf("%v_%v", ColData, idx) + columns = append(columns, c) + } + + cValues, err := pr.MaterializeColumn(ctx, rows, sts, columns...) + if err != nil { + return nil, err + } + + results := make([]ParquetRow, len(cValues[0])) + for i, v := range cValues[0] { + results[i] = ParquetRow{ + Hash: v.Uint64(), + Columns: map[string]string{}, + } + } + + for i, colValues := range cValues[1:] { + for j, v := range colValues { + if !v.IsNull() { + if pr.isColData(columns[i+1]) { + if len(v.ByteArray()) > 0 { + results[j].Data = append(results[j].Data, v.ByteArray()) + } + } else { + results[j].Columns[columns[i+1]] = v.String() + } + } + } + } + + return results, nil +} + +func (pr *ParquetReader) MaterializeColumn(ctx context.Context, rows [][]int64, sts *Stats, columns ...string) ([][]parquet.Value, error) { + mtx := sync.Mutex{} + errGroup := &errgroup.Group{} + errGroup.SetLimit(runtime.GOMAXPROCS(0)) + + // Pre allocate results slices + results := make([][]parquet.Value, len(columns)) + totalResults := 0 + for _, r := range rows { + totalResults += len(r) + } + + for i := range columns { + results[i] = make([]parquet.Value, totalResults) + } + + for cIndex, c := range columns { + ci, ok := pr.columnIndexMap[c] + if !ok { + continue + } + sts.AddColumnMaterialized(c) + resultIdx := 0 + for rgi, rg := range pr.f.RowGroups() { + rowsResultsIdx := make(map[int64]int, len(results[cIndex])) + if len(rows[rgi]) == 0 { + continue + } + offsetIndex, err := rg.ColumnChunks()[ci].OffsetIndex() + if err != nil { + return nil, err + } + + pagesToRowsMap := make(map[int][]int64, len(rows[rgi])) + for _, v := range rows[rgi] { + idx := sort.Search(offsetIndex.NumPages(), func(i int) bool { + return offsetIndex.FirstRowIndex(i) > v + }) - 1 + pagesToRowsMap[idx] = append(pagesToRowsMap[idx], v) + mtx.Lock() + rowsResultsIdx[v] = resultIdx + mtx.Unlock() + resultIdx++ + } + + for _, pagesToRead := range mergePagesIdx(pagesToRowsMap, offsetIndex) { + errGroup.Go(func() error { + var ( + pagesScanned int + pagesSizeBytes int64 + ) + defer func() { + sts.AddPagesMaterializeScanned(c, pagesScanned) + sts.AddPagesMaterializeSizeBytes(c, pagesSizeBytes) + }() + pages := pr.getPage(ctx, rgi, c, offsetIndex, pagesToRead.pages) + pagesSizeBytes += getCompressedPageSizes(pagesToRead.pages, offsetIndex) + + defer pr.releasePage(pages) + next := pagesToRead.rows[0] + currentRow := next + err := pages.SeekToRow(next) + if err != nil { + return err + } + for { + page, err := pages.ReadPage() + if err != nil { + return err + } + pagesScanned++ + + vr := page.Values() + for { + values := [1024]parquet.Value{} + n, err := vr.ReadValues(values[:]) + if err != nil && err != io.EOF { + parquet.Release(page) + return err + } + for _, value := range values[:n] { + if currentRow == next { + mtx.Lock() + results[cIndex][rowsResultsIdx[currentRow]] = value.Clone() + mtx.Unlock() + pagesToRead.rows = pagesToRead.rows[1:] + if len(pagesToRead.rows) == 0 { + parquet.Release(page) + return nil + } + next = pagesToRead.rows[0] + } + currentRow++ + if currentRow > next { + parquet.Release(page) + return errors.Newf("did not find rows: next %d current %d remaining %d", next, currentRow, len(pagesToRead.rows)) + } + } + if err == io.EOF { + parquet.Release(page) + break + } + } + } + }) + } + } + } + err := errGroup.Wait() + return results, err +} + +func (pr *ParquetReader) MaterializeColumnNames(ctx context.Context, rows [][]int64, sts *Stats) ([]string, error) { + cValues, err := pr.MaterializeColumn(ctx, rows, sts, ColIndexes) + if err != nil { + return nil, err + } + + colsMap := make(map[string]struct{}, len(pr.columnIndexMap)) + for _, v := range cValues[0] { + colIndexes, err := DecodeUintSlice(v.Bytes()) + if err != nil { + return nil, err + } + for _, idx := range colIndexes { + if _, ok := colsMap[pr.f.Schema().Columns()[idx][0]]; !ok { + colsMap[pr.f.Schema().Columns()[idx][0]] = struct{}{} + } + } + } + cols := make([]string, 0, len(colsMap)) + for col := range colsMap { + cols = append(cols, col) + } + sort.Strings(cols) + return cols, nil +} + +func (pr *ParquetReader) isColData(c string) bool { + return strings.HasPrefix(c, ColData) +} + +func (pr *ParquetReader) getPage(ctx context.Context, rgi int, c string, offset parquet.OffsetIndex, pagesToRead []int) *parquet.FilePages { + ci, ok := pr.columnIndexMap[c] + if !ok { + return nil + } + + key := fmt.Sprintf("%d_%d", rgi, ci) + dic := pr.dictionaryCache.Get(key) + var minOffset, maxOffset int64 + if len(pagesToRead) > 0 && offset != nil { + minOffset = offset.Offset(pagesToRead[0]) + maxOffset = offset.Offset(pagesToRead[len(pagesToRead)-1]) + offset.CompressedPageSize(pagesToRead[len(pagesToRead)-1]) + } + + colChunk := pr.f.RowGroups()[rgi].ColumnChunks()[ci].(*parquet.FileColumnChunk) + bufferedReader := newBufferedReaderAt(pr.readerFactory(ctx), minOffset, maxOffset) + pages := colChunk.PagesFrom(bufferedReader, defaultReadBufferSize) + + if dic != nil { + pages.SetDictionary(dic) + } else { + d, _ := pages.ReadDictionary() + pr.dictionaryCache.Set(key, d) + } + + return pages +} + +func (pr *ParquetReader) releasePage(p parquet.Pages) { + p.Close() +} + +func usePageIndex(m *labels.Matcher) (useIndex bool, matchNull bool, matchNotNull bool) { + if matchNullMatcher(m) { + return true, true, false + } + if matchNotNullMatcher(m) { + return true, false, true + } + switch m.Type { + case labels.MatchEqual, labels.MatchNotEqual: + return true, false, false + case labels.MatchRegexp: + return len(m.SetMatches()) > 0 || len(m.Prefix()) > 0, false, false + case labels.MatchNotRegexp: + return len(m.SetMatches()) == 1, false, false + } + return false, false, false +} + +func matchNullMatcher(matcher *labels.Matcher) bool { + // If the matcher is ="" or =~"" or !~".+", it is the same as removing all values for the label. + // Match null value only. + if (matcher.Value == "" && (matcher.Type == labels.MatchEqual || matcher.Type == labels.MatchRegexp)) || + (matcher.Type == labels.MatchNotRegexp && matcher.Value == ".+") { + return true + } + return false +} + +func matchNotNullMatcher(matcher *labels.Matcher) bool { + // If the matcher is !="" or !~"" or =~".+", it is the same as adding all values for the label. + // Filter out null value. + if (matcher.Value == "" && (matcher.Type == labels.MatchNotEqual || matcher.Type == labels.MatchNotRegexp)) || + (matcher.Type == labels.MatchRegexp && matcher.Value == ".+") { + return true + } + return false +} + +func filterPageWithPageIndex(m *labels.Matcher, columnIndex parquet.ColumnIndex, pageId int, matchNull, matchNotNull bool) bool { + if matchNotNull { + // If the matcher asks for not null values, skip any null page. + return !columnIndex.NullPage(pageId) + } + if matchNull { + // If the matcher asks for null values, skip any page that doesn't have null values. + return columnIndex.NullCount(pageId) > 0 + } + // For any other matcher scenario, filter out any page that is null. + if columnIndex.NullPage(pageId) { + return false + } + minVal := columnIndex.MinValue(pageId) + maxVal := columnIndex.MaxValue(pageId) + cmp := parquet.CompareNullsLast(parquet.ByteArrayType.Compare) + var valuesToMatch []string + switch m.Type { + case labels.MatchEqual: + valuesToMatch = []string{m.Value} + case labels.MatchNotEqual: + if cmp(minVal, maxVal) == 0 && cmp(minVal, parquet.ValueOf(m.Value)) == 0 { + return false + } + return true + case labels.MatchNotRegexp: + // Only scenario we can skip page for not regexp is not regexp + // has only 1 match which is the same as not equal. + if cmp(minVal, maxVal) == 0 && cmp(minVal, parquet.ValueOf(m.SetMatches()[0])) == 0 { + return false + } + return true + case labels.MatchRegexp: + if len(m.SetMatches()) > 0 { + valuesToMatch = m.SetMatches() + } else if len(m.Prefix()) > 0 { + prefix := m.Prefix() + // MaxValue shouldn't be null as we already skipped null page above. + maxValStr := maxVal.String() + if minVal.IsNull() { + return prefix <= maxValStr[:min(len(prefix), len(maxValStr))] + } + minValStr := minVal.String() + return prefix >= minValStr[:max(len(prefix), len(minValStr))] && + prefix <= maxValStr[:max(len(prefix), len(maxValStr))] + } + } + + for _, v := range valuesToMatch { + // As long as a single value that matches, we don't filter out the page. + if cmp(parquet.ValueOf(v), minVal) >= 0 && cmp(parquet.ValueOf(v), maxVal) <= 0 { + return true + } + } + return false +} + +type pageEntryRead struct { + pages []int + rows []int64 +} + +func mergePagesIdx(pagedIdx map[int][]int64, offset parquet.OffsetIndex) []pageEntryRead { + partitioner := store.NewGapBasedPartitioner(10 * 1024) + if len(pagedIdx) == 0 { + return []pageEntryRead{} + } + idxs := make([]int, 0, len(pagedIdx)) + for idx := range pagedIdx { + idxs = append(idxs, idx) + } + slices.Sort(idxs) + + parts := partitioner.Partition(len(idxs), func(i int) (uint64, uint64) { + return uint64(offset.Offset(idxs[i])), uint64(offset.Offset(idxs[i]) + offset.CompressedPageSize(idxs[i])) + }) + + r := make([]pageEntryRead, 0, len(parts)) + for _, part := range parts { + pagesToRead := pageEntryRead{} + for i := part.ElemRng[0]; i < part.ElemRng[1]; i++ { + pagesToRead.pages = append(pagesToRead.pages, idxs[i]) + pagesToRead.rows = append(pagesToRead.rows, pagedIdx[idxs[i]]...) + } + r = append(r, pagesToRead) + } + + return r +} + +func getCompressedPageSizes(pagesToRead []int, offset parquet.OffsetIndex) int64 { + if len(pagesToRead) > 0 && offset != nil { + minOffset := offset.Offset(pagesToRead[0]) + maxOffset := offset.Offset(pagesToRead[len(pagesToRead)-1]) + offset.CompressedPageSize(pagesToRead[len(pagesToRead)-1]) + return maxOffset - minOffset + } + return 0 +} + +func yoloString(buf []byte) string { + return *((*string)(unsafe.Pointer(&buf))) +} diff --git a/pkg/storage/parquet/reader_test.go b/pkg/storage/parquet/reader_test.go new file mode 100644 index 0000000000..d87beb313e --- /dev/null +++ b/pkg/storage/parquet/reader_test.go @@ -0,0 +1,190 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "sort" + "testing" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" +) + +func TestSearchRows(t *testing.T) { + buffer := bytes.NewBuffer(nil) + cols := []string{"sorted_key", "col_0", "col_1", "col_2", "unique"} + w := NewParquetWriter(buffer, 200, 100, 3, cols, "sorted_key") + + numberOfRows := 10000 + numberOfKeys := 500 + rows := generateRows("sorted_key", 3, numberOfRows, numberOfKeys) + + err := w.WriteRows(rows) + require.NoError(t, err) + require.NoError(t, w.Close()) + + r, err := NewParquetReader(BufferReadAt{buffer: buffer}.CreateReadAtWithContext, int64(buffer.Len()), false, NewCacheMetrics(prometheus.NewPedanticRegistry()), 10, nil, nil) + require.NoError(t, err) + sts := NewStats() + + // should not allow search rows on cols not sorted + _, err = r.SearchRows(context.Background(), "col_1", "val", sts) + require.Error(t, err) + + // searching sorted key + allFounds := make([]ParquetRow, 0, len(rows)) + for i := 0; i < numberOfKeys; i++ { + rows, err := r.SearchRows(context.Background(), "sorted_key", fmt.Sprintf("key_%d", i), sts) + require.NoError(t, err) + require.Equal(t, numberOfRows/numberOfKeys, countResults(rows)) + founds, err := r.Materialize(context.Background(), rows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, founds, numberOfRows/numberOfKeys) + allFounds = append(allFounds, founds...) + } + + require.Equal(t, len(rows), len(allFounds)) + for _, found := range allFounds { + require.Contains(t, rows, found) + } +} + +func TestScanRows(t *testing.T) { + buffer := bytes.NewBuffer(nil) + cols := []string{"country", "stats", "indices"} + + countries := []string{"brazil", "canada", "equator", "japan"} + stats := []string{"max", "min", "avg"} + indices := []string{"population", "size"} + + numberOfExtraCountries := 1000 + + for i := 0; i < numberOfExtraCountries; i++ { + countries = append(countries, fmt.Sprintf("country_%d", i)) + } + rows := make([]ParquetRow, 0, len(countries)) + + for _, country := range countries { + for _, stat := range stats { + for _, ind := range indices { + rows = append(rows, ParquetRow{ + Hash: uint64(rand.Int63()), + Data: [][]byte{[]byte(fmt.Sprintf("%v_%v_%v", country, stat, ind))}, + Columns: map[string]string{ + "country": country, + "stats": stat, + "indices": ind, + }, + }) + } + } + } + + sort.Slice(rows, func(i, j int) bool { + return rows[i].Columns["country"] < rows[j].Columns["country"] + }) + + w := NewParquetWriter(buffer, 200, 100, 3, cols, "country") + err := w.WriteRows(rows) + require.NoError(t, err) + require.NoError(t, w.Close()) + + r, err := NewParquetReader(BufferReadAt{buffer: buffer}.CreateReadAtWithContext, int64(buffer.Len()), false, NewCacheMetrics(prometheus.NewPedanticRegistry()), 10, nil, nil) + require.NoError(t, err) + + sts := NewStats() + + // should not allow search rows not sorted + foundRows, err := r.SearchRows(context.Background(), "country", "brazil", sts) + require.NoError(t, err) + result, err := r.Materialize(context.Background(), foundRows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, result, len(stats)*len(indices)) + for _, r := range result { + require.Equal(t, "brazil", r.Columns["country"]) + } + + // Scan for stats max + foundRows, err = r.ScanRows(context.Background(), foundRows, false, labels.MustNewMatcher(labels.MatchEqual, "stats", "max"), sts) + require.NoError(t, err) + result, err = r.Materialize(context.Background(), foundRows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, result, len(indices)) + + for _, r := range result { + require.Equal(t, "brazil", r.Columns["country"]) + require.Equal(t, "max", r.Columns["stats"]) + } + + // Scan for indices:population + foundRows, err = r.ScanRows(context.Background(), foundRows, false, labels.MustNewMatcher(labels.MatchEqual, "indices", "population"), sts) + require.NoError(t, err) + result, err = r.Materialize(context.Background(), foundRows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, result, 1) + require.Equal(t, "population", result[0].Columns["indices"]) + require.Equal(t, "brazil_max_population", string(result[0].Data[0])) + + // Lets create a full scan + foundRows, err = r.ScanRows(context.Background(), foundRows, true, labels.MustNewMatcher(labels.MatchEqual, "indices", "population"), sts) + require.NoError(t, err) + result, err = r.Materialize(context.Background(), foundRows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, result, len(countries)*len(stats)) + for _, r := range result { + require.Equal(t, "population", r.Columns["indices"]) + } + + foundRows, err = r.ScanRows(context.Background(), foundRows, false, labels.MustNewMatcher(labels.MatchEqual, "stats", "max"), sts) + require.NoError(t, err) + result, err = r.Materialize(context.Background(), foundRows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, result, len(countries)) + allCountriesFromResults := map[string]int{} + + for _, r := range result { + require.Equal(t, "max", r.Columns["stats"]) + allCountriesFromResults[r.Columns["country"]]++ + } + + require.Equal(t, len(countries), len(allCountriesFromResults)) + + // Should return empty for cols that does not exits + foundRows, err = r.ScanRows(context.Background(), foundRows, true, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"), sts) + require.NoError(t, err) + result, err = r.Materialize(context.Background(), foundRows, []int{0, 1, 2}, nil, false, sts) + require.NoError(t, err) + require.Len(t, result, 0) +} + +func generateRows(sortedKey string, numberOfCols, numberOfRows int, numberOfKeys int) []ParquetRow { + var rows []ParquetRow + for i := 0; i < numberOfRows; i++ { + rows = append(rows, ParquetRow{ + Hash: uint64(int64(i)), + Data: [][]byte{[]byte("oi")}, + Columns: map[string]string{ + sortedKey: fmt.Sprintf("key_%d", i%numberOfKeys), + "unique": fmt.Sprintf("unique_%d", i), + fmt.Sprintf("col_%v", i%numberOfCols): fmt.Sprintf("col_%v", i%numberOfCols), + }, + }) + } + + sort.Slice(rows, func(i, j int) bool { + return rows[i].Columns[sortedKey] < rows[j].Columns[sortedKey] + }) + return rows +} + +func countResults(r [][]int64) int { + count := 0 + for _, row := range r { + count += len(row) + } + return count +} diff --git a/pkg/storage/parquet/row.go b/pkg/storage/parquet/row.go new file mode 100644 index 0000000000..8d14de3aaf --- /dev/null +++ b/pkg/storage/parquet/row.go @@ -0,0 +1,16 @@ +package storage + +const ( + ColHash = "s_hash" + ColIndexes = "s_col_indexes" + ColData = "s_data" + SortedColMetadataKey = "sorting_col_metadata_key" + NumberOfDataColumnsMetadataKey = "number_of_data_columns_metadata_key" + IndexSizeLimitMetadataKey = "index_size_limit_metadata_key" +) + +type ParquetRow struct { + Hash uint64 + Data [][]byte + Columns map[string]string +} diff --git a/pkg/storage/parquet/stats.go b/pkg/storage/parquet/stats.go new file mode 100644 index 0000000000..b52c3affb2 --- /dev/null +++ b/pkg/storage/parquet/stats.go @@ -0,0 +1,247 @@ +package storage + +import ( + "sort" + "strconv" + "strings" + "sync" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" +) + +type Stats struct { + Mtx sync.Mutex + // Number of series we overfetched due to chunks time range. + FilesScanned int + ParquetReaderCacheMiss int + RowsFetched int + OverfetchedRows int + // Key is column name and value is how many rows after searching the column + ColumnsSearched map[string]int + ColumnsMaterialize map[string]struct{} + PagesSearchScanned map[string]int + PagesMaterializeScanned map[string]int + PagesSkipped map[string]int + PagesSearchSizeBytes map[string]int64 + PagesMaterializeSizeBytes map[string]int64 + + RequestDuration int64 + OpenParquetReaderDuration int64 + SearchWallTime int64 + MaterializeWallTime int64 + CreateSeriesSetWallTime int64 + StorageWallTime int64 +} + +func setToStrings(m map[string]struct{}) string { + res := make([]string, 0, len(m)) + for k := range m { + res = append(res, k) + } + sort.Strings(res) + return strings.Join(res, ",") +} + +func mapToString(m map[string]int) string { + type kv struct { + key string + value int64 + } + kvs := make([]kv, 0, len(m)) + for k, v := range m { + kvs = append(kvs, kv{key: k, value: int64(v)}) + } + sort.Slice(kvs, func(i, j int) bool { + return kvs[i].value > kvs[j].value + }) + sb := strings.Builder{} + for i, kv := range kvs { + sb.WriteString(kv.key) + sb.WriteString(":") + sb.WriteString(strconv.FormatInt(kv.value, 10)) + if i != len(kvs)-1 { + sb.WriteString(",") + } + } + return sb.String() +} + +func mapToStringInt64(m map[string]int64) string { + type kv struct { + key string + value int64 + } + kvs := make([]kv, 0, len(m)) + for k, v := range m { + kvs = append(kvs, kv{key: k, value: v}) + } + sort.Slice(kvs, func(i, j int) bool { + return kvs[i].value > kvs[j].value + }) + sb := strings.Builder{} + for i, kv := range kvs { + sb.WriteString(kv.key) + sb.WriteString(":") + sb.WriteString(strconv.FormatInt(kv.value, 10)) + if i != len(kvs)-1 { + sb.WriteString(",") + } + } + return sb.String() +} + +func NewStats() *Stats { + return &Stats{ + ColumnsSearched: make(map[string]int), + ColumnsMaterialize: make(map[string]struct{}), + PagesSearchScanned: make(map[string]int), + PagesMaterializeScanned: make(map[string]int), + PagesSkipped: make(map[string]int), + PagesSearchSizeBytes: make(map[string]int64), + PagesMaterializeSizeBytes: make(map[string]int64), + } +} + +func (s *Stats) Display(logger log.Logger, extraFields ...interface{}) { + pagesSkippedTotal := 0 + for _, n := range s.PagesSkipped { + pagesSkippedTotal += n + } + pagesSearchSizeBytesTotal := int64(0) + for _, n := range s.PagesSearchSizeBytes { + pagesSearchSizeBytesTotal += n + } + pagesMaterializeSizeBytesTotal := int64(0) + for _, n := range s.PagesMaterializeSizeBytes { + pagesMaterializeSizeBytesTotal += n + } + fields := []interface{}{ + "msg", "stats", + "filesScanned", s.FilesScanned, + "rowsFetched", s.RowsFetched, + "overfetchedRows", s.OverfetchedRows, + "columnsMaterialize", setToStrings(s.ColumnsMaterialize), + "columnsSearched", mapToString(s.ColumnsSearched), + "pagesSearchScanned", mapToString(s.PagesSearchScanned), + "pagesMaterializeScanned", mapToString(s.PagesMaterializeScanned), + "pagesSkipped", mapToString(s.PagesSkipped), + "totalPagesSkipped", pagesSkippedTotal, + "pagesSearchSizeBytes", mapToStringInt64(s.PagesSearchSizeBytes), + "pagesSearchSizeBytesTotal", pagesSearchSizeBytesTotal, + "pagesMaterializeSizeBytes", mapToStringInt64(s.PagesMaterializeSizeBytes), + "pagesMaterializeSizeBytesTotal", pagesMaterializeSizeBytesTotal, + "pagesSizeBytesTotal", pagesSearchSizeBytesTotal + pagesMaterializeSizeBytesTotal, + "parquetReaderCacheMiss", s.ParquetReaderCacheMiss, + "duration", s.RequestDuration, + "openParquetReaderDuration", s.OpenParquetReaderDuration, + "searchWallTime", s.SearchWallTime, + "materializeWallTime", s.MaterializeWallTime, + "createSeriesSetWallTime", s.CreateSeriesSetWallTime, + "storageWallTime", s.StorageWallTime, + } + fields = append(fields, extraFields...) + level.Info(logger).Log(fields...) +} + +func (s *Stats) AddRowsFetched(rowsFetched int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.RowsFetched += rowsFetched +} + +func (s *Stats) AddFilesScanned(filesScanned int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.FilesScanned += filesScanned +} + +func (s *Stats) AddParquetReaderCacheMiss(miss int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.ParquetReaderCacheMiss += miss +} + +func (s *Stats) AddOverfetchedRows(overfetchedRows int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.OverfetchedRows += overfetchedRows +} + +func (s *Stats) AddPagesSearchScanned(col string, pages int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.PagesSearchScanned[col] += pages +} + +func (s *Stats) AddPagesMaterializeScanned(col string, pages int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.PagesMaterializeScanned[col] += pages +} + +func (s *Stats) AddPagesSkipped(col string, pages int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.PagesSkipped[col] += pages +} + +func (s *Stats) AddPagesSearchSizeBytes(col string, size int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.PagesSearchSizeBytes[col] += size +} + +func (s *Stats) AddPagesMaterializeSizeBytes(col string, size int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.PagesMaterializeSizeBytes[col] += size +} + +func (s *Stats) AddColumnSearched(col string, rows int) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.ColumnsSearched[col] += rows +} + +func (s *Stats) AddColumnMaterialized(col string) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.ColumnsMaterialize[col] = struct{}{} +} + +func (s *Stats) AddRequestDuration(d int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.RequestDuration += d +} + +func (s *Stats) AddOpenParquetReaderDuration(d int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.OpenParquetReaderDuration += d +} + +func (s *Stats) AddSearchWallTime(d int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.SearchWallTime += d +} + +func (s *Stats) AddMaterializeWallTime(d int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.MaterializeWallTime += d +} + +func (s *Stats) AddCreateSeriesSetWallTime(d int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.CreateSeriesSetWallTime += d +} + +func (s *Stats) AddStorageWallTime(d int64) { + s.Mtx.Lock() + defer s.Mtx.Unlock() + s.StorageWallTime += d +} diff --git a/pkg/storage/parquet/stats_test.go b/pkg/storage/parquet/stats_test.go new file mode 100644 index 0000000000..b9881cbd36 --- /dev/null +++ b/pkg/storage/parquet/stats_test.go @@ -0,0 +1,21 @@ +package storage + +import ( + "os" + "testing" + + "github.com/go-kit/log" +) + +func TestDisplay(t *testing.T) { + stats := NewStats() + logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) + extraFields := []interface{}{ + "request", "labelNames", + "mint", 0, + "maxt", 100000, + } + stats.AddColumnSearched("name", 1000) + stats.AddPagesSearchSizeBytes("job", 100000) + stats.Display(logger, extraFields...) +} diff --git a/pkg/storage/parquet/util_test.go b/pkg/storage/parquet/util_test.go new file mode 100644 index 0000000000..d9f1494d67 --- /dev/null +++ b/pkg/storage/parquet/util_test.go @@ -0,0 +1,20 @@ +package storage + +import ( + "bytes" + "context" + "io" +) + +type BufferReadAt struct { + buffer *bytes.Buffer +} + +func (b BufferReadAt) ReadAt(p []byte, off int64) (n int, err error) { + n = copy(p, b.buffer.Bytes()[off:off+int64(len(p))]) + return +} + +func (b BufferReadAt) CreateReadAtWithContext(_ context.Context) io.ReaderAt { + return b +} diff --git a/pkg/storage/parquet/writer.go b/pkg/storage/parquet/writer.go new file mode 100644 index 0000000000..bf5960aa21 --- /dev/null +++ b/pkg/storage/parquet/writer.go @@ -0,0 +1,121 @@ +package storage + +import ( + "errors" + "fmt" + "io" + "slices" + "strconv" + "time" + + "github.com/parquet-go/parquet-go" +) + +const ( + ChunkColumnLength = 8 * time.Hour + ChunkColumnsPerDay = 3 +) + +var ( + errRowOutOrder = errors.New("row out of order") +) + +type ParquetWriter struct { + w *parquet.GenericWriter[any] + rb *parquet.RowBuilder + + sortedColumn string + // keep the last value for each sorted column to make sure they are inserted in order + sortedColumnLastValue string + columnIndexMap map[string]int +} + +func NewParquetWriter(w io.Writer, rowsPerRowGroup int64, indexSizeLimit, numDataCols int, columns []string, sortedColumn string) *ParquetWriter { + schemaGroup := parquet.Group{} + skipPageBounds := []string{ColHash, ColIndexes} + // No dictionary here as it this has no repetition, and we would need to download the whole dic + // when materializing this column to get the final result + schemaGroup[ColHash] = parquet.Compressed(parquet.Int(64), &parquet.Zstd) + schemaGroup[ColIndexes] = parquet.Compressed(parquet.Encoded(parquet.Leaf(parquet.ByteArrayType), &parquet.DeltaByteArray), &parquet.Zstd) + for i := 0; i < numDataCols; i++ { + c := fmt.Sprintf("%v_%v", ColData, i) + skipPageBounds = append(skipPageBounds, c) + schemaGroup[c] = parquet.Compressed(parquet.Encoded(parquet.Leaf(parquet.ByteArrayType), &parquet.DeltaByteArray), &parquet.Zstd) + } + skipPageBounds = append(skipPageBounds) + + for _, n := range columns { + if _, ok := schemaGroup[n]; !ok { + schemaGroup[n] = parquet.Compressed(parquet.Optional(parquet.Encoded(parquet.String(), &parquet.RLEDictionary)), &parquet.Zstd) + } + } + + schema := parquet.NewSchema("", schemaGroup) + + columnIndexMap := make(map[string]int) + + for i, c := range schema.Columns() { + columnIndexMap[c[0]] = i + } + + return &ParquetWriter{ + columnIndexMap: columnIndexMap, + sortedColumn: sortedColumn, + w: parquet.NewGenericWriter[any]( + w, + schema, + parquet.MaxRowsPerRowGroup(rowsPerRowGroup), + parquet.ColumnIndexSizeLimit(indexSizeLimit), + parquet.SkipPageBounds(skipPageBounds...), + parquet.KeyValueMetadata(SortedColMetadataKey, sortedColumn), + parquet.KeyValueMetadata(NumberOfDataColumnsMetadataKey, strconv.Itoa(numDataCols)), + parquet.KeyValueMetadata(IndexSizeLimitMetadataKey, strconv.Itoa(indexSizeLimit)), + ), + rb: parquet.NewRowBuilder(schema), + } +} + +func (p *ParquetWriter) WriteRows(series []ParquetRow) error { + rows := make([]parquet.Row, 0, len(series)) + colIdxs := make([]uint64, 0, 1024) + + for _, s := range series { + p.rb.Reset() + colIdxs = colIdxs[:0] + + p.rb.Add(p.columnIndexMap[ColHash], parquet.ValueOf(s.Hash)) + for k, v := range s.Columns { + + if p.sortedColumn == k { + if v < p.sortedColumnLastValue { + return errRowOutOrder + } + p.sortedColumnLastValue = v + } + + cIdx, ok := p.columnIndexMap[k] + if !ok { + return fmt.Errorf("column not found: %s", k) + } + p.rb.Add(cIdx, parquet.ValueOf(v)) + colIdxs = append(colIdxs, uint64(p.columnIndexMap[k])) + } + + slices.Sort(colIdxs) + + p.rb.Add(p.columnIndexMap[ColIndexes], parquet.ValueOf(EncodeUintSlice(colIdxs))) + for i, data := range s.Data { + c := fmt.Sprintf("%v_%v", ColData, i) + p.rb.Add(p.columnIndexMap[c], parquet.ValueOf(data)) + } + rows = append(rows, p.rb.Row()) + } + + _, err := p.w.WriteRows(rows) + + return err +} + +func (p *ParquetWriter) Close() error { + return p.w.Close() +} diff --git a/pkg/storage/parquet/writer_test.go b/pkg/storage/parquet/writer_test.go new file mode 100644 index 0000000000..416e587ee3 --- /dev/null +++ b/pkg/storage/parquet/writer_test.go @@ -0,0 +1,50 @@ +package storage + +import ( + "bytes" + "testing" + + "github.com/parquet-go/parquet-go" + "github.com/stretchr/testify/require" +) + +func TestWriteColumnIstNotFound(t *testing.T) { + buffer := bytes.NewBuffer(nil) + cols := []string{"country", "stats"} + w := NewParquetWriter(buffer, 100, 100, 3, cols, "") + + err := w.WriteRows([]ParquetRow{{Columns: map[string]string{"not_defined": ""}}}) + require.Error(t, err) + + err = w.WriteRows([]ParquetRow{{Columns: map[string]string{"country": "brazil"}}}) + require.NoError(t, err) + require.NoError(t, w.Close()) + + r, err := parquet.Read[any](BufferReadAt{buffer: buffer}, int64(buffer.Len()), w.w.Schema()) + require.NoError(t, err) + require.Len(t, r, 1) + require.Equal(t, r[0].(map[string]any)["country"], "brazil") +} + +func TestWriteColumnInsertOutOfOrder(t *testing.T) { + buffer := bytes.NewBuffer(nil) + cols := []string{"country", "stats"} + w := NewParquetWriter(buffer, 100, 100, 3, cols, "country") + + err := w.WriteRows([]ParquetRow{{Columns: map[string]string{"country": "brazil"}}}) + require.NoError(t, err) + + err = w.WriteRows([]ParquetRow{{Columns: map[string]string{"country": "zimbabue"}}}) + require.NoError(t, err) + + // Out of order insert + err = w.WriteRows([]ParquetRow{{Columns: map[string]string{"country": "columbia"}}}) + require.Error(t, err) + require.NoError(t, w.Close()) + + r, err := parquet.Read[any](BufferReadAt{buffer: buffer}, int64(buffer.Len()), w.w.Schema()) + require.NoError(t, err) + require.Len(t, r, 2) + require.Equal(t, r[0].(map[string]any)["country"], "brazil") + require.Equal(t, r[1].(map[string]any)["country"], "zimbabue") +} diff --git a/pkg/storage/tsdb/bucketindex/parquet_bucket_index.go b/pkg/storage/tsdb/bucketindex/parquet_bucket_index.go new file mode 100644 index 0000000000..9da94eef96 --- /dev/null +++ b/pkg/storage/tsdb/bucketindex/parquet_bucket_index.go @@ -0,0 +1,93 @@ +package bucketindex + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/go-kit/log" + "github.com/klauspost/compress/gzip" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/thanos-io/objstore" + "github.com/thanos-io/thanos/pkg/runutil" +) + +const ( + parquetIndex = "bucket-index-parquet.json" + parquetIndexCompressedName = parquetIndex + ".gz" +) + +type BlockWithExtension struct { + *Block + Extensions Extensions `json:"extensions,omitempty"` +} + +type ParquetIndex struct { + Blocks map[ulid.ULID]BlockWithExtension `json:"blocks"` +} + +type Extensions struct { + PartitionInfo PartitionInfo `json:"partition_info"` +} + +type PartitionInfo struct { + MetricNamePartitionCount int `json:"metric_name_partition_count"` + MetricNamePartitionID int `json:"metric_name_partition_id"` +} + +func ReadParquetIndex(ctx context.Context, userBkt objstore.InstrumentedBucket, logger log.Logger) (*ParquetIndex, error) { + // Get the bucket index. + reader, err := userBkt.WithExpectedErrs(tsdb.IsOneOfTheExpectedErrors(userBkt.IsAccessDeniedErr, userBkt.IsObjNotFoundErr)).Get(ctx, parquetIndexCompressedName) + if err != nil { + if userBkt.IsObjNotFoundErr(err) { + return &ParquetIndex{Blocks: map[ulid.ULID]BlockWithExtension{}}, nil + } + + return nil, err + } + defer runutil.CloseWithLogOnErr(logger, reader, "close bucket index reader") + + // Read all the content. + gzipReader, err := gzip.NewReader(reader) + if err != nil { + return nil, ErrIndexCorrupted + } + defer runutil.CloseWithLogOnErr(logger, gzipReader, "close bucket index gzip reader") + + // Deserialize it. + index := &ParquetIndex{} + d := json.NewDecoder(gzipReader) + if err := d.Decode(index); err != nil { + return nil, ErrIndexCorrupted + } + return index, nil +} + +func WriteParquetIndex(ctx context.Context, bkt objstore.Bucket, idx *ParquetIndex) error { + // Marshal the index. + content, err := json.Marshal(idx) + if err != nil { + return errors.Wrap(err, "marshal bucket index") + } + + // Compress it. + var gzipContent bytes.Buffer + gzip := gzip.NewWriter(&gzipContent) + gzip.Name = parquetIndex + + if _, err := gzip.Write(content); err != nil { + return errors.Wrap(err, "gzip bucket index") + } + if err := gzip.Close(); err != nil { + return errors.Wrap(err, "close gzip bucket index") + } + + // Upload the index to the storage. + if err := bkt.Upload(ctx, parquetIndexCompressedName, bytes.NewReader(gzipContent.Bytes())); err != nil { + return errors.Wrap(err, "upload bucket index") + } + + return nil +} diff --git a/pkg/storage/tsdb/bucketindex/parquet_loader.go b/pkg/storage/tsdb/bucketindex/parquet_loader.go new file mode 100644 index 0000000000..8ad5a30490 --- /dev/null +++ b/pkg/storage/tsdb/bucketindex/parquet_loader.go @@ -0,0 +1,301 @@ +package bucketindex + +import ( + "context" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/thanos-io/objstore" + "go.uber.org/atomic" + + "github.com/cortexproject/cortex/pkg/storage/bucket" + "github.com/cortexproject/cortex/pkg/util" + util_log "github.com/cortexproject/cortex/pkg/util/log" + "github.com/cortexproject/cortex/pkg/util/services" +) + +// ParquetLoader is responsible to lazy load bucket indexes and, once loaded for the first time, +// keep them updated in background. Loaded indexes are automatically offloaded once the +// idle timeout expires. +type ParquetLoader struct { + services.Service + + bkt objstore.Bucket + logger log.Logger + cfg LoaderConfig + cfgProvider bucket.TenantConfigProvider + + indexesMx sync.RWMutex + indexes map[string]*cachedParquetIndex + + // Metrics. + loadAttempts prometheus.Counter + loadFailures prometheus.Counter + loadDuration prometheus.Histogram + loaded prometheus.GaugeFunc +} + +// NewLParquetLoader makes a new ParquetLoader. +func NewLParquetLoader(cfg LoaderConfig, bucketClient objstore.Bucket, cfgProvider bucket.TenantConfigProvider, logger log.Logger, reg prometheus.Registerer) *ParquetLoader { + l := &ParquetLoader{ + bkt: bucketClient, + logger: logger, + cfg: cfg, + cfgProvider: cfgProvider, + indexes: map[string]*cachedParquetIndex{}, + + loadAttempts: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "cortex_bucket_index_loads_total", + Help: "Total number of bucket index loading attempts.", + }), + loadFailures: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "cortex_bucket_index_load_failures_total", + Help: "Total number of bucket index loading failures.", + }), + loadDuration: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_bucket_index_load_duration_seconds", + Help: "Duration of the a single bucket index loading operation in seconds.", + Buckets: []float64{0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 1, 10}, + }), + } + + l.loaded = promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_bucket_index_loaded", + Help: "Number of bucket indexes currently loaded in-memory.", + }, l.countLoadedIndexesMetric) + + // Apply a jitter to the sync frequency in order to increase the probability + // of hitting the shared cache (if any). + checkInterval := util.DurationWithJitter(cfg.CheckInterval, 0.2) + l.Service = services.NewTimerService(checkInterval, nil, l.checkCachedIndexes, nil) + + return l +} + +// GetIndex returns the bucket index for the given user. It returns the in-memory cached +// index if available, or load it from the bucket otherwise. +func (l *ParquetLoader) GetIndex(ctx context.Context, userID string) (*ParquetIndex, Status, error) { + l.indexesMx.RLock() + if entry := l.indexes[userID]; entry != nil { + idx := entry.index + err := entry.err + ss := entry.syncStatus + l.indexesMx.RUnlock() + + // We don't check if the index is stale because it's the responsibility + // of the background job to keep it updated. + entry.requestedAt.Store(time.Now().Unix()) + return idx, ss, err + } + l.indexesMx.RUnlock() + + ss, err := ReadSyncStatus(ctx, l.bkt, userID, l.logger) + + if err != nil { + level.Warn(l.logger).Log("msg", "unable to read bucket index status", "user", userID, "err", err) + } + + startTime := time.Now() + l.loadAttempts.Inc() + uBucket := bucket.NewUserBucketClient(userID, l.bkt, l.cfgProvider) + idx, err := ReadParquetIndex(ctx, uBucket, l.logger) + if err != nil { + // Cache the error, to avoid hammering the object store in case of persistent issues + // (eg. corrupted bucket index or not existing). + l.cacheIndex(userID, nil, ss, err) + + if ctx.Err() != nil { + level.Warn(util_log.WithContext(ctx, l.logger)).Log("msg", "received context error when reading bucket index", "err", ctx.Err()) + return nil, UnknownStatus, ctx.Err() + } + + if errors.Is(err, ErrIndexNotFound) { + level.Warn(l.logger).Log("msg", "bucket index not found", "user", userID) + } else if errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) { + level.Warn(l.logger).Log("msg", "key access denied when reading bucket index", "user", userID) + } else { + // We don't track ErrIndexNotFound as failure because it's a legit case (eg. a tenant just + // started to remote write and its blocks haven't uploaded to storage yet). + l.loadFailures.Inc() + level.Error(l.logger).Log("msg", "unable to load bucket index", "user", userID, "err", err) + } + + return nil, ss, err + } + + // Cache the index. + l.cacheIndex(userID, idx, ss, nil) + + elapsedTime := time.Since(startTime) + l.loadDuration.Observe(elapsedTime.Seconds()) + level.Info(l.logger).Log("msg", "loaded bucket index", "user", userID, "duration", elapsedTime) + return idx, ss, nil +} + +func (l *ParquetLoader) cacheIndex(userID string, idx *ParquetIndex, ss Status, err error) { + if errors.Is(err, context.Canceled) { + level.Info(l.logger).Log("msg", "skipping cache bucket index", "err", err) + return + } + l.indexesMx.Lock() + defer l.indexesMx.Unlock() + + // Not an issue if, due to concurrency, another index was already cached + // and we overwrite it: last will win. + l.indexes[userID] = newParquetCachedIndex(idx, ss, err) +} + +// checkCachedIndexes checks all cached indexes and, for each of them, does two things: +// 1. Offload indexes not requested since >= idle timeout +// 2. Update indexes which have been updated last time since >= update timeout +func (l *ParquetLoader) checkCachedIndexes(ctx context.Context) error { + // Build a list of users for which we should update or delete the index. + toUpdate, toDelete := l.checkCachedIndexesToUpdateAndDelete() + + // Delete unused indexes. + for _, userID := range toDelete { + l.deleteCachedIndex(userID) + } + + // Update actively used indexes. + for _, userID := range toUpdate { + l.updateCachedIndex(ctx, userID) + } + + // Never return error, otherwise the service terminates. + return nil +} + +func (l *ParquetLoader) checkCachedIndexesToUpdateAndDelete() (toUpdate, toDelete []string) { + now := time.Now() + + l.indexesMx.RLock() + defer l.indexesMx.RUnlock() + + for userID, entry := range l.indexes { + // Given ErrIndexNotFound is a legit case and assuming UpdateOnErrorInterval is lower than + // UpdateOnStaleInterval, we don't consider ErrIndexNotFound as an error with regards to the + // refresh interval and so it will updated once stale. + isError := entry.err != nil && !errors.Is(entry.err, ErrIndexNotFound) + + switch { + case now.Sub(entry.getRequestedAt()) >= l.cfg.IdleTimeout: + toDelete = append(toDelete, userID) + case isError && now.Sub(entry.getUpdatedAt()) >= l.cfg.UpdateOnErrorInterval: + toUpdate = append(toUpdate, userID) + case !isError && now.Sub(entry.getUpdatedAt()) >= l.cfg.UpdateOnStaleInterval: + toUpdate = append(toUpdate, userID) + } + } + + return +} + +func (l *ParquetLoader) updateCachedIndex(ctx context.Context, userID string) { + readCtx, cancel := context.WithTimeout(ctx, readIndexTimeout) + defer cancel() + + l.loadAttempts.Inc() + startTime := time.Now() + ss, err := ReadSyncStatus(ctx, l.bkt, userID, l.logger) + if err != nil { + level.Warn(l.logger).Log("msg", "unable to read bucket index status", "user", userID, "err", err) + } + + // Update Sync Status + l.indexesMx.Lock() + l.indexes[userID].syncStatus = ss + l.indexesMx.Unlock() + + uBucket := bucket.NewUserBucketClient(userID, l.bkt, l.cfgProvider) + idx, err := ReadParquetIndex(readCtx, uBucket, l.logger) + if err != nil && + !errors.Is(err, ErrIndexNotFound) && + !errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) && + !errors.Is(err, context.Canceled) { + l.loadFailures.Inc() + level.Warn(l.logger).Log("msg", "unable to update bucket index", "user", userID, "err", err) + return + } + + l.loadDuration.Observe(time.Since(startTime).Seconds()) + + // We cache it either it was successfully refreshed, wasn't found or when is a CMK error. An use case for caching the ErrIndexNotFound + // is when a tenant has rules configured but hasn't started remote writing yet. Rules will be evaluated and + // bucket index loaded by the ruler. + l.indexesMx.Lock() + l.indexes[userID].index = idx + l.indexes[userID].err = err + l.indexes[userID].setUpdatedAt(startTime) + l.indexesMx.Unlock() +} + +func (l *ParquetLoader) deleteCachedIndex(userID string) { + l.indexesMx.Lock() + delete(l.indexes, userID) + l.indexesMx.Unlock() + + level.Info(l.logger).Log("msg", "unloaded bucket index", "user", userID, "reason", "idle") +} + +func (l *ParquetLoader) countLoadedIndexesMetric() float64 { + l.indexesMx.RLock() + defer l.indexesMx.RUnlock() + + count := 0 + for _, idx := range l.indexes { + if idx.index != nil { + count++ + } + } + return float64(count) +} + +type cachedParquetIndex struct { + // We cache either the index or the error occurred while fetching it. They're + // mutually exclusive. + index *ParquetIndex + syncStatus Status + err error + + // Unix timestamp (seconds) of when the index has been updated from the storage the last time. + updatedAt atomic.Int64 + + // Unix timestamp (seconds) of when the index has been requested the last time. + requestedAt atomic.Int64 +} + +func newParquetCachedIndex(idx *ParquetIndex, ss Status, err error) *cachedParquetIndex { + entry := &cachedParquetIndex{ + index: idx, + err: err, + syncStatus: ss, + } + + now := time.Now() + entry.setUpdatedAt(now) + entry.setRequestedAt(now) + + return entry +} + +func (i *cachedParquetIndex) setUpdatedAt(ts time.Time) { + i.updatedAt.Store(ts.Unix()) +} + +func (i *cachedParquetIndex) getUpdatedAt() time.Time { + return time.Unix(i.updatedAt.Load(), 0) +} + +func (i *cachedParquetIndex) setRequestedAt(ts time.Time) { + i.requestedAt.Store(ts.Unix()) +} + +func (i *cachedParquetIndex) getRequestedAt() time.Time { + return time.Unix(i.requestedAt.Load(), 0) +} diff --git a/vendor/github.com/andybalholm/brotli/LICENSE b/vendor/github.com/andybalholm/brotli/LICENSE new file mode 100644 index 0000000000..33b7cdd2db --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/andybalholm/brotli/README.md b/vendor/github.com/andybalholm/brotli/README.md new file mode 100644 index 0000000000..00625211d7 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/README.md @@ -0,0 +1,14 @@ +This package is a brotli compressor and decompressor implemented in Go. +It was translated from the reference implementation (https://github.com/google/brotli) +with the `c2go` tool at https://github.com/andybalholm/c2go. + +I have been working on new compression algorithms (not translated from C) +in the matchfinder package. +You can use them with the NewWriterV2 function. +Currently they give better results than the old implementation +(at least for compressing my test file, Newtonโ€™s *Opticks*) +on levels 2 to 6. + +I am using it in production with https://github.com/andybalholm/redwood. + +API documentation is found at https://pkg.go.dev/github.com/andybalholm/brotli?tab=doc. diff --git a/vendor/github.com/andybalholm/brotli/backward_references.go b/vendor/github.com/andybalholm/brotli/backward_references.go new file mode 100644 index 0000000000..008c054d1c --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/backward_references.go @@ -0,0 +1,185 @@ +package brotli + +import ( + "sync" +) + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Function to find backward reference copies. */ + +func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint { + if distance <= max_distance { + var distance_plus_3 uint = distance + 3 + var offset0 uint = distance_plus_3 - uint(dist_cache[0]) + var offset1 uint = distance_plus_3 - uint(dist_cache[1]) + if distance == uint(dist_cache[0]) { + return 0 + } else if distance == uint(dist_cache[1]) { + return 1 + } else if offset0 < 7 { + return (0x9750468 >> (4 * offset0)) & 0xF + } else if offset1 < 7 { + return (0xFDB1ACE >> (4 * offset1)) & 0xF + } else if distance == uint(dist_cache[2]) { + return 2 + } else if distance == uint(dist_cache[3]) { + return 3 + } + } + + return distance + numDistanceShortCodes - 1 +} + +var hasherSearchResultPool sync.Pool + +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var insert_length uint = *last_insert_len + var pos_end uint = position + num_bytes + var store_end uint + if num_bytes >= hasher.StoreLookahead() { + store_end = position + num_bytes - hasher.StoreLookahead() + 1 + } else { + store_end = position + } + var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params) + var apply_random_heuristics uint = position + random_heuristics_window_size + var gap uint = 0 + /* Set maximum distance, see section 9.1. of the spec. */ + + const kMinScore uint = scoreBase + 100 + + /* For speed up heuristics for random data. */ + + /* Minimum score to accept a backward reference. */ + hasher.PrepareDistanceCache(dist_cache) + sr2, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr2 == nil { + sr2 = &hasherSearchResult{} + } + sr, _ := hasherSearchResultPool.Get().(*hasherSearchResult) + if sr == nil { + sr = &hasherSearchResult{} + } + + for position+hasher.HashTypeLength() < pos_end { + var max_length uint = pos_end - position + var max_distance uint = brotli_min_size_t(position, max_backward_limit) + sr.len = 0 + sr.len_code_delta = 0 + sr.distance = 0 + sr.score = kMinScore + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, sr) + if sr.score > kMinScore { + /* Found a match. Let's look for something even better ahead. */ + var delayed_backward_references_in_row int = 0 + max_length-- + for ; ; max_length-- { + var cost_diff_lazy uint = 175 + if params.quality < minQualityForExtensiveReferenceSearch { + sr2.len = brotli_min_size_t(sr.len-1, max_length) + } else { + sr2.len = 0 + } + sr2.len_code_delta = 0 + sr2.distance = 0 + sr2.score = kMinScore + max_distance = brotli_min_size_t(position+1, max_backward_limit) + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, sr2) + if sr2.score >= sr.score+cost_diff_lazy { + /* Ok, let's just write one byte for now and start a match from the + next byte. */ + position++ + + insert_length++ + *sr = *sr2 + delayed_backward_references_in_row++ + if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end { + continue + } + } + + break + } + + apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size + max_distance = brotli_min_size_t(position, max_backward_limit) + { + /* The first 16 codes are special short-codes, + and the minimum offset is 1. */ + var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache) + if (sr.distance <= (max_distance + gap)) && distance_code > 0 { + dist_cache[3] = dist_cache[2] + dist_cache[2] = dist_cache[1] + dist_cache[1] = dist_cache[0] + dist_cache[0] = int(sr.distance) + hasher.PrepareDistanceCache(dist_cache) + } + + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code)) + } + + *num_literals += insert_length + insert_length = 0 + /* Put the hash keys into the table, if there are enough bytes left. + Depending on the hasher implementation, it can push all positions + in the given range or only a subset of them. + Avoid hash poisoning with RLE data. */ + { + var range_start uint = position + 2 + var range_end uint = brotli_min_size_t(position+sr.len, store_end) + if sr.distance < sr.len>>2 { + range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2))) + } + + hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end) + } + + position += sr.len + } else { + insert_length++ + position++ + + /* If we have not seen matches for a long time, we can skip some + match lookups. Unsuccessful match lookups are very very expensive + and this kind of a heuristic speeds up compression quite + a lot. */ + if position > apply_random_heuristics { + /* Going through uncompressible data, jump. */ + if position > apply_random_heuristics+4*random_heuristics_window_size { + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4) + /* It is quite a long time since we saw a copy, so we assume + that this data is not compressible, and store hashes less + often. Hashes of non compressible data are less likely to + turn out to be useful in the future, too, so we store less of + them to not to flood out the hash table of good compressible + data. */ + + var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin) + for ; position < pos_jump; position += 4 { + hasher.Store(ringbuffer, ringbuffer_mask, position) + insert_length += 4 + } + } else { + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2) + var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin) + for ; position < pos_jump; position += 2 { + hasher.Store(ringbuffer, ringbuffer_mask, position) + insert_length += 2 + } + } + } + } + } + + insert_length += pos_end - position + *last_insert_len = insert_length + + hasherSearchResultPool.Put(sr) + hasherSearchResultPool.Put(sr2) +} diff --git a/vendor/github.com/andybalholm/brotli/backward_references_hq.go b/vendor/github.com/andybalholm/brotli/backward_references_hq.go new file mode 100644 index 0000000000..21629c1cdb --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/backward_references_hq.go @@ -0,0 +1,796 @@ +package brotli + +import "math" + +type zopfliNode struct { + length uint32 + distance uint32 + dcode_insert_length uint32 + u struct { + cost float32 + next uint32 + shortcut uint32 + } +} + +const maxEffectiveDistanceAlphabetSize = 544 + +const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */ + +var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} + +var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3} + +func initZopfliNodes(array []zopfliNode, length uint) { + var stub zopfliNode + var i uint + stub.length = 1 + stub.distance = 0 + stub.dcode_insert_length = 0 + stub.u.cost = kInfinity + for i = 0; i < length; i++ { + array[i] = stub + } +} + +func zopfliNodeCopyLength(self *zopfliNode) uint32 { + return self.length & 0x1FFFFFF +} + +func zopfliNodeLengthCode(self *zopfliNode) uint32 { + var modifier uint32 = self.length >> 25 + return zopfliNodeCopyLength(self) + 9 - modifier +} + +func zopfliNodeCopyDistance(self *zopfliNode) uint32 { + return self.distance +} + +func zopfliNodeDistanceCode(self *zopfliNode) uint32 { + var short_code uint32 = self.dcode_insert_length >> 27 + if short_code == 0 { + return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1 + } else { + return short_code - 1 + } +} + +func zopfliNodeCommandLength(self *zopfliNode) uint32 { + return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF) +} + +/* Histogram based cost model for zopflification. */ +type zopfliCostModel struct { + cost_cmd_ [numCommandSymbols]float32 + cost_dist_ []float32 + distance_histogram_size uint32 + literal_costs_ []float32 + min_cost_cmd_ float32 + num_bytes_ uint +} + +func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) { + var distance_histogram_size uint32 = dist.alphabet_size + if distance_histogram_size > maxEffectiveDistanceAlphabetSize { + distance_histogram_size = maxEffectiveDistanceAlphabetSize + } + + self.num_bytes_ = num_bytes + self.literal_costs_ = make([]float32, (num_bytes + 2)) + self.cost_dist_ = make([]float32, (dist.alphabet_size)) + self.distance_histogram_size = distance_histogram_size +} + +func cleanupZopfliCostModel(self *zopfliCostModel) { + self.literal_costs_ = nil + self.cost_dist_ = nil +} + +func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) { + var sum uint = 0 + var missing_symbol_sum uint + var log2sum float32 + var missing_symbol_cost float32 + var i uint + for i = 0; i < histogram_size; i++ { + sum += uint(histogram[i]) + } + + log2sum = float32(fastLog2(sum)) + missing_symbol_sum = sum + if !literal_histogram { + for i = 0; i < histogram_size; i++ { + if histogram[i] == 0 { + missing_symbol_sum++ + } + } + } + + missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2 + for i = 0; i < histogram_size; i++ { + if histogram[i] == 0 { + cost[i] = missing_symbol_cost + continue + } + + /* Shannon bits for this symbol. */ + cost[i] = log2sum - float32(fastLog2(uint(histogram[i]))) + + /* Cannot be coded with less than 1 bit */ + if cost[i] < 1 { + cost[i] = 1 + } + } +} + +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) { + var histogram_literal [numLiteralSymbols]uint32 + var histogram_cmd [numCommandSymbols]uint32 + var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 + var cost_literal [numLiteralSymbols]float32 + var pos uint = position - last_insert_len + var min_cost_cmd float32 = kInfinity + var cost_cmd []float32 = self.cost_cmd_[:] + var literal_costs []float32 + + histogram_literal = [numLiteralSymbols]uint32{} + histogram_cmd = [numCommandSymbols]uint32{} + histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} + + for i := range commands { + var inslength uint = uint(commands[i].insert_len_) + var copylength uint = uint(commandCopyLen(&commands[i])) + var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF + var cmdcode uint = uint(commands[i].cmd_prefix_) + var j uint + + histogram_cmd[cmdcode]++ + if cmdcode >= 128 { + histogram_dist[distcode]++ + } + + for j = 0; j < inslength; j++ { + histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++ + } + + pos += inslength + copylength + } + + setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:]) + setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) + setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) + + for i := 0; i < numCommandSymbols; i++ { + min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) + } + + self.min_cost_cmd_ = min_cost_cmd + { + literal_costs = self.literal_costs_ + var literal_carry float32 = 0.0 + num_bytes := int(self.num_bytes_) + literal_costs[0] = 0.0 + for i := 0; i < num_bytes; i++ { + literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]] + literal_costs[i+1] = literal_costs[i] + literal_carry + literal_carry -= literal_costs[i+1] - literal_costs[i] + } + } +} + +func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) { + var literal_costs []float32 = self.literal_costs_ + var literal_carry float32 = 0.0 + var cost_dist []float32 = self.cost_dist_ + var cost_cmd []float32 = self.cost_cmd_[:] + var num_bytes uint = self.num_bytes_ + var i uint + estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:]) + literal_costs[0] = 0.0 + for i = 0; i < num_bytes; i++ { + literal_carry += literal_costs[i+1] + literal_costs[i+1] = literal_costs[i] + literal_carry + literal_carry -= literal_costs[i+1] - literal_costs[i] + } + + for i = 0; i < numCommandSymbols; i++ { + cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i)))) + } + + for i = 0; uint32(i) < self.distance_histogram_size; i++ { + cost_dist[i] = float32(fastLog2(uint(20 + uint32(i)))) + } + + self.min_cost_cmd_ = float32(fastLog2(11)) +} + +func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 { + return self.cost_cmd_[cmdcode] +} + +func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 { + return self.cost_dist_[distcode] +} + +func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 { + return self.literal_costs_[to] - self.literal_costs_[from] +} + +func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 { + return self.min_cost_cmd_ +} + +/* REQUIRES: len >= 2, start_pos <= pos */ +/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */ +/* Maintains the "ZopfliNode array invariant". */ +func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) { + var next *zopfliNode = &nodes[pos+len] + next.length = uint32(len | (len+9-len_code)<<25) + next.distance = uint32(dist) + next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos)) + next.u.cost = cost +} + +type posData struct { + pos uint + distance_cache [4]int + costdiff float32 + cost float32 +} + +/* Maintains the smallest 8 cost difference together with their positions */ +type startPosQueue struct { + q_ [8]posData + idx_ uint +} + +func initStartPosQueue(self *startPosQueue) { + self.idx_ = 0 +} + +func startPosQueueSize(self *startPosQueue) uint { + return brotli_min_size_t(self.idx_, 8) +} + +func startPosQueuePush(self *startPosQueue, posdata *posData) { + var offset uint = ^(self.idx_) & 7 + self.idx_++ + var len uint = startPosQueueSize(self) + var i uint + var q []posData = self.q_[:] + q[offset] = *posdata + + /* Restore the sorted order. In the list of |len| items at most |len - 1| + adjacent element comparisons / swaps are required. */ + for i = 1; i < len; i++ { + if q[offset&7].costdiff > q[(offset+1)&7].costdiff { + var tmp posData = q[offset&7] + q[offset&7] = q[(offset+1)&7] + q[(offset+1)&7] = tmp + } + + offset++ + } +} + +func startPosQueueAt(self *startPosQueue, k uint) *posData { + return &self.q_[(k-self.idx_)&7] +} + +/* Returns the minimum possible copy length that can improve the cost of any */ +/* future position. */ +func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint { + var min_cost float32 = start_cost + var len uint = 2 + var next_len_bucket uint = 4 + /* Compute the minimum possible cost of reaching any future position. */ + + var next_len_offset uint = 10 + for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost { + /* We already reached (pos + len) with no more cost than the minimum + possible cost of reaching anything from this pos, so there is no point in + looking for lengths <= len. */ + len++ + + if len == next_len_offset { + /* We reached the next copy length code bucket, so we add one more + extra bit to the minimum cost. */ + min_cost += 1.0 + + next_len_offset += next_len_bucket + next_len_bucket *= 2 + } + } + + return uint(len) +} + +/* REQUIRES: nodes[pos].cost < kInfinity + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ +func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 { + var clen uint = uint(zopfliNodeCopyLength(&nodes[pos])) + var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF) + var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos])) + + /* Since |block_start + pos| is the end position of the command, the copy part + starts from |block_start + pos - clen|. Distances that are greater than + this or greater than |max_backward_limit| + |gap| are static dictionary + references, and do not update the last distances. + Also distance code 0 (last distance) does not update the last distances. */ + if pos == 0 { + return 0 + } else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 { + return uint32(pos) + } else { + return nodes[pos-clen-ilen].u.shortcut + } +} + +/* Fills in dist_cache[0..3] with the last four distances (as defined by + Section 4. of the Spec) that would be used at (block_start + pos) if we + used the shortest path of commands from block_start, computed from + nodes[0..pos]. The last four distances at block_start are in + starting_dist_cache[0..3]. + REQUIRES: nodes[pos].cost < kInfinity + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ +func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) { + var idx int = 0 + var p uint = uint(nodes[pos].u.shortcut) + for idx < 4 && p > 0 { + var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF) + var clen uint = uint(zopfliNodeCopyLength(&nodes[p])) + var dist uint = uint(zopfliNodeCopyDistance(&nodes[p])) + dist_cache[idx] = int(dist) + idx++ + + /* Because of prerequisite, p >= clen + ilen >= 2. */ + p = uint(nodes[p-clen-ilen].u.shortcut) + } + + for ; idx < 4; idx++ { + dist_cache[idx] = starting_dist_cache[0] + starting_dist_cache = starting_dist_cache[1:] + } +} + +/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it + is eligible. */ +func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) { + /* Save cost, because ComputeDistanceCache invalidates it. */ + var node_cost float32 = nodes[pos].u.cost + nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes) + if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) { + var posdata posData + posdata.pos = pos + posdata.cost = node_cost + posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos) + computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:]) + startPosQueuePush(queue, &posdata) + } +} + +/* Returns longest copy length. */ +func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint { + var cur_ix uint = block_start + pos + var cur_ix_masked uint = cur_ix & ringbuffer_mask + var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit) + var max_len uint = num_bytes - pos + var max_zopfli_len uint = maxZopfliLen(params) + var max_iters uint = maxZopfliCandidates(params) + var min_len uint + var result uint = 0 + var k uint + var gap uint = 0 + + evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes) + { + var posdata *posData = startPosQueueAt(queue, 0) + var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos)) + min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos) + } + + /* Go over the command starting positions in order of increasing cost + difference. */ + for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ { + var posdata *posData = startPosQueueAt(queue, k) + var start uint = posdata.pos + var inscode uint16 = getInsertLengthCode(pos - start) + var start_costdiff float32 = posdata.costdiff + var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos) + var best_len uint = min_len - 1 + var j uint = 0 + /* Look for last distance matches using the distance cache from this + starting position. */ + for ; j < numDistanceShortCodes && best_len < max_len; j++ { + var idx uint = uint(kDistanceCacheIndex[j]) + var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j]) + var prev_ix uint = cur_ix - backward + var len uint = 0 + var continuation byte = ringbuffer[cur_ix_masked+best_len] + if cur_ix_masked+best_len > ringbuffer_mask { + break + } + + if backward > max_distance+gap { + /* Word dictionary -> ignore. */ + continue + } + + if backward <= max_distance { + /* Regular backward reference. */ + if prev_ix >= cur_ix { + continue + } + + prev_ix &= ringbuffer_mask + if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] { + continue + } + + len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len) + } else { + continue + } + { + var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j) + var l uint + for l = best_len + 1; l <= len; l++ { + var copycode uint16 = getCopyLengthCode(l) + var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0) + var tmp float32 + if cmdcode < 128 { + tmp = base_cost + } else { + tmp = dist_cost + } + var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) + if cost < nodes[pos+l].u.cost { + updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost) + result = brotli_max_size_t(result, l) + } + + best_len = l + } + } + } + + /* At higher iterations look only for new last distance matches, since + looking only for new command start positions with the same distances + does not help much. */ + if k >= 2 { + continue + } + { + /* Loop through all possible copy lengths at this position. */ + var len uint = min_len + for j = 0; j < num_matches; j++ { + var match backwardMatch = matches[j] + var dist uint = uint(match.distance) + var is_dictionary_match bool = (dist > max_distance+gap) + var dist_code uint = dist + numDistanceShortCodes - 1 + var dist_symbol uint16 + var distextra uint32 + var distnumextra uint32 + var dist_cost float32 + var max_match_len uint + /* We already tried all possible last distance matches, so we can use + normal distance code here. */ + prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra) + + distnumextra = uint32(dist_symbol) >> 10 + dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF) + + /* Try all copy lengths up until the maximum copy length corresponding + to this distance. If the distance refers to the static dictionary, or + the maximum length is long enough, try only one maximum length. */ + max_match_len = backwardMatchLength(&match) + + if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) { + len = max_match_len + } + + for ; len <= max_match_len; len++ { + var len_code uint + if is_dictionary_match { + len_code = backwardMatchLengthCode(&match) + } else { + len_code = len + } + var copycode uint16 = getCopyLengthCode(len_code) + var cmdcode uint16 = combineLengthCodes(inscode, copycode, false) + var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) + if cost < nodes[pos+len].u.cost { + updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost) + if len > result { + result = len + } + } + } + } + } + } + + return result +} + +func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { + var index uint = num_bytes + var num_commands uint = 0 + for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 { + index-- + } + nodes[index].u.next = math.MaxUint32 + for index != 0 { + var len uint = uint(zopfliNodeCommandLength(&nodes[index])) + index -= uint(len) + nodes[index].u.next = uint32(len) + num_commands++ + } + + return num_commands +} + +/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var pos uint = 0 + var offset uint32 = nodes[0].u.next + var i uint + var gap uint = 0 + for i = 0; offset != math.MaxUint32; i++ { + var next *zopfliNode = &nodes[uint32(pos)+offset] + var copy_length uint = uint(zopfliNodeCopyLength(next)) + var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF) + pos += insert_length + offset = next.u.next + if i == 0 { + insert_length += *last_insert_len + *last_insert_len = 0 + } + { + var distance uint = uint(zopfliNodeCopyDistance(next)) + var len_code uint = uint(zopfliNodeLengthCode(next)) + var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) + var is_dictionary bool = (distance > max_distance+gap) + var dist_code uint = uint(zopfliNodeDistanceCode(next)) + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)) + + if !is_dictionary && dist_code > 0 { + dist_cache[3] = dist_cache[2] + dist_cache[2] = dist_cache[1] + dist_cache[1] = dist_cache[0] + dist_cache[0] = int(distance) + } + } + + *num_literals += insert_length + pos += copy_length + } + + *last_insert_len += num_bytes - pos +} + +func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var max_zopfli_len uint = maxZopfliLen(params) + var queue startPosQueue + var cur_match_pos uint = 0 + var i uint + nodes[0].length = 0 + nodes[0].u.cost = 0 + initStartPosQueue(&queue) + for i = 0; i+3 < num_bytes; i++ { + var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes) + if skip < longCopyQuickStep { + skip = 0 + } + cur_match_pos += uint(num_matches[i]) + if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len { + skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip) + } + + if skip > 1 { + skip-- + for skip != 0 { + i++ + if i+3 >= num_bytes { + break + } + evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes) + cur_match_pos += uint(num_matches[i]) + skip-- + } + } + } + + return computeShortestPathFromNodes(num_bytes, nodes) +} + +/* Computes the shortest path of commands from position to at most + position + num_bytes. + + On return, path->size() is the number of commands found and path[i] is the + length of the i-th command (copy length plus insert length). + Note that the sum of the lengths of all commands can be less than num_bytes. + + On return, the nodes[0..num_bytes] array will have the following + "ZopfliNode array invariant": + For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then + (1) nodes[i].copy_length() >= 2 + (2) nodes[i].command_length() <= i and + (3) nodes[i - nodes[i].command_length()].cost < kInfinity + + REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */ +func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var max_zopfli_len uint = maxZopfliLen(params) + var model zopfliCostModel + var queue startPosQueue + var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch + var store_end uint + if num_bytes >= hasher.StoreLookahead() { + store_end = position + num_bytes - hasher.StoreLookahead() + 1 + } else { + store_end = position + } + var i uint + var gap uint = 0 + var lz_matches_offset uint = 0 + nodes[0].length = 0 + nodes[0].u.cost = 0 + initZopfliCostModel(&model, ¶ms.dist, num_bytes) + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) + initStartPosQueue(&queue) + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { + var pos uint = position + i + var max_distance uint = brotli_min_size_t(pos, max_backward_limit) + var skip uint + var num_matches uint + num_matches = findAllMatchesH10(hasher, ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:]) + if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len { + matches[0] = matches[num_matches-1] + num_matches = 1 + } + + skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes) + if skip < longCopyQuickStep { + skip = 0 + } + if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len { + skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip) + } + + if skip > 1 { + /* Add the tail of the copy to the hasher. */ + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end)) + + skip-- + for skip != 0 { + i++ + if i+hasher.HashTypeLength()-1 >= num_bytes { + break + } + evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes) + skip-- + } + } + } + + cleanupZopfliCostModel(&model) + return computeShortestPathFromNodes(num_bytes, nodes) +} + +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { + var nodes []zopfliNode + nodes = make([]zopfliNode, (num_bytes + 1)) + initZopfliNodes(nodes, num_bytes+1) + zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) + nodes = nil +} + +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { + var max_backward_limit uint = maxBackwardLimit(params.lgwin) + var num_matches []uint32 = make([]uint32, num_bytes) + var matches_size uint = 4 * num_bytes + var store_end uint + if num_bytes >= hasher.StoreLookahead() { + store_end = position + num_bytes - hasher.StoreLookahead() + 1 + } else { + store_end = position + } + var cur_match_pos uint = 0 + var i uint + var orig_num_literals uint + var orig_last_insert_len uint + var orig_dist_cache [4]int + var orig_num_commands int + var model zopfliCostModel + var nodes []zopfliNode + var matches []backwardMatch = make([]backwardMatch, matches_size) + var gap uint = 0 + var shadow_matches uint = 0 + var new_array []backwardMatch + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { + var pos uint = position + i + var max_distance uint = brotli_min_size_t(pos, max_backward_limit) + var max_length uint = num_bytes - i + var num_found_matches uint + var cur_match_end uint + var j uint + + /* Ensure that we have enough free slots. */ + if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches { + var new_size uint = matches_size + if new_size == 0 { + new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches + } + + for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches { + new_size *= 2 + } + + new_array = make([]backwardMatch, new_size) + if matches_size != 0 { + copy(new_array, matches[:matches_size]) + } + + matches = new_array + matches_size = new_size + } + + num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:]) + cur_match_end = cur_match_pos + num_found_matches + for j = cur_match_pos; j+1 < cur_match_end; j++ { + assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1])) + } + + num_matches[i] = uint32(num_found_matches) + if num_found_matches > 0 { + var match_len uint = backwardMatchLength(&matches[cur_match_end-1]) + if match_len > maxZopfliLenQuality11 { + var skip uint = match_len - 1 + matches[cur_match_pos] = matches[cur_match_end-1] + cur_match_pos++ + num_matches[i] = 1 + + /* Add the tail of the copy to the hasher. */ + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end)) + var pos uint = i + for i := 0; i < int(skip); i++ { + num_matches[pos+1:][i] = 0 + } + i += skip + } else { + cur_match_pos = cur_match_end + } + } + } + + orig_num_literals = *num_literals + orig_last_insert_len = *last_insert_len + copy(orig_dist_cache[:], dist_cache[:4]) + orig_num_commands = len(*commands) + nodes = make([]zopfliNode, (num_bytes + 1)) + initZopfliCostModel(&model, ¶ms.dist, num_bytes) + for i = 0; i < 2; i++ { + initZopfliNodes(nodes, num_bytes+1) + if i == 0 { + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) + } else { + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len) + } + + *commands = (*commands)[:orig_num_commands] + *num_literals = orig_num_literals + *last_insert_len = orig_last_insert_len + copy(dist_cache, orig_dist_cache[:4]) + zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) + } + + cleanupZopfliCostModel(&model) + nodes = nil + matches = nil + num_matches = nil +} diff --git a/vendor/github.com/andybalholm/brotli/bit_cost.go b/vendor/github.com/andybalholm/brotli/bit_cost.go new file mode 100644 index 0000000000..0005fc15e6 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bit_cost.go @@ -0,0 +1,436 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Functions to estimate the bit cost of Huffman trees. */ +func shannonEntropy(population []uint32, size uint, total *uint) float64 { + var sum uint = 0 + var retval float64 = 0 + var population_end []uint32 = population[size:] + var p uint + for -cap(population) < -cap(population_end) { + p = uint(population[0]) + population = population[1:] + sum += p + retval -= float64(p) * fastLog2(p) + } + + if sum != 0 { + retval += float64(sum) * fastLog2(sum) + } + *total = sum + return retval +} + +func bitsEntropy(population []uint32, size uint) float64 { + var sum uint + var retval float64 = shannonEntropy(population, size, &sum) + if retval < float64(sum) { + /* At least one bit per literal is needed. */ + retval = float64(sum) + } + + return retval +} + +const kOneSymbolHistogramCost float64 = 12 +const kTwoSymbolHistogramCost float64 = 20 +const kThreeSymbolHistogramCost float64 = 28 +const kFourSymbolHistogramCost float64 = 37 + +func populationCostLiteral(histogram *histogramLiteral) float64 { + var data_size uint = histogramDataSizeLiteral() + var count int = 0 + var s [5]uint + var bits float64 = 0.0 + var i uint + if histogram.total_count_ == 0 { + return kOneSymbolHistogramCost + } + + for i = 0; i < data_size; i++ { + if histogram.data_[i] > 0 { + s[count] = i + count++ + if count > 4 { + break + } + } + } + + if count == 1 { + return kOneSymbolHistogramCost + } + + if count == 2 { + return kTwoSymbolHistogramCost + float64(histogram.total_count_) + } + + if count == 3 { + var histo0 uint32 = histogram.data_[s[0]] + var histo1 uint32 = histogram.data_[s[1]] + var histo2 uint32 = histogram.data_[s[2]] + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) + } + + if count == 4 { + var histo [4]uint32 + var h23 uint32 + var histomax uint32 + for i = 0; i < 4; i++ { + histo[i] = histogram.data_[s[i]] + } + + /* Sort */ + for i = 0; i < 4; i++ { + var j uint + for j = i + 1; j < 4; j++ { + if histo[j] > histo[i] { + var tmp uint32 = histo[j] + histo[j] = histo[i] + histo[i] = tmp + } + } + } + + h23 = histo[2] + histo[3] + histomax = brotli_max_uint32_t(h23, histo[0]) + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) + } + { + var max_depth uint = 1 + var depth_histo = [codeLengthCodes]uint32{0} + /* In this loop we compute the entropy of the histogram and simultaneously + build a simplified histogram of the code length codes where we use the + zero repeat code 17, but we don't use the non-zero repeat code 16. */ + + var log2total float64 = fastLog2(histogram.total_count_) + for i = 0; i < data_size; { + if histogram.data_[i] > 0 { + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = + = log2(total_count) - log2(count(symbol)) */ + + var depth uint = uint(log2p + 0.5) + /* Approximate the bit depth by round(-log2(P(symbol))) */ + bits += float64(histogram.data_[i]) * log2p + + if depth > 15 { + depth = 15 + } + + if depth > max_depth { + max_depth = depth + } + + depth_histo[depth]++ + i++ + } else { + var reps uint32 = 1 + /* Compute the run length of zeros and add the appropriate number of 0 + and 17 code length codes to the code length code histogram. */ + + var k uint + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { + reps++ + } + + i += uint(reps) + if i == data_size { + /* Don't add any cost for the last zero run, since these are encoded + only implicitly. */ + break + } + + if reps < 3 { + depth_histo[0] += reps + } else { + reps -= 2 + for reps > 0 { + depth_histo[repeatZeroCodeLength]++ + + /* Add the 3 extra bits for the 17 code length code. */ + bits += 3 + + reps >>= 3 + } + } + } + } + + /* Add the estimated encoding cost of the code length code histogram. */ + bits += float64(18 + 2*max_depth) + + /* Add the entropy of the code length code histogram. */ + bits += bitsEntropy(depth_histo[:], codeLengthCodes) + } + + return bits +} + +func populationCostCommand(histogram *histogramCommand) float64 { + var data_size uint = histogramDataSizeCommand() + var count int = 0 + var s [5]uint + var bits float64 = 0.0 + var i uint + if histogram.total_count_ == 0 { + return kOneSymbolHistogramCost + } + + for i = 0; i < data_size; i++ { + if histogram.data_[i] > 0 { + s[count] = i + count++ + if count > 4 { + break + } + } + } + + if count == 1 { + return kOneSymbolHistogramCost + } + + if count == 2 { + return kTwoSymbolHistogramCost + float64(histogram.total_count_) + } + + if count == 3 { + var histo0 uint32 = histogram.data_[s[0]] + var histo1 uint32 = histogram.data_[s[1]] + var histo2 uint32 = histogram.data_[s[2]] + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) + } + + if count == 4 { + var histo [4]uint32 + var h23 uint32 + var histomax uint32 + for i = 0; i < 4; i++ { + histo[i] = histogram.data_[s[i]] + } + + /* Sort */ + for i = 0; i < 4; i++ { + var j uint + for j = i + 1; j < 4; j++ { + if histo[j] > histo[i] { + var tmp uint32 = histo[j] + histo[j] = histo[i] + histo[i] = tmp + } + } + } + + h23 = histo[2] + histo[3] + histomax = brotli_max_uint32_t(h23, histo[0]) + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) + } + { + var max_depth uint = 1 + var depth_histo = [codeLengthCodes]uint32{0} + /* In this loop we compute the entropy of the histogram and simultaneously + build a simplified histogram of the code length codes where we use the + zero repeat code 17, but we don't use the non-zero repeat code 16. */ + + var log2total float64 = fastLog2(histogram.total_count_) + for i = 0; i < data_size; { + if histogram.data_[i] > 0 { + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = + = log2(total_count) - log2(count(symbol)) */ + + var depth uint = uint(log2p + 0.5) + /* Approximate the bit depth by round(-log2(P(symbol))) */ + bits += float64(histogram.data_[i]) * log2p + + if depth > 15 { + depth = 15 + } + + if depth > max_depth { + max_depth = depth + } + + depth_histo[depth]++ + i++ + } else { + var reps uint32 = 1 + /* Compute the run length of zeros and add the appropriate number of 0 + and 17 code length codes to the code length code histogram. */ + + var k uint + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { + reps++ + } + + i += uint(reps) + if i == data_size { + /* Don't add any cost for the last zero run, since these are encoded + only implicitly. */ + break + } + + if reps < 3 { + depth_histo[0] += reps + } else { + reps -= 2 + for reps > 0 { + depth_histo[repeatZeroCodeLength]++ + + /* Add the 3 extra bits for the 17 code length code. */ + bits += 3 + + reps >>= 3 + } + } + } + } + + /* Add the estimated encoding cost of the code length code histogram. */ + bits += float64(18 + 2*max_depth) + + /* Add the entropy of the code length code histogram. */ + bits += bitsEntropy(depth_histo[:], codeLengthCodes) + } + + return bits +} + +func populationCostDistance(histogram *histogramDistance) float64 { + var data_size uint = histogramDataSizeDistance() + var count int = 0 + var s [5]uint + var bits float64 = 0.0 + var i uint + if histogram.total_count_ == 0 { + return kOneSymbolHistogramCost + } + + for i = 0; i < data_size; i++ { + if histogram.data_[i] > 0 { + s[count] = i + count++ + if count > 4 { + break + } + } + } + + if count == 1 { + return kOneSymbolHistogramCost + } + + if count == 2 { + return kTwoSymbolHistogramCost + float64(histogram.total_count_) + } + + if count == 3 { + var histo0 uint32 = histogram.data_[s[0]] + var histo1 uint32 = histogram.data_[s[1]] + var histo2 uint32 = histogram.data_[s[2]] + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) + } + + if count == 4 { + var histo [4]uint32 + var h23 uint32 + var histomax uint32 + for i = 0; i < 4; i++ { + histo[i] = histogram.data_[s[i]] + } + + /* Sort */ + for i = 0; i < 4; i++ { + var j uint + for j = i + 1; j < 4; j++ { + if histo[j] > histo[i] { + var tmp uint32 = histo[j] + histo[j] = histo[i] + histo[i] = tmp + } + } + } + + h23 = histo[2] + histo[3] + histomax = brotli_max_uint32_t(h23, histo[0]) + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) + } + { + var max_depth uint = 1 + var depth_histo = [codeLengthCodes]uint32{0} + /* In this loop we compute the entropy of the histogram and simultaneously + build a simplified histogram of the code length codes where we use the + zero repeat code 17, but we don't use the non-zero repeat code 16. */ + + var log2total float64 = fastLog2(histogram.total_count_) + for i = 0; i < data_size; { + if histogram.data_[i] > 0 { + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = + = log2(total_count) - log2(count(symbol)) */ + + var depth uint = uint(log2p + 0.5) + /* Approximate the bit depth by round(-log2(P(symbol))) */ + bits += float64(histogram.data_[i]) * log2p + + if depth > 15 { + depth = 15 + } + + if depth > max_depth { + max_depth = depth + } + + depth_histo[depth]++ + i++ + } else { + var reps uint32 = 1 + /* Compute the run length of zeros and add the appropriate number of 0 + and 17 code length codes to the code length code histogram. */ + + var k uint + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { + reps++ + } + + i += uint(reps) + if i == data_size { + /* Don't add any cost for the last zero run, since these are encoded + only implicitly. */ + break + } + + if reps < 3 { + depth_histo[0] += reps + } else { + reps -= 2 + for reps > 0 { + depth_histo[repeatZeroCodeLength]++ + + /* Add the 3 extra bits for the 17 code length code. */ + bits += 3 + + reps >>= 3 + } + } + } + } + + /* Add the estimated encoding cost of the code length code histogram. */ + bits += float64(18 + 2*max_depth) + + /* Add the entropy of the code length code histogram. */ + bits += bitsEntropy(depth_histo[:], codeLengthCodes) + } + + return bits +} diff --git a/vendor/github.com/andybalholm/brotli/bit_reader.go b/vendor/github.com/andybalholm/brotli/bit_reader.go new file mode 100644 index 0000000000..fba8687c69 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bit_reader.go @@ -0,0 +1,266 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Bit reading helpers */ + +const shortFillBitWindowRead = (8 >> 1) + +var kBitMask = [33]uint32{ + 0x00000000, + 0x00000001, + 0x00000003, + 0x00000007, + 0x0000000F, + 0x0000001F, + 0x0000003F, + 0x0000007F, + 0x000000FF, + 0x000001FF, + 0x000003FF, + 0x000007FF, + 0x00000FFF, + 0x00001FFF, + 0x00003FFF, + 0x00007FFF, + 0x0000FFFF, + 0x0001FFFF, + 0x0003FFFF, + 0x0007FFFF, + 0x000FFFFF, + 0x001FFFFF, + 0x003FFFFF, + 0x007FFFFF, + 0x00FFFFFF, + 0x01FFFFFF, + 0x03FFFFFF, + 0x07FFFFFF, + 0x0FFFFFFF, + 0x1FFFFFFF, + 0x3FFFFFFF, + 0x7FFFFFFF, + 0xFFFFFFFF, +} + +func bitMask(n uint32) uint32 { + return kBitMask[n] +} + +type bitReader struct { + val_ uint64 + bit_pos_ uint32 + input []byte + input_len uint + byte_pos uint +} + +type bitReaderState struct { + val_ uint64 + bit_pos_ uint32 + input []byte + input_len uint + byte_pos uint +} + +/* Initializes the BrotliBitReader fields. */ + +/* Ensures that accumulator is not empty. + May consume up to sizeof(brotli_reg_t) - 1 bytes of input. + Returns false if data is required but there is no input available. + For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned + reading. */ +func bitReaderSaveState(from *bitReader, to *bitReaderState) { + to.val_ = from.val_ + to.bit_pos_ = from.bit_pos_ + to.input = from.input + to.input_len = from.input_len + to.byte_pos = from.byte_pos +} + +func bitReaderRestoreState(to *bitReader, from *bitReaderState) { + to.val_ = from.val_ + to.bit_pos_ = from.bit_pos_ + to.input = from.input + to.input_len = from.input_len + to.byte_pos = from.byte_pos +} + +func getAvailableBits(br *bitReader) uint32 { + return 64 - br.bit_pos_ +} + +/* Returns amount of unread bytes the bit reader still has buffered from the + BrotliInput, including whole bytes in br->val_. */ +func getRemainingBytes(br *bitReader) uint { + return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3)) +} + +/* Checks if there is at least |num| bytes left in the input ring-buffer + (excluding the bits remaining in br->val_). */ +func checkInputAmount(br *bitReader, num uint) bool { + return br.input_len-br.byte_pos >= num +} + +/* Guarantees that there are at least |n_bits| + 1 bits in accumulator. + Precondition: accumulator contains at least 1 bit. + |n_bits| should be in the range [1..24] for regular build. For portable + non-64-bit little-endian build only 16 bits are safe to request. */ +func fillBitWindow(br *bitReader, n_bits uint32) { + if br.bit_pos_ >= 32 { + br.val_ >>= 32 + br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */ + br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32 + br.byte_pos += 4 + } +} + +/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no + more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */ +func fillBitWindow16(br *bitReader) { + fillBitWindow(br, 17) +} + +/* Tries to pull one byte of input to accumulator. + Returns false if there is no input available. */ +func pullByte(br *bitReader) bool { + if br.byte_pos == br.input_len { + return false + } + + br.val_ >>= 8 + br.val_ |= (uint64(br.input[br.byte_pos])) << 56 + br.bit_pos_ -= 8 + br.byte_pos++ + return true +} + +/* Returns currently available bits. + The number of valid bits could be calculated by BrotliGetAvailableBits. */ +func getBitsUnmasked(br *bitReader) uint64 { + return br.val_ >> br.bit_pos_ +} + +/* Like BrotliGetBits, but does not mask the result. + The result contains at least 16 valid bits. */ +func get16BitsUnmasked(br *bitReader) uint32 { + fillBitWindow(br, 16) + return uint32(getBitsUnmasked(br)) +} + +/* Returns the specified number of bits from |br| without advancing bit + position. */ +func getBits(br *bitReader, n_bits uint32) uint32 { + fillBitWindow(br, n_bits) + return uint32(getBitsUnmasked(br)) & bitMask(n_bits) +} + +/* Tries to peek the specified amount of bits. Returns false, if there + is not enough input. */ +func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool { + for getAvailableBits(br) < n_bits { + if !pullByte(br) { + return false + } + } + + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) + return true +} + +/* Advances the bit pos by |n_bits|. */ +func dropBits(br *bitReader, n_bits uint32) { + br.bit_pos_ += n_bits +} + +func bitReaderUnload(br *bitReader) { + var unused_bytes uint32 = getAvailableBits(br) >> 3 + var unused_bits uint32 = unused_bytes << 3 + br.byte_pos -= uint(unused_bytes) + if unused_bits == 64 { + br.val_ = 0 + } else { + br.val_ <<= unused_bits + } + + br.bit_pos_ += unused_bits +} + +/* Reads the specified number of bits from |br| and advances the bit pos. + Precondition: accumulator MUST contain at least |n_bits|. */ +func takeBits(br *bitReader, n_bits uint32, val *uint32) { + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) + dropBits(br, n_bits) +} + +/* Reads the specified number of bits from |br| and advances the bit pos. + Assumes that there is enough input to perform BrotliFillBitWindow. */ +func readBits(br *bitReader, n_bits uint32) uint32 { + var val uint32 + fillBitWindow(br, n_bits) + takeBits(br, n_bits, &val) + return val +} + +/* Tries to read the specified amount of bits. Returns false, if there + is not enough input. |n_bits| MUST be positive. */ +func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool { + for getAvailableBits(br) < n_bits { + if !pullByte(br) { + return false + } + } + + takeBits(br, n_bits, val) + return true +} + +/* Advances the bit reader position to the next byte boundary and verifies + that any skipped bits are set to zero. */ +func bitReaderJumpToByteBoundary(br *bitReader) bool { + var pad_bits_count uint32 = getAvailableBits(br) & 0x7 + var pad_bits uint32 = 0 + if pad_bits_count != 0 { + takeBits(br, pad_bits_count, &pad_bits) + } + + return pad_bits == 0 +} + +/* Copies remaining input bytes stored in the bit reader to the output. Value + |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be + warmed up again after this. */ +func copyBytes(dest []byte, br *bitReader, num uint) { + for getAvailableBits(br) >= 8 && num > 0 { + dest[0] = byte(getBitsUnmasked(br)) + dropBits(br, 8) + dest = dest[1:] + num-- + } + + copy(dest, br.input[br.byte_pos:][:num]) + br.byte_pos += num +} + +func initBitReader(br *bitReader) { + br.val_ = 0 + br.bit_pos_ = 64 +} + +func warmupBitReader(br *bitReader) bool { + /* Fixing alignment after unaligned BrotliFillWindow would result accumulator + overflow. If unalignment is caused by BrotliSafeReadBits, then there is + enough space in accumulator to fix alignment. */ + if getAvailableBits(br) == 0 { + if !pullByte(br) { + return false + } + } + + return true +} diff --git a/vendor/github.com/andybalholm/brotli/bitwriter.go b/vendor/github.com/andybalholm/brotli/bitwriter.go new file mode 100644 index 0000000000..dfc60360f3 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/bitwriter.go @@ -0,0 +1,56 @@ +package brotli + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Write bits into a byte array. */ + +type bitWriter struct { + dst []byte + + // Data waiting to be written is the low nbits of bits. + bits uint64 + nbits uint +} + +func (w *bitWriter) writeBits(nb uint, b uint64) { + w.bits |= b << w.nbits + w.nbits += nb + if w.nbits >= 32 { + bits := w.bits + w.bits >>= 32 + w.nbits -= 32 + w.dst = append(w.dst, + byte(bits), + byte(bits>>8), + byte(bits>>16), + byte(bits>>24), + ) + } +} + +func (w *bitWriter) writeSingleBit(bit bool) { + if bit { + w.writeBits(1, 1) + } else { + w.writeBits(1, 0) + } +} + +func (w *bitWriter) jumpToByteBoundary() { + dst := w.dst + for w.nbits != 0 { + dst = append(dst, byte(w.bits)) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + } + w.bits = 0 + w.dst = dst +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter.go b/vendor/github.com/andybalholm/brotli/block_splitter.go new file mode 100644 index 0000000000..978a131474 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter.go @@ -0,0 +1,144 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Block split point selection utilities. */ + +type blockSplit struct { + num_types uint + num_blocks uint + types []byte + lengths []uint32 + types_alloc_size uint + lengths_alloc_size uint +} + +const ( + kMaxLiteralHistograms uint = 100 + kMaxCommandHistograms uint = 50 + kLiteralBlockSwitchCost float64 = 28.1 + kCommandBlockSwitchCost float64 = 13.5 + kDistanceBlockSwitchCost float64 = 14.6 + kLiteralStrideLength uint = 70 + kCommandStrideLength uint = 40 + kSymbolsPerLiteralHistogram uint = 544 + kSymbolsPerCommandHistogram uint = 530 + kSymbolsPerDistanceHistogram uint = 544 + kMinLengthForBlockSplitting uint = 128 + kIterMulForRefining uint = 2 + kMinItersForRefining uint = 100 +) + +func countLiterals(cmds []command) uint { + var total_length uint = 0 + /* Count how many we have. */ + + for i := range cmds { + total_length += uint(cmds[i].insert_len_) + } + + return total_length +} + +func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) { + var pos uint = 0 + var from_pos uint = offset & mask + for i := range cmds { + var insert_len uint = uint(cmds[i].insert_len_) + if from_pos+insert_len > mask { + var head_size uint = mask + 1 - from_pos + copy(literals[pos:], data[from_pos:][:head_size]) + from_pos = 0 + pos += head_size + insert_len -= head_size + } + + if insert_len > 0 { + copy(literals[pos:], data[from_pos:][:insert_len]) + pos += insert_len + } + + from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask)) + } +} + +func myRand(seed *uint32) uint32 { + /* Initial seed should be 7. In this case, loop length is (1 << 29). */ + *seed *= 16807 + + return *seed +} + +func bitCost(count uint) float64 { + if count == 0 { + return -2.0 + } else { + return fastLog2(count) + } +} + +const histogramsPerBatch = 64 + +const clustersPerBatch = 16 + +func initBlockSplit(self *blockSplit) { + self.num_types = 0 + self.num_blocks = 0 + self.types = self.types[:0] + self.lengths = self.lengths[:0] + self.types_alloc_size = 0 + self.lengths_alloc_size = 0 +} + +func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { + { + var literals_count uint = countLiterals(cmds) + var literals []byte = make([]byte, literals_count) + + /* Create a continuous array of literals. */ + copyLiteralsToByteArray(cmds, data, pos, mask, literals) + + /* Create the block split on the array of literals. + Literal histograms have alphabet size 256. */ + splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split) + + literals = nil + } + { + var insert_and_copy_codes []uint16 = make([]uint16, len(cmds)) + /* Compute prefix codes for commands. */ + + for i := range cmds { + insert_and_copy_codes[i] = cmds[i].cmd_prefix_ + } + + /* Create the block split on the array of command prefixes. */ + splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) + + /* TODO: reuse for distances? */ + + insert_and_copy_codes = nil + } + { + var distance_prefixes []uint16 = make([]uint16, len(cmds)) + var j uint = 0 + /* Create a continuous array of distance prefixes. */ + + for i := range cmds { + var cmd *command = &cmds[i] + if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { + distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF + j++ + } + } + + /* Create the block split on the array of distance prefixes. */ + splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split) + + distance_prefixes = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_command.go b/vendor/github.com/andybalholm/brotli/block_splitter_command.go new file mode 100644 index 0000000000..9dec13e4d9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter_command.go @@ -0,0 +1,434 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { + var seed uint32 = 7 + var block_length uint = length / num_histograms + var i uint + clearHistogramsCommand(histograms, num_histograms) + for i = 0; i < num_histograms; i++ { + var pos uint = length * i / num_histograms + if i != 0 { + pos += uint(myRand(&seed) % uint32(block_length)) + } + + if pos+stride >= length { + pos = length - stride - 1 + } + + histogramAddVectorCommand(&histograms[i], data[pos:], stride) + } +} + +func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) { + var pos uint = 0 + if stride >= length { + stride = length + } else { + pos = uint(myRand(seed) % uint32(length-stride+1)) + } + + histogramAddVectorCommand(sample, data[pos:], stride) +} + +func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining + var seed uint32 = 7 + var iter uint + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms + for iter = 0; iter < iters; iter++ { + var sample histogramCommand + histogramClearCommand(&sample) + randomSampleCommand(&seed, data, length, stride, &sample) + histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample) + } +} + +/* Assigns a block id from the range [0, num_histograms) to each data element + in data[0..length) and fills in block_id[0..length) with the assigned values. + Returns the number of blocks, i.e. one plus the number of block switches. */ +func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { + var data_size uint = histogramDataSizeCommand() + var bitmaplen uint = (num_histograms + 7) >> 3 + var num_blocks uint = 1 + var i uint + var j uint + assert(num_histograms <= 256) + if num_histograms <= 1 { + for i = 0; i < length; i++ { + block_id[i] = 0 + } + + return 1 + } + + for i := 0; i < int(data_size*num_histograms); i++ { + insert_cost[i] = 0 + } + for i = 0; i < num_histograms; i++ { + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) + } + + for i = data_size; i != 0; { + i-- + for j = 0; j < num_histograms; j++ { + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) + } + } + + for i := 0; i < int(num_histograms); i++ { + cost[i] = 0 + } + for i := 0; i < int(length*bitmaplen); i++ { + switch_signal[i] = 0 + } + + /* After each iteration of this loop, cost[k] will contain the difference + between the minimum cost of arriving at the current byte position using + entropy code k, and the minimum cost of arriving at the current byte + position. This difference is capped at the block switch cost, and if it + reaches block switch cost, it means that when we trace back from the last + position, we need to switch here. */ + for i = 0; i < length; i++ { + var byte_ix uint = i + var ix uint = byte_ix * bitmaplen + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms + var min_cost float64 = 1e99 + var block_switch_cost float64 = block_switch_bitcost + var k uint + for k = 0; k < num_histograms; k++ { + /* We are coding the symbol in data[byte_ix] with entropy code k. */ + cost[k] += insert_cost[insert_cost_ix+k] + + if cost[k] < min_cost { + min_cost = cost[k] + block_id[byte_ix] = byte(k) + } + } + + /* More blocks for the beginning. */ + if byte_ix < 2000 { + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 + } + + for k = 0; k < num_histograms; k++ { + cost[k] -= min_cost + if cost[k] >= block_switch_cost { + var mask byte = byte(1 << (k & 7)) + cost[k] = block_switch_cost + assert(k>>3 < bitmaplen) + switch_signal[ix+(k>>3)] |= mask + /* Trace back from the last position and switch at the marked places. */ + } + } + } + { + var byte_ix uint = length - 1 + var ix uint = byte_ix * bitmaplen + var cur_id byte = block_id[byte_ix] + for byte_ix > 0 { + var mask byte = byte(1 << (cur_id & 7)) + assert(uint(cur_id)>>3 < bitmaplen) + byte_ix-- + ix -= bitmaplen + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { + if cur_id != block_id[byte_ix] { + cur_id = block_id[byte_ix] + num_blocks++ + } + } + + block_id[byte_ix] = cur_id + } + } + + return num_blocks +} + +var remapBlockIdsCommand_kInvalidId uint16 = 256 + +func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { + var next_id uint16 = 0 + var i uint + for i = 0; i < num_histograms; i++ { + new_id[i] = remapBlockIdsCommand_kInvalidId + } + + for i = 0; i < length; i++ { + assert(uint(block_ids[i]) < num_histograms) + if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId { + new_id[block_ids[i]] = next_id + next_id++ + } + } + + for i = 0; i < length; i++ { + block_ids[i] = byte(new_id[block_ids[i]]) + assert(uint(block_ids[i]) < num_histograms) + } + + assert(uint(next_id) <= num_histograms) + return uint(next_id) +} + +func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) { + var i uint + clearHistogramsCommand(histograms, num_histograms) + for i = 0; i < length; i++ { + histogramAddCommand(&histograms[block_ids[i]], uint(data[i])) + } +} + +var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32 + +func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { + var histogram_symbols []uint32 = make([]uint32, num_blocks) + var block_lengths []uint32 = make([]uint32, num_blocks) + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch + var all_histograms_size uint = 0 + var all_histograms_capacity uint = expected_num_clusters + var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity) + var cluster_size_size uint = 0 + var cluster_size_capacity uint = expected_num_clusters + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) + var num_clusters uint = 0 + var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch)) + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 + var pairs_capacity uint = max_num_pairs + 1 + var pairs []histogramPair = make([]histogramPair, pairs_capacity) + var pos uint = 0 + var clusters []uint32 + var num_final_clusters uint + var new_index []uint32 + var i uint + var sizes = [histogramsPerBatch]uint32{0} + var new_clusters = [histogramsPerBatch]uint32{0} + var symbols = [histogramsPerBatch]uint32{0} + var remap = [histogramsPerBatch]uint32{0} + + for i := 0; i < int(num_blocks); i++ { + block_lengths[i] = 0 + } + { + var block_idx uint = 0 + for i = 0; i < length; i++ { + assert(block_idx < num_blocks) + block_lengths[block_idx]++ + if i+1 == length || block_ids[i] != block_ids[i+1] { + block_idx++ + } + } + + assert(block_idx == num_blocks) + } + + for i = 0; i < num_blocks; i += histogramsPerBatch { + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + var k uint + histogramClearCommand(&histograms[j]) + for k = 0; uint32(k) < block_lengths[i+j]; k++ { + histogramAddCommand(&histograms[j], uint(data[pos])) + pos++ + } + + histograms[j].bit_cost_ = populationCostCommand(&histograms[j]) + new_clusters[j] = uint32(j) + symbols[j] = uint32(j) + sizes[j] = 1 + } + + num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { + var _new_size uint + if all_histograms_capacity == 0 { + _new_size = all_histograms_size + num_new_clusters + } else { + _new_size = all_histograms_capacity + } + var new_array []histogramCommand + for _new_size < (all_histograms_size + num_new_clusters) { + _new_size *= 2 + } + new_array = make([]histogramCommand, _new_size) + if all_histograms_capacity != 0 { + copy(new_array, all_histograms[:all_histograms_capacity]) + } + + all_histograms = new_array + all_histograms_capacity = _new_size + } + + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) + for j = 0; j < num_new_clusters; j++ { + all_histograms[all_histograms_size] = histograms[new_clusters[j]] + all_histograms_size++ + cluster_size[cluster_size_size] = sizes[new_clusters[j]] + cluster_size_size++ + remap[new_clusters[j]] = uint32(j) + } + + for j = 0; j < num_to_combine; j++ { + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] + } + + num_clusters += num_new_clusters + assert(num_clusters == cluster_size_size) + assert(num_clusters == all_histograms_size) + } + + histograms = nil + + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < max_num_pairs+1 { + pairs = nil + pairs = make([]histogramPair, (max_num_pairs + 1)) + } + + clusters = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + clusters[i] = uint32(i) + } + + num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) + pairs = nil + cluster_size = nil + + new_index = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + new_index[i] = clusterBlocksCommand_kInvalidIndex + } + pos = 0 + { + var next_index uint32 = 0 + for i = 0; i < num_blocks; i++ { + var histo histogramCommand + var j uint + var best_out uint32 + var best_bits float64 + histogramClearCommand(&histo) + for j = 0; uint32(j) < block_lengths[i]; j++ { + histogramAddCommand(&histo, uint(data[pos])) + pos++ + } + + if i == 0 { + best_out = histogram_symbols[0] + } else { + best_out = histogram_symbols[i-1] + } + best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out]) + for j = 0; j < num_final_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + histogram_symbols[i] = best_out + if new_index[best_out] == clusterBlocksCommand_kInvalidIndex { + new_index[best_out] = next_index + next_index++ + } + } + } + + clusters = nil + all_histograms = nil + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) + { + var cur_length uint32 = 0 + var block_idx uint = 0 + var max_type byte = 0 + for i = 0; i < num_blocks; i++ { + cur_length += block_lengths[i] + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { + var id byte = byte(new_index[histogram_symbols[i]]) + split.types[block_idx] = id + split.lengths[block_idx] = cur_length + max_type = brotli_max_uint8_t(max_type, id) + cur_length = 0 + block_idx++ + } + } + + split.num_blocks = block_idx + split.num_types = uint(max_type) + 1 + } + + new_index = nil + block_lengths = nil + histogram_symbols = nil +} + +func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + length := uint(len(data)) + var data_size uint = histogramDataSizeCommand() + var num_histograms uint = length/literals_per_histogram + 1 + var histograms []histogramCommand + if num_histograms > max_histograms { + num_histograms = max_histograms + } + + if length == 0 { + split.num_types = 1 + return + } else if length < kMinLengthForBlockSplitting { + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) + split.num_types = 1 + split.types[split.num_blocks] = 0 + split.lengths[split.num_blocks] = uint32(length) + split.num_blocks++ + return + } + + histograms = make([]histogramCommand, num_histograms) + + /* Find good entropy codes. */ + initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) + + refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) + { + var block_ids []byte = make([]byte, length) + var num_blocks uint = 0 + var bitmaplen uint = (num_histograms + 7) >> 3 + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) + var cost []float64 = make([]float64, num_histograms) + var switch_signal []byte = make([]byte, (length * bitmaplen)) + var new_id []uint16 = make([]uint16, num_histograms) + var iters uint + if params.quality < hqZopflificationQuality { + iters = 3 + } else { + iters = 10 + } + /* Find a good path through literals with the good entropy codes. */ + + var i uint + for i = 0; i < iters; i++ { + num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) + num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms) + buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms) + } + + insert_cost = nil + cost = nil + switch_signal = nil + new_id = nil + histograms = nil + clusterBlocksCommand(data, length, num_blocks, block_ids, split) + block_ids = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_distance.go b/vendor/github.com/andybalholm/brotli/block_splitter_distance.go new file mode 100644 index 0000000000..953530d518 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter_distance.go @@ -0,0 +1,433 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { + var seed uint32 = 7 + var block_length uint = length / num_histograms + var i uint + clearHistogramsDistance(histograms, num_histograms) + for i = 0; i < num_histograms; i++ { + var pos uint = length * i / num_histograms + if i != 0 { + pos += uint(myRand(&seed) % uint32(block_length)) + } + + if pos+stride >= length { + pos = length - stride - 1 + } + + histogramAddVectorDistance(&histograms[i], data[pos:], stride) + } +} + +func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) { + var pos uint = 0 + if stride >= length { + stride = length + } else { + pos = uint(myRand(seed) % uint32(length-stride+1)) + } + + histogramAddVectorDistance(sample, data[pos:], stride) +} + +func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining + var seed uint32 = 7 + var iter uint + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms + for iter = 0; iter < iters; iter++ { + var sample histogramDistance + histogramClearDistance(&sample) + randomSampleDistance(&seed, data, length, stride, &sample) + histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample) + } +} + +/* Assigns a block id from the range [0, num_histograms) to each data element + in data[0..length) and fills in block_id[0..length) with the assigned values. + Returns the number of blocks, i.e. one plus the number of block switches. */ +func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { + var data_size uint = histogramDataSizeDistance() + var bitmaplen uint = (num_histograms + 7) >> 3 + var num_blocks uint = 1 + var i uint + var j uint + assert(num_histograms <= 256) + if num_histograms <= 1 { + for i = 0; i < length; i++ { + block_id[i] = 0 + } + + return 1 + } + + for i := 0; i < int(data_size*num_histograms); i++ { + insert_cost[i] = 0 + } + for i = 0; i < num_histograms; i++ { + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) + } + + for i = data_size; i != 0; { + i-- + for j = 0; j < num_histograms; j++ { + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) + } + } + + for i := 0; i < int(num_histograms); i++ { + cost[i] = 0 + } + for i := 0; i < int(length*bitmaplen); i++ { + switch_signal[i] = 0 + } + + /* After each iteration of this loop, cost[k] will contain the difference + between the minimum cost of arriving at the current byte position using + entropy code k, and the minimum cost of arriving at the current byte + position. This difference is capped at the block switch cost, and if it + reaches block switch cost, it means that when we trace back from the last + position, we need to switch here. */ + for i = 0; i < length; i++ { + var byte_ix uint = i + var ix uint = byte_ix * bitmaplen + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms + var min_cost float64 = 1e99 + var block_switch_cost float64 = block_switch_bitcost + var k uint + for k = 0; k < num_histograms; k++ { + /* We are coding the symbol in data[byte_ix] with entropy code k. */ + cost[k] += insert_cost[insert_cost_ix+k] + + if cost[k] < min_cost { + min_cost = cost[k] + block_id[byte_ix] = byte(k) + } + } + + /* More blocks for the beginning. */ + if byte_ix < 2000 { + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 + } + + for k = 0; k < num_histograms; k++ { + cost[k] -= min_cost + if cost[k] >= block_switch_cost { + var mask byte = byte(1 << (k & 7)) + cost[k] = block_switch_cost + assert(k>>3 < bitmaplen) + switch_signal[ix+(k>>3)] |= mask + /* Trace back from the last position and switch at the marked places. */ + } + } + } + { + var byte_ix uint = length - 1 + var ix uint = byte_ix * bitmaplen + var cur_id byte = block_id[byte_ix] + for byte_ix > 0 { + var mask byte = byte(1 << (cur_id & 7)) + assert(uint(cur_id)>>3 < bitmaplen) + byte_ix-- + ix -= bitmaplen + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { + if cur_id != block_id[byte_ix] { + cur_id = block_id[byte_ix] + num_blocks++ + } + } + + block_id[byte_ix] = cur_id + } + } + + return num_blocks +} + +var remapBlockIdsDistance_kInvalidId uint16 = 256 + +func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { + var next_id uint16 = 0 + var i uint + for i = 0; i < num_histograms; i++ { + new_id[i] = remapBlockIdsDistance_kInvalidId + } + + for i = 0; i < length; i++ { + assert(uint(block_ids[i]) < num_histograms) + if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId { + new_id[block_ids[i]] = next_id + next_id++ + } + } + + for i = 0; i < length; i++ { + block_ids[i] = byte(new_id[block_ids[i]]) + assert(uint(block_ids[i]) < num_histograms) + } + + assert(uint(next_id) <= num_histograms) + return uint(next_id) +} + +func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) { + var i uint + clearHistogramsDistance(histograms, num_histograms) + for i = 0; i < length; i++ { + histogramAddDistance(&histograms[block_ids[i]], uint(data[i])) + } +} + +var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32 + +func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { + var histogram_symbols []uint32 = make([]uint32, num_blocks) + var block_lengths []uint32 = make([]uint32, num_blocks) + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch + var all_histograms_size uint = 0 + var all_histograms_capacity uint = expected_num_clusters + var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity) + var cluster_size_size uint = 0 + var cluster_size_capacity uint = expected_num_clusters + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) + var num_clusters uint = 0 + var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch)) + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 + var pairs_capacity uint = max_num_pairs + 1 + var pairs []histogramPair = make([]histogramPair, pairs_capacity) + var pos uint = 0 + var clusters []uint32 + var num_final_clusters uint + var new_index []uint32 + var i uint + var sizes = [histogramsPerBatch]uint32{0} + var new_clusters = [histogramsPerBatch]uint32{0} + var symbols = [histogramsPerBatch]uint32{0} + var remap = [histogramsPerBatch]uint32{0} + + for i := 0; i < int(num_blocks); i++ { + block_lengths[i] = 0 + } + { + var block_idx uint = 0 + for i = 0; i < length; i++ { + assert(block_idx < num_blocks) + block_lengths[block_idx]++ + if i+1 == length || block_ids[i] != block_ids[i+1] { + block_idx++ + } + } + + assert(block_idx == num_blocks) + } + + for i = 0; i < num_blocks; i += histogramsPerBatch { + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + var k uint + histogramClearDistance(&histograms[j]) + for k = 0; uint32(k) < block_lengths[i+j]; k++ { + histogramAddDistance(&histograms[j], uint(data[pos])) + pos++ + } + + histograms[j].bit_cost_ = populationCostDistance(&histograms[j]) + new_clusters[j] = uint32(j) + symbols[j] = uint32(j) + sizes[j] = 1 + } + + num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { + var _new_size uint + if all_histograms_capacity == 0 { + _new_size = all_histograms_size + num_new_clusters + } else { + _new_size = all_histograms_capacity + } + var new_array []histogramDistance + for _new_size < (all_histograms_size + num_new_clusters) { + _new_size *= 2 + } + new_array = make([]histogramDistance, _new_size) + if all_histograms_capacity != 0 { + copy(new_array, all_histograms[:all_histograms_capacity]) + } + + all_histograms = new_array + all_histograms_capacity = _new_size + } + + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) + for j = 0; j < num_new_clusters; j++ { + all_histograms[all_histograms_size] = histograms[new_clusters[j]] + all_histograms_size++ + cluster_size[cluster_size_size] = sizes[new_clusters[j]] + cluster_size_size++ + remap[new_clusters[j]] = uint32(j) + } + + for j = 0; j < num_to_combine; j++ { + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] + } + + num_clusters += num_new_clusters + assert(num_clusters == cluster_size_size) + assert(num_clusters == all_histograms_size) + } + + histograms = nil + + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < max_num_pairs+1 { + pairs = nil + pairs = make([]histogramPair, (max_num_pairs + 1)) + } + + clusters = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + clusters[i] = uint32(i) + } + + num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) + pairs = nil + cluster_size = nil + + new_index = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + new_index[i] = clusterBlocksDistance_kInvalidIndex + } + pos = 0 + { + var next_index uint32 = 0 + for i = 0; i < num_blocks; i++ { + var histo histogramDistance + var j uint + var best_out uint32 + var best_bits float64 + histogramClearDistance(&histo) + for j = 0; uint32(j) < block_lengths[i]; j++ { + histogramAddDistance(&histo, uint(data[pos])) + pos++ + } + + if i == 0 { + best_out = histogram_symbols[0] + } else { + best_out = histogram_symbols[i-1] + } + best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out]) + for j = 0; j < num_final_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + histogram_symbols[i] = best_out + if new_index[best_out] == clusterBlocksDistance_kInvalidIndex { + new_index[best_out] = next_index + next_index++ + } + } + } + + clusters = nil + all_histograms = nil + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) + { + var cur_length uint32 = 0 + var block_idx uint = 0 + var max_type byte = 0 + for i = 0; i < num_blocks; i++ { + cur_length += block_lengths[i] + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { + var id byte = byte(new_index[histogram_symbols[i]]) + split.types[block_idx] = id + split.lengths[block_idx] = cur_length + max_type = brotli_max_uint8_t(max_type, id) + cur_length = 0 + block_idx++ + } + } + + split.num_blocks = block_idx + split.num_types = uint(max_type) + 1 + } + + new_index = nil + block_lengths = nil + histogram_symbols = nil +} + +func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + var data_size uint = histogramDataSizeDistance() + var num_histograms uint = length/literals_per_histogram + 1 + var histograms []histogramDistance + if num_histograms > max_histograms { + num_histograms = max_histograms + } + + if length == 0 { + split.num_types = 1 + return + } else if length < kMinLengthForBlockSplitting { + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) + split.num_types = 1 + split.types[split.num_blocks] = 0 + split.lengths[split.num_blocks] = uint32(length) + split.num_blocks++ + return + } + + histograms = make([]histogramDistance, num_histograms) + + /* Find good entropy codes. */ + initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) + + refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) + { + var block_ids []byte = make([]byte, length) + var num_blocks uint = 0 + var bitmaplen uint = (num_histograms + 7) >> 3 + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) + var cost []float64 = make([]float64, num_histograms) + var switch_signal []byte = make([]byte, (length * bitmaplen)) + var new_id []uint16 = make([]uint16, num_histograms) + var iters uint + if params.quality < hqZopflificationQuality { + iters = 3 + } else { + iters = 10 + } + /* Find a good path through literals with the good entropy codes. */ + + var i uint + for i = 0; i < iters; i++ { + num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) + num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms) + buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms) + } + + insert_cost = nil + cost = nil + switch_signal = nil + new_id = nil + histograms = nil + clusterBlocksDistance(data, length, num_blocks, block_ids, split) + block_ids = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/block_splitter_literal.go b/vendor/github.com/andybalholm/brotli/block_splitter_literal.go new file mode 100644 index 0000000000..1c895cf388 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/block_splitter_literal.go @@ -0,0 +1,433 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { + var seed uint32 = 7 + var block_length uint = length / num_histograms + var i uint + clearHistogramsLiteral(histograms, num_histograms) + for i = 0; i < num_histograms; i++ { + var pos uint = length * i / num_histograms + if i != 0 { + pos += uint(myRand(&seed) % uint32(block_length)) + } + + if pos+stride >= length { + pos = length - stride - 1 + } + + histogramAddVectorLiteral(&histograms[i], data[pos:], stride) + } +} + +func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) { + var pos uint = 0 + if stride >= length { + stride = length + } else { + pos = uint(myRand(seed) % uint32(length-stride+1)) + } + + histogramAddVectorLiteral(sample, data[pos:], stride) +} + +func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining + var seed uint32 = 7 + var iter uint + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms + for iter = 0; iter < iters; iter++ { + var sample histogramLiteral + histogramClearLiteral(&sample) + randomSampleLiteral(&seed, data, length, stride, &sample) + histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample) + } +} + +/* Assigns a block id from the range [0, num_histograms) to each data element + in data[0..length) and fills in block_id[0..length) with the assigned values. + Returns the number of blocks, i.e. one plus the number of block switches. */ +func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { + var data_size uint = histogramDataSizeLiteral() + var bitmaplen uint = (num_histograms + 7) >> 3 + var num_blocks uint = 1 + var i uint + var j uint + assert(num_histograms <= 256) + if num_histograms <= 1 { + for i = 0; i < length; i++ { + block_id[i] = 0 + } + + return 1 + } + + for i := 0; i < int(data_size*num_histograms); i++ { + insert_cost[i] = 0 + } + for i = 0; i < num_histograms; i++ { + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) + } + + for i = data_size; i != 0; { + i-- + for j = 0; j < num_histograms; j++ { + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) + } + } + + for i := 0; i < int(num_histograms); i++ { + cost[i] = 0 + } + for i := 0; i < int(length*bitmaplen); i++ { + switch_signal[i] = 0 + } + + /* After each iteration of this loop, cost[k] will contain the difference + between the minimum cost of arriving at the current byte position using + entropy code k, and the minimum cost of arriving at the current byte + position. This difference is capped at the block switch cost, and if it + reaches block switch cost, it means that when we trace back from the last + position, we need to switch here. */ + for i = 0; i < length; i++ { + var byte_ix uint = i + var ix uint = byte_ix * bitmaplen + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms + var min_cost float64 = 1e99 + var block_switch_cost float64 = block_switch_bitcost + var k uint + for k = 0; k < num_histograms; k++ { + /* We are coding the symbol in data[byte_ix] with entropy code k. */ + cost[k] += insert_cost[insert_cost_ix+k] + + if cost[k] < min_cost { + min_cost = cost[k] + block_id[byte_ix] = byte(k) + } + } + + /* More blocks for the beginning. */ + if byte_ix < 2000 { + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 + } + + for k = 0; k < num_histograms; k++ { + cost[k] -= min_cost + if cost[k] >= block_switch_cost { + var mask byte = byte(1 << (k & 7)) + cost[k] = block_switch_cost + assert(k>>3 < bitmaplen) + switch_signal[ix+(k>>3)] |= mask + /* Trace back from the last position and switch at the marked places. */ + } + } + } + { + var byte_ix uint = length - 1 + var ix uint = byte_ix * bitmaplen + var cur_id byte = block_id[byte_ix] + for byte_ix > 0 { + var mask byte = byte(1 << (cur_id & 7)) + assert(uint(cur_id)>>3 < bitmaplen) + byte_ix-- + ix -= bitmaplen + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { + if cur_id != block_id[byte_ix] { + cur_id = block_id[byte_ix] + num_blocks++ + } + } + + block_id[byte_ix] = cur_id + } + } + + return num_blocks +} + +var remapBlockIdsLiteral_kInvalidId uint16 = 256 + +func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { + var next_id uint16 = 0 + var i uint + for i = 0; i < num_histograms; i++ { + new_id[i] = remapBlockIdsLiteral_kInvalidId + } + + for i = 0; i < length; i++ { + assert(uint(block_ids[i]) < num_histograms) + if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId { + new_id[block_ids[i]] = next_id + next_id++ + } + } + + for i = 0; i < length; i++ { + block_ids[i] = byte(new_id[block_ids[i]]) + assert(uint(block_ids[i]) < num_histograms) + } + + assert(uint(next_id) <= num_histograms) + return uint(next_id) +} + +func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) { + var i uint + clearHistogramsLiteral(histograms, num_histograms) + for i = 0; i < length; i++ { + histogramAddLiteral(&histograms[block_ids[i]], uint(data[i])) + } +} + +var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32 + +func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { + var histogram_symbols []uint32 = make([]uint32, num_blocks) + var block_lengths []uint32 = make([]uint32, num_blocks) + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch + var all_histograms_size uint = 0 + var all_histograms_capacity uint = expected_num_clusters + var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity) + var cluster_size_size uint = 0 + var cluster_size_capacity uint = expected_num_clusters + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) + var num_clusters uint = 0 + var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch)) + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 + var pairs_capacity uint = max_num_pairs + 1 + var pairs []histogramPair = make([]histogramPair, pairs_capacity) + var pos uint = 0 + var clusters []uint32 + var num_final_clusters uint + var new_index []uint32 + var i uint + var sizes = [histogramsPerBatch]uint32{0} + var new_clusters = [histogramsPerBatch]uint32{0} + var symbols = [histogramsPerBatch]uint32{0} + var remap = [histogramsPerBatch]uint32{0} + + for i := 0; i < int(num_blocks); i++ { + block_lengths[i] = 0 + } + { + var block_idx uint = 0 + for i = 0; i < length; i++ { + assert(block_idx < num_blocks) + block_lengths[block_idx]++ + if i+1 == length || block_ids[i] != block_ids[i+1] { + block_idx++ + } + } + + assert(block_idx == num_blocks) + } + + for i = 0; i < num_blocks; i += histogramsPerBatch { + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + var k uint + histogramClearLiteral(&histograms[j]) + for k = 0; uint32(k) < block_lengths[i+j]; k++ { + histogramAddLiteral(&histograms[j], uint(data[pos])) + pos++ + } + + histograms[j].bit_cost_ = populationCostLiteral(&histograms[j]) + new_clusters[j] = uint32(j) + symbols[j] = uint32(j) + sizes[j] = 1 + } + + num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { + var _new_size uint + if all_histograms_capacity == 0 { + _new_size = all_histograms_size + num_new_clusters + } else { + _new_size = all_histograms_capacity + } + var new_array []histogramLiteral + for _new_size < (all_histograms_size + num_new_clusters) { + _new_size *= 2 + } + new_array = make([]histogramLiteral, _new_size) + if all_histograms_capacity != 0 { + copy(new_array, all_histograms[:all_histograms_capacity]) + } + + all_histograms = new_array + all_histograms_capacity = _new_size + } + + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) + for j = 0; j < num_new_clusters; j++ { + all_histograms[all_histograms_size] = histograms[new_clusters[j]] + all_histograms_size++ + cluster_size[cluster_size_size] = sizes[new_clusters[j]] + cluster_size_size++ + remap[new_clusters[j]] = uint32(j) + } + + for j = 0; j < num_to_combine; j++ { + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] + } + + num_clusters += num_new_clusters + assert(num_clusters == cluster_size_size) + assert(num_clusters == all_histograms_size) + } + + histograms = nil + + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < max_num_pairs+1 { + pairs = nil + pairs = make([]histogramPair, (max_num_pairs + 1)) + } + + clusters = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + clusters[i] = uint32(i) + } + + num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) + pairs = nil + cluster_size = nil + + new_index = make([]uint32, num_clusters) + for i = 0; i < num_clusters; i++ { + new_index[i] = clusterBlocksLiteral_kInvalidIndex + } + pos = 0 + { + var next_index uint32 = 0 + for i = 0; i < num_blocks; i++ { + var histo histogramLiteral + var j uint + var best_out uint32 + var best_bits float64 + histogramClearLiteral(&histo) + for j = 0; uint32(j) < block_lengths[i]; j++ { + histogramAddLiteral(&histo, uint(data[pos])) + pos++ + } + + if i == 0 { + best_out = histogram_symbols[0] + } else { + best_out = histogram_symbols[i-1] + } + best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out]) + for j = 0; j < num_final_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + histogram_symbols[i] = best_out + if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex { + new_index[best_out] = next_index + next_index++ + } + } + } + + clusters = nil + all_histograms = nil + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) + { + var cur_length uint32 = 0 + var block_idx uint = 0 + var max_type byte = 0 + for i = 0; i < num_blocks; i++ { + cur_length += block_lengths[i] + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { + var id byte = byte(new_index[histogram_symbols[i]]) + split.types[block_idx] = id + split.lengths[block_idx] = cur_length + max_type = brotli_max_uint8_t(max_type, id) + cur_length = 0 + block_idx++ + } + } + + split.num_blocks = block_idx + split.num_types = uint(max_type) + 1 + } + + new_index = nil + block_lengths = nil + histogram_symbols = nil +} + +func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + var data_size uint = histogramDataSizeLiteral() + var num_histograms uint = length/literals_per_histogram + 1 + var histograms []histogramLiteral + if num_histograms > max_histograms { + num_histograms = max_histograms + } + + if length == 0 { + split.num_types = 1 + return + } else if length < kMinLengthForBlockSplitting { + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) + split.num_types = 1 + split.types[split.num_blocks] = 0 + split.lengths[split.num_blocks] = uint32(length) + split.num_blocks++ + return + } + + histograms = make([]histogramLiteral, num_histograms) + + /* Find good entropy codes. */ + initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) + + refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) + { + var block_ids []byte = make([]byte, length) + var num_blocks uint = 0 + var bitmaplen uint = (num_histograms + 7) >> 3 + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) + var cost []float64 = make([]float64, num_histograms) + var switch_signal []byte = make([]byte, (length * bitmaplen)) + var new_id []uint16 = make([]uint16, num_histograms) + var iters uint + if params.quality < hqZopflificationQuality { + iters = 3 + } else { + iters = 10 + } + /* Find a good path through literals with the good entropy codes. */ + + var i uint + for i = 0; i < iters; i++ { + num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) + num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms) + buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms) + } + + insert_cost = nil + cost = nil + switch_signal = nil + new_id = nil + histograms = nil + clusterBlocksLiteral(data, length, num_blocks, block_ids, split) + block_ids = nil + } +} diff --git a/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go new file mode 100644 index 0000000000..ee6552982b --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/brotli_bit_stream.go @@ -0,0 +1,1539 @@ +package brotli + +import ( + "math" + "sync" +) + +const maxHuffmanTreeSize = (2*numCommandSymbols + 1) + +/* +The maximum size of Huffman dictionary for distances assuming that + + NPOSTFIX = 0 and NDIRECT = 0. +*/ +const maxSimpleDistanceAlphabetSize = 140 + +/* +Represents the range of values belonging to a prefix code: + + [offset, offset + 2^nbits) +*/ +type prefixCodeRange struct { + offset uint32 + nbits uint32 +} + +var kBlockLengthPrefixCode = [numBlockLenSymbols]prefixCodeRange{ + prefixCodeRange{1, 2}, + prefixCodeRange{5, 2}, + prefixCodeRange{9, 2}, + prefixCodeRange{13, 2}, + prefixCodeRange{17, 3}, + prefixCodeRange{25, 3}, + prefixCodeRange{33, 3}, + prefixCodeRange{41, 3}, + prefixCodeRange{49, 4}, + prefixCodeRange{65, 4}, + prefixCodeRange{81, 4}, + prefixCodeRange{97, 4}, + prefixCodeRange{113, 5}, + prefixCodeRange{145, 5}, + prefixCodeRange{177, 5}, + prefixCodeRange{209, 5}, + prefixCodeRange{241, 6}, + prefixCodeRange{305, 6}, + prefixCodeRange{369, 7}, + prefixCodeRange{497, 8}, + prefixCodeRange{753, 9}, + prefixCodeRange{1265, 10}, + prefixCodeRange{2289, 11}, + prefixCodeRange{4337, 12}, + prefixCodeRange{8433, 13}, + prefixCodeRange{16625, 24}, +} + +func blockLengthPrefixCode(len uint32) uint32 { + var code uint32 + if len >= 177 { + if len >= 753 { + code = 20 + } else { + code = 14 + } + } else if len >= 41 { + code = 7 + } else { + code = 0 + } + for code < (numBlockLenSymbols-1) && len >= kBlockLengthPrefixCode[code+1].offset { + code++ + } + return code +} + +func getBlockLengthPrefixCode(len uint32, code *uint, n_extra *uint32, extra *uint32) { + *code = uint(blockLengthPrefixCode(uint32(len))) + *n_extra = kBlockLengthPrefixCode[*code].nbits + *extra = len - kBlockLengthPrefixCode[*code].offset +} + +type blockTypeCodeCalculator struct { + last_type uint + second_last_type uint +} + +func initBlockTypeCodeCalculator(self *blockTypeCodeCalculator) { + self.last_type = 1 + self.second_last_type = 0 +} + +func nextBlockTypeCode(calculator *blockTypeCodeCalculator, type_ byte) uint { + var type_code uint + if uint(type_) == calculator.last_type+1 { + type_code = 1 + } else if uint(type_) == calculator.second_last_type { + type_code = 0 + } else { + type_code = uint(type_) + 2 + } + calculator.second_last_type = calculator.last_type + calculator.last_type = uint(type_) + return type_code +} + +/* +|nibblesbits| represents the 2 bits to encode MNIBBLES (0-3) + + REQUIRES: length > 0 + REQUIRES: length <= (1 << 24) +*/ +func encodeMlen(length uint, bits *uint64, numbits *uint, nibblesbits *uint64) { + var lg uint + if length == 1 { + lg = 1 + } else { + lg = uint(log2FloorNonZero(uint(uint32(length-1)))) + 1 + } + var tmp uint + if lg < 16 { + tmp = 16 + } else { + tmp = (lg + 3) + } + var mnibbles uint = tmp / 4 + assert(length > 0) + assert(length <= 1<<24) + assert(lg <= 24) + *nibblesbits = uint64(mnibbles) - 4 + *numbits = mnibbles * 4 + *bits = uint64(length) - 1 +} + +func storeCommandExtra(cmd *command, storage_ix *uint, storage []byte) { + var copylen_code uint32 = commandCopyLenCode(cmd) + var inscode uint16 = getInsertLengthCode(uint(cmd.insert_len_)) + var copycode uint16 = getCopyLengthCode(uint(copylen_code)) + var insnumextra uint32 = getInsertExtra(inscode) + var insextraval uint64 = uint64(cmd.insert_len_) - uint64(getInsertBase(inscode)) + var copyextraval uint64 = uint64(copylen_code) - uint64(getCopyBase(copycode)) + var bits uint64 = copyextraval< 0 + REQUIRES: length <= (1 << 24) +*/ +func storeCompressedMetaBlockHeader(is_final_block bool, length uint, storage_ix *uint, storage []byte) { + var lenbits uint64 + var nlenbits uint + var nibblesbits uint64 + var is_final uint64 + if is_final_block { + is_final = 1 + } else { + is_final = 0 + } + + /* Write ISLAST bit. */ + writeBits(1, is_final, storage_ix, storage) + + /* Write ISEMPTY bit. */ + if is_final_block { + writeBits(1, 0, storage_ix, storage) + } + + encodeMlen(length, &lenbits, &nlenbits, &nibblesbits) + writeBits(2, nibblesbits, storage_ix, storage) + writeBits(nlenbits, lenbits, storage_ix, storage) + + if !is_final_block { + /* Write ISUNCOMPRESSED bit. */ + writeBits(1, 0, storage_ix, storage) + } +} + +/* +Stores the uncompressed meta-block header. + + REQUIRES: length > 0 + REQUIRES: length <= (1 << 24) +*/ +func storeUncompressedMetaBlockHeader(length uint, storage_ix *uint, storage []byte) { + var lenbits uint64 + var nlenbits uint + var nibblesbits uint64 + + /* Write ISLAST bit. + Uncompressed block cannot be the last one, so set to 0. */ + writeBits(1, 0, storage_ix, storage) + + encodeMlen(length, &lenbits, &nlenbits, &nibblesbits) + writeBits(2, nibblesbits, storage_ix, storage) + writeBits(nlenbits, lenbits, storage_ix, storage) + + /* Write ISUNCOMPRESSED bit. */ + writeBits(1, 1, storage_ix, storage) +} + +var storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder = [codeLengthCodes]byte{1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15} + +var storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols = [6]byte{0, 7, 3, 2, 1, 15} +var storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths = [6]byte{2, 4, 3, 2, 2, 4} + +func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth []byte, storage_ix *uint, storage []byte) { + var skip_some uint = 0 + var codes_to_store uint = codeLengthCodes + /* The bit lengths of the Huffman code over the code length alphabet + are compressed with the following static Huffman code: + Symbol Code + ------ ---- + 0 00 + 1 1110 + 2 110 + 3 01 + 4 10 + 5 1111 */ + + /* Throw away trailing zeros: */ + if num_codes > 1 { + for ; codes_to_store > 0; codes_to_store-- { + if code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[codes_to_store-1]] != 0 { + break + } + } + } + + if code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[0]] == 0 && code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[1]] == 0 { + skip_some = 2 /* skips two. */ + if code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[2]] == 0 { + skip_some = 3 /* skips three. */ + } + } + + writeBits(2, uint64(skip_some), storage_ix, storage) + { + var i uint + for i = skip_some; i < codes_to_store; i++ { + var l uint = uint(code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[i]]) + writeBits(uint(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths[l]), uint64(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols[l]), storage_ix, storage) + } + } +} + +func storeHuffmanTreeToBitMask(huffman_tree_size uint, huffman_tree []byte, huffman_tree_extra_bits []byte, code_length_bitdepth []byte, code_length_bitdepth_symbols []uint16, storage_ix *uint, storage []byte) { + var i uint + for i = 0; i < huffman_tree_size; i++ { + var ix uint = uint(huffman_tree[i]) + writeBits(uint(code_length_bitdepth[ix]), uint64(code_length_bitdepth_symbols[ix]), storage_ix, storage) + + /* Extra bits */ + switch ix { + case repeatPreviousCodeLength: + writeBits(2, uint64(huffman_tree_extra_bits[i]), storage_ix, storage) + + case repeatZeroCodeLength: + writeBits(3, uint64(huffman_tree_extra_bits[i]), storage_ix, storage) + } + } +} + +func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max_bits uint, storage_ix *uint, storage []byte) { + /* value of 1 indicates a simple Huffman code */ + writeBits(2, 1, storage_ix, storage) + + writeBits(2, uint64(num_symbols)-1, storage_ix, storage) /* NSYM - 1 */ + { + /* Sort */ + var i uint + for i = 0; i < num_symbols; i++ { + var j uint + for j = i + 1; j < num_symbols; j++ { + if depths[symbols[j]] < depths[symbols[i]] { + var tmp uint = symbols[j] + symbols[j] = symbols[i] + symbols[i] = tmp + } + } + } + } + + if num_symbols == 2 { + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + } else if num_symbols == 3 { + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + } else { + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[3]), storage_ix, storage) + + /* tree-select */ + var tmp int + if depths[symbols[0]] == 1 { + tmp = 1 + } else { + tmp = 0 + } + writeBits(1, uint64(tmp), storage_ix, storage) + } +} + +/* +num = alphabet size + + depths = symbol depths +*/ +func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *uint, storage []byte) { + var huffman_tree [numCommandSymbols]byte + var huffman_tree_extra_bits [numCommandSymbols]byte + var huffman_tree_size uint = 0 + var code_length_bitdepth = [codeLengthCodes]byte{0} + var code_length_bitdepth_symbols [codeLengthCodes]uint16 + var huffman_tree_histogram = [codeLengthCodes]uint32{0} + var i uint + var num_codes int = 0 + /* Write the Huffman tree into the brotli-representation. + The command alphabet is the largest, so this allocation will fit all + alphabets. */ + + var code uint = 0 + + assert(num <= numCommandSymbols) + + writeHuffmanTree(depths, num, &huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:]) + + /* Calculate the statistics of the Huffman tree in brotli-representation. */ + for i = 0; i < huffman_tree_size; i++ { + huffman_tree_histogram[huffman_tree[i]]++ + } + + for i = 0; i < codeLengthCodes; i++ { + if huffman_tree_histogram[i] != 0 { + if num_codes == 0 { + code = i + num_codes = 1 + } else if num_codes == 1 { + num_codes = 2 + break + } + } + } + + /* Calculate another Huffman tree to use for compressing both the + earlier Huffman tree with. */ + createHuffmanTree(huffman_tree_histogram[:], codeLengthCodes, 5, tree, code_length_bitdepth[:]) + + convertBitDepthsToSymbols(code_length_bitdepth[:], codeLengthCodes, code_length_bitdepth_symbols[:]) + + /* Now, we have all the data, let's start storing it */ + storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth[:], storage_ix, storage) + + if num_codes == 1 { + code_length_bitdepth[code] = 0 + } + + /* Store the real Huffman tree now. */ + storeHuffmanTreeToBitMask(huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:], code_length_bitdepth[:], code_length_bitdepth_symbols[:], storage_ix, storage) +} + +/* +Builds a Huffman tree from histogram[0:length] into depth[0:length] and + + bits[0:length] and stores the encoded tree to the bit stream. +*/ +func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { + var count uint = 0 + var s4 = [4]uint{0} + var i uint + var max_bits uint = 0 + for i = 0; i < histogram_length; i++ { + if histogram[i] != 0 { + if count < 4 { + s4[count] = i + } else if count > 4 { + break + } + + count++ + } + } + { + var max_bits_counter uint = alphabet_size - 1 + for max_bits_counter != 0 { + max_bits_counter >>= 1 + max_bits++ + } + } + + if count <= 1 { + writeBits(4, 1, storage_ix, storage) + writeBits(max_bits, uint64(s4[0]), storage_ix, storage) + depth[s4[0]] = 0 + bits[s4[0]] = 0 + return + } + + for i := 0; i < int(histogram_length); i++ { + depth[i] = 0 + } + createHuffmanTree(histogram, histogram_length, 15, tree, depth) + convertBitDepthsToSymbols(depth, histogram_length, bits) + + if count <= 4 { + storeSimpleHuffmanTree(depth, s4[:], count, max_bits, storage_ix, storage) + } else { + storeHuffmanTree(depth, histogram_length, tree, storage_ix, storage) + } +} + +func sortHuffmanTree1(v0 huffmanTree, v1 huffmanTree) bool { + return v0.total_count_ < v1.total_count_ +} + +var huffmanTreePool sync.Pool + +func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { + var count uint = 0 + var symbols = [4]uint{0} + var length uint = 0 + var total uint = histogram_total + for total != 0 { + if histogram[length] != 0 { + if count < 4 { + symbols[count] = length + } + + count++ + total -= uint(histogram[length]) + } + + length++ + } + + if count <= 1 { + writeBits(4, 1, storage_ix, storage) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + depth[symbols[0]] = 0 + bits[symbols[0]] = 0 + return + } + + for i := 0; i < int(length); i++ { + depth[i] = 0 + } + { + var max_tree_size uint = 2*length + 1 + tree, _ := huffmanTreePool.Get().(*[]huffmanTree) + if tree == nil || cap(*tree) < int(max_tree_size) { + tmp := make([]huffmanTree, max_tree_size) + tree = &tmp + } else { + *tree = (*tree)[:max_tree_size] + } + var count_limit uint32 + for count_limit = 1; ; count_limit *= 2 { + var node int = 0 + var l uint + for l = length; l != 0; { + l-- + if histogram[l] != 0 { + if histogram[l] >= count_limit { + initHuffmanTree(&(*tree)[node:][0], histogram[l], -1, int16(l)) + } else { + initHuffmanTree(&(*tree)[node:][0], count_limit, -1, int16(l)) + } + + node++ + } + } + { + var n int = node + /* Points to the next leaf node. */ /* Points to the next non-leaf node. */ + var sentinel huffmanTree + var i int = 0 + var j int = n + 1 + var k int + + sortHuffmanTreeItems(*tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) + + /* The nodes are: + [0, n): the sorted leaf nodes that we start with. + [n]: we add a sentinel here. + [n + 1, 2n): new parent nodes are added here, starting from + (n+1). These are naturally in ascending order. + [2n]: we add a sentinel at the end as well. + There will be (2n+1) elements at the end. */ + initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) + + (*tree)[node] = sentinel + node++ + (*tree)[node] = sentinel + node++ + + for k = n - 1; k > 0; k-- { + var left int + var right int + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { + left = i + i++ + } else { + left = j + j++ + } + + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { + right = i + i++ + } else { + right = j + j++ + } + + /* The sentinel node becomes the parent node. */ + (*tree)[node-1].total_count_ = (*tree)[left].total_count_ + (*tree)[right].total_count_ + + (*tree)[node-1].index_left_ = int16(left) + (*tree)[node-1].index_right_or_value_ = int16(right) + + /* Add back the last sentinel node. */ + (*tree)[node] = sentinel + node++ + } + + if setDepth(2*n-1, *tree, depth, 14) { + /* We need to pack the Huffman tree in 14 bits. If this was not + successful, add fake entities to the lowest values and retry. */ + break + } + } + } + + huffmanTreePool.Put(tree) + } + + convertBitDepthsToSymbols(depth, length, bits) + if count <= 4 { + var i uint + + /* value of 1 indicates a simple Huffman code */ + writeBits(2, 1, storage_ix, storage) + + writeBits(2, uint64(count)-1, storage_ix, storage) /* NSYM - 1 */ + + /* Sort */ + for i = 0; i < count; i++ { + var j uint + for j = i + 1; j < count; j++ { + if depth[symbols[j]] < depth[symbols[i]] { + var tmp uint = symbols[j] + symbols[j] = symbols[i] + symbols[i] = tmp + } + } + } + + if count == 2 { + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + } else if count == 3 { + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + } else { + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[3]), storage_ix, storage) + + /* tree-select */ + var tmp int + if depth[symbols[0]] == 1 { + tmp = 1 + } else { + tmp = 0 + } + writeBits(1, uint64(tmp), storage_ix, storage) + } + } else { + var previous_value byte = 8 + var i uint + + /* Complex Huffman Tree */ + storeStaticCodeLengthCode(storage_ix, storage) + + /* Actual RLE coding. */ + for i = 0; i < length; { + var value byte = depth[i] + var reps uint = 1 + var k uint + for k = i + 1; k < length && depth[k] == value; k++ { + reps++ + } + + i += reps + if value == 0 { + writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps], storage_ix, storage) + } else { + if previous_value != value { + writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value]), storage_ix, storage) + reps-- + } + + if reps < 3 { + for reps != 0 { + reps-- + writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value]), storage_ix, storage) + } + } else { + reps -= 3 + writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps], storage_ix, storage) + } + + previous_value = value + } + } + } +} + +func buildAndStoreHuffmanTreeFastBW(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, bw *bitWriter) { + var count uint = 0 + var symbols = [4]uint{0} + var length uint = 0 + var total uint = histogram_total + for total != 0 { + if histogram[length] != 0 { + if count < 4 { + symbols[count] = length + } + + count++ + total -= uint(histogram[length]) + } + + length++ + } + + if count <= 1 { + bw.writeBits(4, 1) + bw.writeBits(max_bits, uint64(symbols[0])) + depth[symbols[0]] = 0 + bits[symbols[0]] = 0 + return + } + + for i := 0; i < int(length); i++ { + depth[i] = 0 + } + { + var max_tree_size uint = 2*length + 1 + tree, _ := huffmanTreePool.Get().(*[]huffmanTree) + if tree == nil || cap(*tree) < int(max_tree_size) { + tmp := make([]huffmanTree, max_tree_size) + tree = &tmp + } else { + *tree = (*tree)[:max_tree_size] + } + var count_limit uint32 + for count_limit = 1; ; count_limit *= 2 { + var node int = 0 + var l uint + for l = length; l != 0; { + l-- + if histogram[l] != 0 { + if histogram[l] >= count_limit { + initHuffmanTree(&(*tree)[node:][0], histogram[l], -1, int16(l)) + } else { + initHuffmanTree(&(*tree)[node:][0], count_limit, -1, int16(l)) + } + + node++ + } + } + { + var n int = node + /* Points to the next leaf node. */ /* Points to the next non-leaf node. */ + var sentinel huffmanTree + var i int = 0 + var j int = n + 1 + var k int + + sortHuffmanTreeItems(*tree, uint(n), huffmanTreeComparator(sortHuffmanTree1)) + + /* The nodes are: + [0, n): the sorted leaf nodes that we start with. + [n]: we add a sentinel here. + [n + 1, 2n): new parent nodes are added here, starting from + (n+1). These are naturally in ascending order. + [2n]: we add a sentinel at the end as well. + There will be (2n+1) elements at the end. */ + initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) + + (*tree)[node] = sentinel + node++ + (*tree)[node] = sentinel + node++ + + for k = n - 1; k > 0; k-- { + var left int + var right int + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { + left = i + i++ + } else { + left = j + j++ + } + + if (*tree)[i].total_count_ <= (*tree)[j].total_count_ { + right = i + i++ + } else { + right = j + j++ + } + + /* The sentinel node becomes the parent node. */ + (*tree)[node-1].total_count_ = (*tree)[left].total_count_ + (*tree)[right].total_count_ + + (*tree)[node-1].index_left_ = int16(left) + (*tree)[node-1].index_right_or_value_ = int16(right) + + /* Add back the last sentinel node. */ + (*tree)[node] = sentinel + node++ + } + + if setDepth(2*n-1, *tree, depth, 14) { + /* We need to pack the Huffman tree in 14 bits. If this was not + successful, add fake entities to the lowest values and retry. */ + break + } + } + } + + huffmanTreePool.Put(tree) + } + + convertBitDepthsToSymbols(depth, length, bits) + if count <= 4 { + var i uint + + /* value of 1 indicates a simple Huffman code */ + bw.writeBits(2, 1) + + bw.writeBits(2, uint64(count)-1) /* NSYM - 1 */ + + /* Sort */ + for i = 0; i < count; i++ { + var j uint + for j = i + 1; j < count; j++ { + if depth[symbols[j]] < depth[symbols[i]] { + var tmp uint = symbols[j] + symbols[j] = symbols[i] + symbols[i] = tmp + } + } + } + + if count == 2 { + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + } else if count == 3 { + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) + } else { + bw.writeBits(max_bits, uint64(symbols[0])) + bw.writeBits(max_bits, uint64(symbols[1])) + bw.writeBits(max_bits, uint64(symbols[2])) + bw.writeBits(max_bits, uint64(symbols[3])) + + /* tree-select */ + bw.writeSingleBit(depth[symbols[0]] == 1) + } + } else { + var previous_value byte = 8 + var i uint + + /* Complex Huffman Tree */ + storeStaticCodeLengthCodeBW(bw) + + /* Actual RLE coding. */ + for i = 0; i < length; { + var value byte = depth[i] + var reps uint = 1 + var k uint + for k = i + 1; k < length && depth[k] == value; k++ { + reps++ + } + + i += reps + if value == 0 { + bw.writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps]) + } else { + if previous_value != value { + bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) + reps-- + } + + if reps < 3 { + for reps != 0 { + reps-- + bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) + } + } else { + reps -= 3 + bw.writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps]) + } + + previous_value = value + } + } + } +} + +func indexOf(v []byte, v_size uint, value byte) uint { + var i uint = 0 + for ; i < v_size; i++ { + if v[i] == value { + return i + } + } + + return i +} + +func moveToFront(v []byte, index uint) { + var value byte = v[index] + var i uint + for i = index; i != 0; i-- { + v[i] = v[i-1] + } + + v[0] = value +} + +func moveToFrontTransform(v_in []uint32, v_size uint, v_out []uint32) { + var i uint + var mtf [256]byte + var max_value uint32 + if v_size == 0 { + return + } + + max_value = v_in[0] + for i = 1; i < v_size; i++ { + if v_in[i] > max_value { + max_value = v_in[i] + } + } + + assert(max_value < 256) + for i = 0; uint32(i) <= max_value; i++ { + mtf[i] = byte(i) + } + { + var mtf_size uint = uint(max_value + 1) + for i = 0; i < v_size; i++ { + var index uint = indexOf(mtf[:], mtf_size, byte(v_in[i])) + assert(index < mtf_size) + v_out[i] = uint32(index) + moveToFront(mtf[:], index) + } + } +} + +/* +Finds runs of zeros in v[0..in_size) and replaces them with a prefix code of + + the run length plus extra bits (lower 9 bits is the prefix code and the rest + are the extra bits). Non-zero values in v[] are shifted by + *max_length_prefix. Will not create prefix codes bigger than the initial + value of *max_run_length_prefix. The prefix code of run length L is simply + Log2Floor(L) and the number of extra bits is the same as the prefix code. +*/ +func runLengthCodeZeros(in_size uint, v []uint32, out_size *uint, max_run_length_prefix *uint32) { + var max_reps uint32 = 0 + var i uint + var max_prefix uint32 + for i = 0; i < in_size; { + var reps uint32 = 0 + for ; i < in_size && v[i] != 0; i++ { + } + for ; i < in_size && v[i] == 0; i++ { + reps++ + } + + max_reps = brotli_max_uint32_t(reps, max_reps) + } + + if max_reps > 0 { + max_prefix = log2FloorNonZero(uint(max_reps)) + } else { + max_prefix = 0 + } + max_prefix = brotli_min_uint32_t(max_prefix, *max_run_length_prefix) + *max_run_length_prefix = max_prefix + *out_size = 0 + for i = 0; i < in_size; { + assert(*out_size <= i) + if v[i] != 0 { + v[*out_size] = v[i] + *max_run_length_prefix + i++ + (*out_size)++ + } else { + var reps uint32 = 1 + var k uint + for k = i + 1; k < in_size && v[k] == 0; k++ { + reps++ + } + + i += uint(reps) + for reps != 0 { + if reps < 2< 0) + writeSingleBit(use_rle, storage_ix, storage) + if use_rle { + writeBits(4, uint64(max_run_length_prefix)-1, storage_ix, storage) + } + } + + buildAndStoreHuffmanTree(histogram[:], uint(uint32(num_clusters)+max_run_length_prefix), uint(uint32(num_clusters)+max_run_length_prefix), tree, depths[:], bits[:], storage_ix, storage) + for i = 0; i < num_rle_symbols; i++ { + var rle_symbol uint32 = rle_symbols[i] & encodeContextMap_kSymbolMask + var extra_bits_val uint32 = rle_symbols[i] >> symbolBits + writeBits(uint(depths[rle_symbol]), uint64(bits[rle_symbol]), storage_ix, storage) + if rle_symbol > 0 && rle_symbol <= max_run_length_prefix { + writeBits(uint(rle_symbol), uint64(extra_bits_val), storage_ix, storage) + } + } + + writeBits(1, 1, storage_ix, storage) /* use move-to-front */ + rle_symbols = nil +} + +/* Stores the block switch command with index block_ix to the bit stream. */ +func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, is_first_block bool, storage_ix *uint, storage []byte) { + var typecode uint = nextBlockTypeCode(&code.type_code_calculator, block_type) + var lencode uint + var len_nextra uint32 + var len_extra uint32 + if !is_first_block { + writeBits(uint(code.type_depths[typecode]), uint64(code.type_bits[typecode]), storage_ix, storage) + } + + getBlockLengthPrefixCode(block_len, &lencode, &len_nextra, &len_extra) + + writeBits(uint(code.length_depths[lencode]), uint64(code.length_bits[lencode]), storage_ix, storage) + writeBits(uint(len_nextra), uint64(len_extra), storage_ix, storage) +} + +/* +Builds a BlockSplitCode data structure from the block split given by the + + vector of block types and block lengths and stores it to the bit stream. +*/ +func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, storage_ix *uint, storage []byte) { + var type_histo [maxBlockTypeSymbols]uint32 + var length_histo [numBlockLenSymbols]uint32 + var i uint + var type_code_calculator blockTypeCodeCalculator + for i := 0; i < int(num_types+2); i++ { + type_histo[i] = 0 + } + length_histo = [numBlockLenSymbols]uint32{} + initBlockTypeCodeCalculator(&type_code_calculator) + for i = 0; i < num_blocks; i++ { + var type_code uint = nextBlockTypeCode(&type_code_calculator, types[i]) + if i != 0 { + type_histo[type_code]++ + } + length_histo[blockLengthPrefixCode(lengths[i])]++ + } + + storeVarLenUint8(num_types-1, storage_ix, storage) + if num_types > 1 { /* TODO: else? could StoreBlockSwitch occur? */ + buildAndStoreHuffmanTree(type_histo[0:], num_types+2, num_types+2, tree, code.type_depths[0:], code.type_bits[0:], storage_ix, storage) + buildAndStoreHuffmanTree(length_histo[0:], numBlockLenSymbols, numBlockLenSymbols, tree, code.length_depths[0:], code.length_bits[0:], storage_ix, storage) + storeBlockSwitch(code, lengths[0], types[0], true, storage_ix, storage) + } +} + +/* Stores a context map where the histogram type is always the block type. */ +func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTree, storage_ix *uint, storage []byte) { + storeVarLenUint8(num_types-1, storage_ix, storage) + if num_types > 1 { + var repeat_code uint = context_bits - 1 + var repeat_bits uint = (1 << repeat_code) - 1 + var alphabet_size uint = num_types + repeat_code + var histogram [maxContextMapSymbols]uint32 + var depths [maxContextMapSymbols]byte + var bits [maxContextMapSymbols]uint16 + var i uint + for i := 0; i < int(alphabet_size); i++ { + histogram[i] = 0 + } + + /* Write RLEMAX. */ + writeBits(1, 1, storage_ix, storage) + + writeBits(4, uint64(repeat_code)-1, storage_ix, storage) + histogram[repeat_code] = uint32(num_types) + histogram[0] = 1 + for i = context_bits; i < alphabet_size; i++ { + histogram[i] = 1 + } + + buildAndStoreHuffmanTree(histogram[:], alphabet_size, alphabet_size, tree, depths[:], bits[:], storage_ix, storage) + for i = 0; i < num_types; i++ { + var tmp uint + if i == 0 { + tmp = 0 + } else { + tmp = i + context_bits - 1 + } + var code uint = tmp + writeBits(uint(depths[code]), uint64(bits[code]), storage_ix, storage) + writeBits(uint(depths[repeat_code]), uint64(bits[repeat_code]), storage_ix, storage) + writeBits(repeat_code, uint64(repeat_bits), storage_ix, storage) + } + + /* Write IMTF (inverse-move-to-front) bit. */ + writeBits(1, 1, storage_ix, storage) + } +} + +/* Manages the encoding of one block category (literal, command or distance). */ +type blockEncoder struct { + histogram_length_ uint + num_block_types_ uint + block_types_ []byte + block_lengths_ []uint32 + num_blocks_ uint + block_split_code_ blockSplitCode + block_ix_ uint + block_len_ uint + entropy_ix_ uint + depths_ []byte + bits_ []uint16 +} + +var blockEncoderPool sync.Pool + +func getBlockEncoder(histogram_length uint, num_block_types uint, block_types []byte, block_lengths []uint32, num_blocks uint) *blockEncoder { + self, _ := blockEncoderPool.Get().(*blockEncoder) + + if self != nil { + self.block_ix_ = 0 + self.entropy_ix_ = 0 + self.depths_ = self.depths_[:0] + self.bits_ = self.bits_[:0] + } else { + self = &blockEncoder{} + } + + self.histogram_length_ = histogram_length + self.num_block_types_ = num_block_types + self.block_types_ = block_types + self.block_lengths_ = block_lengths + self.num_blocks_ = num_blocks + initBlockTypeCodeCalculator(&self.block_split_code_.type_code_calculator) + if num_blocks == 0 { + self.block_len_ = 0 + } else { + self.block_len_ = uint(block_lengths[0]) + } + + return self +} + +func cleanupBlockEncoder(self *blockEncoder) { + blockEncoderPool.Put(self) +} + +/* +Creates entropy codes of block lengths and block types and stores them + + to the bit stream. +*/ +func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, storage_ix *uint, storage []byte) { + buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, storage_ix, storage) +} + +/* +Stores the next symbol with the entropy code of the current block type. + + Updates the block type and block length at block boundaries. +*/ +func storeSymbol(self *blockEncoder, symbol uint, storage_ix *uint, storage []byte) { + if self.block_len_ == 0 { + self.block_ix_++ + var block_ix uint = self.block_ix_ + var block_len uint32 = self.block_lengths_[block_ix] + var block_type byte = self.block_types_[block_ix] + self.block_len_ = uint(block_len) + self.entropy_ix_ = uint(block_type) * self.histogram_length_ + storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, storage_ix, storage) + } + + self.block_len_-- + { + var ix uint = self.entropy_ix_ + symbol + writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix]), storage_ix, storage) + } +} + +/* +Stores the next symbol with the entropy code of the current block type and + + context value. + Updates the block type and block length at block boundaries. +*/ +func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, context_map []uint32, storage_ix *uint, storage []byte, context_bits uint) { + if self.block_len_ == 0 { + self.block_ix_++ + var block_ix uint = self.block_ix_ + var block_len uint32 = self.block_lengths_[block_ix] + var block_type byte = self.block_types_[block_ix] + self.block_len_ = uint(block_len) + self.entropy_ix_ = uint(block_type) << context_bits + storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, storage_ix, storage) + } + + self.block_len_-- + { + var histo_ix uint = uint(context_map[self.entropy_ix_+context]) + var ix uint = histo_ix*self.histogram_length_ + symbol + writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix]), storage_ix, storage) + } +} + +func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { + var table_size uint = histograms_size * self.histogram_length_ + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } + { + var i uint + for i = 0; i < histograms_size; i++ { + var ix uint = i * self.histogram_length_ + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) + } + } +} + +func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { + var table_size uint = histograms_size * self.histogram_length_ + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } + { + var i uint + for i = 0; i < histograms_size; i++ { + var ix uint = i * self.histogram_length_ + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) + } + } +} + +func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { + var table_size uint = histograms_size * self.histogram_length_ + if cap(self.depths_) < int(table_size) { + self.depths_ = make([]byte, table_size) + } else { + self.depths_ = self.depths_[:table_size] + } + if cap(self.bits_) < int(table_size) { + self.bits_ = make([]uint16, table_size) + } else { + self.bits_ = self.bits_[:table_size] + } + { + var i uint + for i = 0; i < histograms_size; i++ { + var ix uint = i * self.histogram_length_ + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) + } + } +} + +func jumpToByteBoundary(storage_ix *uint, storage []byte) { + *storage_ix = (*storage_ix + 7) &^ 7 + storage[*storage_ix>>3] = 0 +} + +func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) { + var pos uint = start_pos + var i uint + var num_distance_symbols uint32 = params.dist.alphabet_size + var num_effective_distance_symbols uint32 = num_distance_symbols + var tree []huffmanTree + var literal_context_lut contextLUT = getContextLUT(literal_context_mode) + var dist *distanceParams = ¶ms.dist + if params.large_window && num_effective_distance_symbols > numHistogramDistanceSymbols { + num_effective_distance_symbols = numHistogramDistanceSymbols + } + + storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) + + tree = make([]huffmanTree, maxHuffmanTreeSize) + literal_enc := getBlockEncoder(numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks) + command_enc := getBlockEncoder(numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks) + distance_enc := getBlockEncoder(uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks) + + buildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(command_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, storage_ix, storage) + + writeBits(2, uint64(dist.distance_postfix_bits), storage_ix, storage) + writeBits(4, uint64(dist.num_direct_distance_codes)>>dist.distance_postfix_bits, storage_ix, storage) + for i = 0; i < mb.literal_split.num_types; i++ { + writeBits(2, uint64(literal_context_mode), storage_ix, storage) + } + + if mb.literal_context_map_size == 0 { + storeTrivialContextMap(mb.literal_histograms_size, literalContextBits, tree, storage_ix, storage) + } else { + encodeContextMap(mb.literal_context_map, mb.literal_context_map_size, mb.literal_histograms_size, tree, storage_ix, storage) + } + + if mb.distance_context_map_size == 0 { + storeTrivialContextMap(mb.distance_histograms_size, distanceContextBits, tree, storage_ix, storage) + } else { + encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, storage_ix, storage) + } + + buildAndStoreEntropyCodesLiteral(literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, storage_ix, storage) + buildAndStoreEntropyCodesCommand(command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, storage_ix, storage) + buildAndStoreEntropyCodesDistance(distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) + tree = nil + + for _, cmd := range commands { + var cmd_code uint = uint(cmd.cmd_prefix_) + storeSymbol(command_enc, cmd_code, storage_ix, storage) + storeCommandExtra(&cmd, storage_ix, storage) + if mb.literal_context_map_size == 0 { + var j uint + for j = uint(cmd.insert_len_); j != 0; j-- { + storeSymbol(literal_enc, uint(input[pos&mask]), storage_ix, storage) + pos++ + } + } else { + var j uint + for j = uint(cmd.insert_len_); j != 0; j-- { + var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut)) + var literal byte = input[pos&mask] + storeSymbolWithContext(literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, literalContextBits) + prev_byte2 = prev_byte + prev_byte = literal + pos++ + } + } + + pos += uint(commandCopyLen(&cmd)) + if commandCopyLen(&cmd) != 0 { + prev_byte2 = input[(pos-2)&mask] + prev_byte = input[(pos-1)&mask] + if cmd.cmd_prefix_ >= 128 { + var dist_code uint = uint(cmd.dist_prefix_) & 0x3FF + var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 + var distextra uint64 = uint64(cmd.dist_extra_) + if mb.distance_context_map_size == 0 { + storeSymbol(distance_enc, dist_code, storage_ix, storage) + } else { + var context uint = uint(commandDistanceContext(&cmd)) + storeSymbolWithContext(distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, distanceContextBits) + } + + writeBits(uint(distnumextra), distextra, storage_ix, storage) + } + } + } + + cleanupBlockEncoder(distance_enc) + cleanupBlockEncoder(command_enc) + cleanupBlockEncoder(literal_enc) + if is_last { + jumpToByteBoundary(storage_ix, storage) + } +} + +func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { + var pos uint = start_pos + for _, cmd := range commands { + var j uint + histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_)) + for j = uint(cmd.insert_len_); j != 0; j-- { + histogramAddLiteral(lit_histo, uint(input[pos&mask])) + pos++ + } + + pos += uint(commandCopyLen(&cmd)) + if commandCopyLen(&cmd) != 0 && cmd.cmd_prefix_ >= 128 { + histogramAddDistance(dist_histo, uint(cmd.dist_prefix_)&0x3FF) + } + } +} + +func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { + var pos uint = start_pos + for _, cmd := range commands { + var cmd_code uint = uint(cmd.cmd_prefix_) + var j uint + writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage) + storeCommandExtra(&cmd, storage_ix, storage) + for j = uint(cmd.insert_len_); j != 0; j-- { + var literal byte = input[pos&mask] + writeBits(uint(lit_depth[literal]), uint64(lit_bits[literal]), storage_ix, storage) + pos++ + } + + pos += uint(commandCopyLen(&cmd)) + if commandCopyLen(&cmd) != 0 && cmd.cmd_prefix_ >= 128 { + var dist_code uint = uint(cmd.dist_prefix_) & 0x3FF + var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 + var distextra uint32 = cmd.dist_extra_ + writeBits(uint(dist_depth[dist_code]), uint64(dist_bits[dist_code]), storage_ix, storage) + writeBits(uint(distnumextra), uint64(distextra), storage_ix, storage) + } + } +} + +func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { + var lit_histo histogramLiteral + var cmd_histo histogramCommand + var dist_histo histogramDistance + var lit_depth [numLiteralSymbols]byte + var lit_bits [numLiteralSymbols]uint16 + var cmd_depth [numCommandSymbols]byte + var cmd_bits [numCommandSymbols]uint16 + var dist_depth [maxSimpleDistanceAlphabetSize]byte + var dist_bits [maxSimpleDistanceAlphabetSize]uint16 + var tree []huffmanTree + var num_distance_symbols uint32 = params.dist.alphabet_size + + storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) + + histogramClearLiteral(&lit_histo) + histogramClearCommand(&cmd_histo) + histogramClearDistance(&dist_histo) + + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) + + writeBits(13, 0, storage_ix, storage) + + tree = make([]huffmanTree, maxHuffmanTreeSize) + buildAndStoreHuffmanTree(lit_histo.data_[:], numLiteralSymbols, numLiteralSymbols, tree, lit_depth[:], lit_bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage) + tree = nil + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + if is_last { + jumpToByteBoundary(storage_ix, storage) + } +} + +func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { + var num_distance_symbols uint32 = params.dist.alphabet_size + var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1 + + storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) + + writeBits(13, 0, storage_ix, storage) + + if len(commands) <= 128 { + var histogram = [numLiteralSymbols]uint32{0} + var pos uint = start_pos + var num_literals uint = 0 + var lit_depth [numLiteralSymbols]byte + var lit_bits [numLiteralSymbols]uint16 + for _, cmd := range commands { + var j uint + for j = uint(cmd.insert_len_); j != 0; j-- { + histogram[input[pos&mask]]++ + pos++ + } + + num_literals += uint(cmd.insert_len_) + pos += uint(commandCopyLen(&cmd)) + } + + buildAndStoreHuffmanTreeFast(histogram[:], num_literals, /* max_bits = */ + 8, lit_depth[:], lit_bits[:], storage_ix, storage) + + storeStaticCommandHuffmanTree(storage_ix, storage) + storeStaticDistanceHuffmanTree(storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) + } else { + var lit_histo histogramLiteral + var cmd_histo histogramCommand + var dist_histo histogramDistance + var lit_depth [numLiteralSymbols]byte + var lit_bits [numLiteralSymbols]uint16 + var cmd_depth [numCommandSymbols]byte + var cmd_bits [numCommandSymbols]uint16 + var dist_depth [maxSimpleDistanceAlphabetSize]byte + var dist_bits [maxSimpleDistanceAlphabetSize]uint16 + histogramClearLiteral(&lit_histo) + histogramClearCommand(&cmd_histo) + histogramClearDistance(&dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) + buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */ + 8, lit_depth[:], lit_bits[:], storage_ix, storage) + + buildAndStoreHuffmanTreeFast(cmd_histo.data_[:], cmd_histo.total_count_, /* max_bits = */ + 10, cmd_depth[:], cmd_bits[:], storage_ix, storage) + + buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */ + uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage) + + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + } + + if is_last { + jumpToByteBoundary(storage_ix, storage) + } +} + +/* +This is for storing uncompressed blocks (simple raw storage of + + bytes-as-bytes). +*/ +func storeUncompressedMetaBlock(is_final_block bool, input []byte, position uint, mask uint, len uint, storage_ix *uint, storage []byte) { + var masked_pos uint = position & mask + storeUncompressedMetaBlockHeader(uint(len), storage_ix, storage) + jumpToByteBoundary(storage_ix, storage) + + if masked_pos+len > mask+1 { + var len1 uint = mask + 1 - masked_pos + copy(storage[*storage_ix>>3:], input[masked_pos:][:len1]) + *storage_ix += len1 << 3 + len -= len1 + masked_pos = 0 + } + + copy(storage[*storage_ix>>3:], input[masked_pos:][:len]) + *storage_ix += uint(len << 3) + + /* We need to clear the next 4 bytes to continue to be + compatible with BrotliWriteBits. */ + writeBitsPrepareStorage(*storage_ix, storage) + + /* Since the uncompressed block itself may not be the final block, add an + empty one after this. */ + if is_final_block { + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + jumpToByteBoundary(storage_ix, storage) + } +} diff --git a/vendor/github.com/andybalholm/brotli/cluster.go b/vendor/github.com/andybalholm/brotli/cluster.go new file mode 100644 index 0000000000..df8a328224 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/cluster.go @@ -0,0 +1,30 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Functions for clustering similar histograms together. */ + +type histogramPair struct { + idx1 uint32 + idx2 uint32 + cost_combo float64 + cost_diff float64 +} + +func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool { + if p1.cost_diff != p2.cost_diff { + return p1.cost_diff > p2.cost_diff + } + + return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1) +} + +/* Returns entropy reduction of the context map when we combine two clusters. */ +func clusterCostDiff(size_a uint, size_b uint) float64 { + var size_c uint = size_a + size_b + return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c) +} diff --git a/vendor/github.com/andybalholm/brotli/cluster_command.go b/vendor/github.com/andybalholm/brotli/cluster_command.go new file mode 100644 index 0000000000..45b569bb2a --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/cluster_command.go @@ -0,0 +1,164 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if + it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ +func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { + var is_good_pair bool = false + var p histogramPair + p.idx2 = 0 + p.idx1 = p.idx2 + p.cost_combo = 0 + p.cost_diff = p.cost_combo + if idx1 == idx2 { + return + } + + if idx2 < idx1 { + var t uint32 = idx2 + idx2 = idx1 + idx1 = t + } + + p.idx1 = idx1 + p.idx2 = idx2 + p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2])) + p.cost_diff -= out[idx1].bit_cost_ + p.cost_diff -= out[idx2].bit_cost_ + + if out[idx1].total_count_ == 0 { + p.cost_combo = out[idx2].bit_cost_ + is_good_pair = true + } else if out[idx2].total_count_ == 0 { + p.cost_combo = out[idx1].bit_cost_ + is_good_pair = true + } else { + var threshold float64 + if *num_pairs == 0 { + threshold = 1e99 + } else { + threshold = brotli_max_double(0.0, pairs[0].cost_diff) + } + var combo histogramCommand = out[idx1] + var cost_combo float64 + histogramAddHistogramCommand(&combo, &out[idx2]) + cost_combo = populationCostCommand(&combo) + if cost_combo < threshold-p.cost_diff { + p.cost_combo = cost_combo + is_good_pair = true + } + } + + if is_good_pair { + p.cost_diff += p.cost_combo + if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) { + /* Replace the top of the queue if needed. */ + if *num_pairs < max_num_pairs { + pairs[*num_pairs] = pairs[0] + (*num_pairs)++ + } + + pairs[0] = p + } else if *num_pairs < max_num_pairs { + pairs[*num_pairs] = p + (*num_pairs)++ + } + } +} + +func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint { + var cost_diff_threshold float64 = 0.0 + var min_cluster_size uint = 1 + var num_pairs uint = 0 + { + /* We maintain a vector of histogram pairs, with the property that the pair + with the maximum bit cost reduction is the first. */ + var idx1 uint + for idx1 = 0; idx1 < num_clusters; idx1++ { + var idx2 uint + for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ { + compareAndPushToQueueCommand(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs) + } + } + } + + for num_clusters > min_cluster_size { + var best_idx1 uint32 + var best_idx2 uint32 + var i uint + if pairs[0].cost_diff >= cost_diff_threshold { + cost_diff_threshold = 1e99 + min_cluster_size = max_clusters + continue + } + + /* Take the best pair from the top of heap. */ + best_idx1 = pairs[0].idx1 + + best_idx2 = pairs[0].idx2 + histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2]) + out[best_idx1].bit_cost_ = pairs[0].cost_combo + cluster_size[best_idx1] += cluster_size[best_idx2] + for i = 0; i < symbols_size; i++ { + if symbols[i] == best_idx2 { + symbols[i] = best_idx1 + } + } + + for i = 0; i < num_clusters; i++ { + if clusters[i] == best_idx2 { + copy(clusters[i:], clusters[i+1:][:num_clusters-i-1]) + break + } + } + + num_clusters-- + { + /* Remove pairs intersecting the just combined best pair. */ + var copy_to_idx uint = 0 + for i = 0; i < num_pairs; i++ { + var p *histogramPair = &pairs[i] + if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 { + /* Remove invalid pair from the queue. */ + continue + } + + if histogramPairIsLess(&pairs[0], p) { + /* Replace the top of the queue if needed. */ + var front histogramPair = pairs[0] + pairs[0] = *p + pairs[copy_to_idx] = front + } else { + pairs[copy_to_idx] = *p + } + + copy_to_idx++ + } + + num_pairs = copy_to_idx + } + + /* Push new pairs formed with the combined histogram to the heap. */ + for i = 0; i < num_clusters; i++ { + compareAndPushToQueueCommand(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs) + } + } + + return num_clusters +} + +/* What is the bit cost of moving histogram from cur_symbol to candidate. */ +func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 { + if histogram.total_count_ == 0 { + return 0.0 + } else { + var tmp histogramCommand = *histogram + histogramAddHistogramCommand(&tmp, candidate) + return populationCostCommand(&tmp) - candidate.bit_cost_ + } +} diff --git a/vendor/github.com/andybalholm/brotli/cluster_distance.go b/vendor/github.com/andybalholm/brotli/cluster_distance.go new file mode 100644 index 0000000000..1aaa86e6ed --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/cluster_distance.go @@ -0,0 +1,326 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if + it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ +func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { + var is_good_pair bool = false + var p histogramPair + p.idx2 = 0 + p.idx1 = p.idx2 + p.cost_combo = 0 + p.cost_diff = p.cost_combo + if idx1 == idx2 { + return + } + + if idx2 < idx1 { + var t uint32 = idx2 + idx2 = idx1 + idx1 = t + } + + p.idx1 = idx1 + p.idx2 = idx2 + p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2])) + p.cost_diff -= out[idx1].bit_cost_ + p.cost_diff -= out[idx2].bit_cost_ + + if out[idx1].total_count_ == 0 { + p.cost_combo = out[idx2].bit_cost_ + is_good_pair = true + } else if out[idx2].total_count_ == 0 { + p.cost_combo = out[idx1].bit_cost_ + is_good_pair = true + } else { + var threshold float64 + if *num_pairs == 0 { + threshold = 1e99 + } else { + threshold = brotli_max_double(0.0, pairs[0].cost_diff) + } + var combo histogramDistance = out[idx1] + var cost_combo float64 + histogramAddHistogramDistance(&combo, &out[idx2]) + cost_combo = populationCostDistance(&combo) + if cost_combo < threshold-p.cost_diff { + p.cost_combo = cost_combo + is_good_pair = true + } + } + + if is_good_pair { + p.cost_diff += p.cost_combo + if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) { + /* Replace the top of the queue if needed. */ + if *num_pairs < max_num_pairs { + pairs[*num_pairs] = pairs[0] + (*num_pairs)++ + } + + pairs[0] = p + } else if *num_pairs < max_num_pairs { + pairs[*num_pairs] = p + (*num_pairs)++ + } + } +} + +func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint { + var cost_diff_threshold float64 = 0.0 + var min_cluster_size uint = 1 + var num_pairs uint = 0 + { + /* We maintain a vector of histogram pairs, with the property that the pair + with the maximum bit cost reduction is the first. */ + var idx1 uint + for idx1 = 0; idx1 < num_clusters; idx1++ { + var idx2 uint + for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ { + compareAndPushToQueueDistance(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs) + } + } + } + + for num_clusters > min_cluster_size { + var best_idx1 uint32 + var best_idx2 uint32 + var i uint + if pairs[0].cost_diff >= cost_diff_threshold { + cost_diff_threshold = 1e99 + min_cluster_size = max_clusters + continue + } + + /* Take the best pair from the top of heap. */ + best_idx1 = pairs[0].idx1 + + best_idx2 = pairs[0].idx2 + histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2]) + out[best_idx1].bit_cost_ = pairs[0].cost_combo + cluster_size[best_idx1] += cluster_size[best_idx2] + for i = 0; i < symbols_size; i++ { + if symbols[i] == best_idx2 { + symbols[i] = best_idx1 + } + } + + for i = 0; i < num_clusters; i++ { + if clusters[i] == best_idx2 { + copy(clusters[i:], clusters[i+1:][:num_clusters-i-1]) + break + } + } + + num_clusters-- + { + /* Remove pairs intersecting the just combined best pair. */ + var copy_to_idx uint = 0 + for i = 0; i < num_pairs; i++ { + var p *histogramPair = &pairs[i] + if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 { + /* Remove invalid pair from the queue. */ + continue + } + + if histogramPairIsLess(&pairs[0], p) { + /* Replace the top of the queue if needed. */ + var front histogramPair = pairs[0] + pairs[0] = *p + pairs[copy_to_idx] = front + } else { + pairs[copy_to_idx] = *p + } + + copy_to_idx++ + } + + num_pairs = copy_to_idx + } + + /* Push new pairs formed with the combined histogram to the heap. */ + for i = 0; i < num_clusters; i++ { + compareAndPushToQueueDistance(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs) + } + } + + return num_clusters +} + +/* What is the bit cost of moving histogram from cur_symbol to candidate. */ +func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 { + if histogram.total_count_ == 0 { + return 0.0 + } else { + var tmp histogramDistance = *histogram + histogramAddHistogramDistance(&tmp, candidate) + return populationCostDistance(&tmp) - candidate.bit_cost_ + } +} + +/* Find the best 'out' histogram for each of the 'in' histograms. + When called, clusters[0..num_clusters) contains the unique values from + symbols[0..in_size), but this property is not preserved in this function. + Note: we assume that out[]->bit_cost_ is already up-to-date. */ +func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) { + var i uint + for i = 0; i < in_size; i++ { + var best_out uint32 + if i == 0 { + best_out = symbols[0] + } else { + best_out = symbols[i-1] + } + var best_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[best_out]) + var j uint + for j = 0; j < num_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + symbols[i] = best_out + } + + /* Recompute each out based on raw and symbols. */ + for i = 0; i < num_clusters; i++ { + histogramClearDistance(&out[clusters[i]]) + } + + for i = 0; i < in_size; i++ { + histogramAddHistogramDistance(&out[symbols[i]], &in[i]) + } +} + +/* Reorders elements of the out[0..length) array and changes values in + symbols[0..length) array in the following way: + * when called, symbols[] contains indexes into out[], and has N unique + values (possibly N < length) + * on return, symbols'[i] = f(symbols[i]) and + out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length, + where f is a bijection between the range of symbols[] and [0..N), and + the first occurrences of values in symbols'[i] come in consecutive + increasing order. + Returns N, the number of unique values in symbols[]. */ + +var histogramReindexDistance_kInvalidIndex uint32 = math.MaxUint32 + +func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint { + var new_index []uint32 = make([]uint32, length) + var next_index uint32 + var tmp []histogramDistance + var i uint + for i = 0; i < length; i++ { + new_index[i] = histogramReindexDistance_kInvalidIndex + } + + next_index = 0 + for i = 0; i < length; i++ { + if new_index[symbols[i]] == histogramReindexDistance_kInvalidIndex { + new_index[symbols[i]] = next_index + next_index++ + } + } + + /* TODO: by using idea of "cycle-sort" we can avoid allocation of + tmp and reduce the number of copying by the factor of 2. */ + tmp = make([]histogramDistance, next_index) + + next_index = 0 + for i = 0; i < length; i++ { + if new_index[symbols[i]] == next_index { + tmp[next_index] = out[symbols[i]] + next_index++ + } + + symbols[i] = new_index[symbols[i]] + } + + new_index = nil + for i = 0; uint32(i) < next_index; i++ { + out[i] = tmp[i] + } + + tmp = nil + return uint(next_index) +} + +func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) { + var cluster_size []uint32 = make([]uint32, in_size) + var clusters []uint32 = make([]uint32, in_size) + var num_clusters uint = 0 + var max_input_histograms uint = 64 + var pairs_capacity uint = max_input_histograms * max_input_histograms / 2 + var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1)) + var i uint + + /* For the first pass of clustering, we allow all pairs. */ + for i = 0; i < in_size; i++ { + cluster_size[i] = 1 + } + + for i = 0; i < in_size; i++ { + out[i] = in[i] + out[i].bit_cost_ = populationCostDistance(&in[i]) + histogram_symbols[i] = uint32(i) + } + + for i = 0; i < in_size; i += max_input_histograms { + var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + clusters[num_clusters+j] = uint32(i + j) + } + + num_new_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity) + num_clusters += num_new_clusters + } + { + /* For the second pass, we limit the total number of histogram pairs. + After this limit is reached, we only keep searching for the best pair. */ + var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < (max_num_pairs + 1) { + var _new_size uint + if pairs_capacity == 0 { + _new_size = max_num_pairs + 1 + } else { + _new_size = pairs_capacity + } + var new_array []histogramPair + for _new_size < (max_num_pairs + 1) { + _new_size *= 2 + } + new_array = make([]histogramPair, _new_size) + if pairs_capacity != 0 { + copy(new_array, pairs[:pairs_capacity]) + } + + pairs = new_array + pairs_capacity = _new_size + } + + /* Collapse similar histograms. */ + num_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs) + } + + pairs = nil + cluster_size = nil + + /* Find the optimal map from original histograms to the final ones. */ + histogramRemapDistance(in, in_size, clusters, num_clusters, out, histogram_symbols) + + clusters = nil + + /* Convert the context map to a canonical form. */ + *out_size = histogramReindexDistance(out, histogram_symbols, in_size) +} diff --git a/vendor/github.com/andybalholm/brotli/cluster_literal.go b/vendor/github.com/andybalholm/brotli/cluster_literal.go new file mode 100644 index 0000000000..6ba66f31b2 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/cluster_literal.go @@ -0,0 +1,326 @@ +package brotli + +import "math" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if + it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ +func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { + var is_good_pair bool = false + var p histogramPair + p.idx2 = 0 + p.idx1 = p.idx2 + p.cost_combo = 0 + p.cost_diff = p.cost_combo + if idx1 == idx2 { + return + } + + if idx2 < idx1 { + var t uint32 = idx2 + idx2 = idx1 + idx1 = t + } + + p.idx1 = idx1 + p.idx2 = idx2 + p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2])) + p.cost_diff -= out[idx1].bit_cost_ + p.cost_diff -= out[idx2].bit_cost_ + + if out[idx1].total_count_ == 0 { + p.cost_combo = out[idx2].bit_cost_ + is_good_pair = true + } else if out[idx2].total_count_ == 0 { + p.cost_combo = out[idx1].bit_cost_ + is_good_pair = true + } else { + var threshold float64 + if *num_pairs == 0 { + threshold = 1e99 + } else { + threshold = brotli_max_double(0.0, pairs[0].cost_diff) + } + var combo histogramLiteral = out[idx1] + var cost_combo float64 + histogramAddHistogramLiteral(&combo, &out[idx2]) + cost_combo = populationCostLiteral(&combo) + if cost_combo < threshold-p.cost_diff { + p.cost_combo = cost_combo + is_good_pair = true + } + } + + if is_good_pair { + p.cost_diff += p.cost_combo + if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) { + /* Replace the top of the queue if needed. */ + if *num_pairs < max_num_pairs { + pairs[*num_pairs] = pairs[0] + (*num_pairs)++ + } + + pairs[0] = p + } else if *num_pairs < max_num_pairs { + pairs[*num_pairs] = p + (*num_pairs)++ + } + } +} + +func histogramCombineLiteral(out []histogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint { + var cost_diff_threshold float64 = 0.0 + var min_cluster_size uint = 1 + var num_pairs uint = 0 + { + /* We maintain a vector of histogram pairs, with the property that the pair + with the maximum bit cost reduction is the first. */ + var idx1 uint + for idx1 = 0; idx1 < num_clusters; idx1++ { + var idx2 uint + for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ { + compareAndPushToQueueLiteral(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs) + } + } + } + + for num_clusters > min_cluster_size { + var best_idx1 uint32 + var best_idx2 uint32 + var i uint + if pairs[0].cost_diff >= cost_diff_threshold { + cost_diff_threshold = 1e99 + min_cluster_size = max_clusters + continue + } + + /* Take the best pair from the top of heap. */ + best_idx1 = pairs[0].idx1 + + best_idx2 = pairs[0].idx2 + histogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2]) + out[best_idx1].bit_cost_ = pairs[0].cost_combo + cluster_size[best_idx1] += cluster_size[best_idx2] + for i = 0; i < symbols_size; i++ { + if symbols[i] == best_idx2 { + symbols[i] = best_idx1 + } + } + + for i = 0; i < num_clusters; i++ { + if clusters[i] == best_idx2 { + copy(clusters[i:], clusters[i+1:][:num_clusters-i-1]) + break + } + } + + num_clusters-- + { + /* Remove pairs intersecting the just combined best pair. */ + var copy_to_idx uint = 0 + for i = 0; i < num_pairs; i++ { + var p *histogramPair = &pairs[i] + if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 { + /* Remove invalid pair from the queue. */ + continue + } + + if histogramPairIsLess(&pairs[0], p) { + /* Replace the top of the queue if needed. */ + var front histogramPair = pairs[0] + pairs[0] = *p + pairs[copy_to_idx] = front + } else { + pairs[copy_to_idx] = *p + } + + copy_to_idx++ + } + + num_pairs = copy_to_idx + } + + /* Push new pairs formed with the combined histogram to the heap. */ + for i = 0; i < num_clusters; i++ { + compareAndPushToQueueLiteral(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs) + } + } + + return num_clusters +} + +/* What is the bit cost of moving histogram from cur_symbol to candidate. */ +func histogramBitCostDistanceLiteral(histogram *histogramLiteral, candidate *histogramLiteral) float64 { + if histogram.total_count_ == 0 { + return 0.0 + } else { + var tmp histogramLiteral = *histogram + histogramAddHistogramLiteral(&tmp, candidate) + return populationCostLiteral(&tmp) - candidate.bit_cost_ + } +} + +/* Find the best 'out' histogram for each of the 'in' histograms. + When called, clusters[0..num_clusters) contains the unique values from + symbols[0..in_size), but this property is not preserved in this function. + Note: we assume that out[]->bit_cost_ is already up-to-date. */ +func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) { + var i uint + for i = 0; i < in_size; i++ { + var best_out uint32 + if i == 0 { + best_out = symbols[0] + } else { + best_out = symbols[i-1] + } + var best_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[best_out]) + var j uint + for j = 0; j < num_clusters; j++ { + var cur_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[clusters[j]]) + if cur_bits < best_bits { + best_bits = cur_bits + best_out = clusters[j] + } + } + + symbols[i] = best_out + } + + /* Recompute each out based on raw and symbols. */ + for i = 0; i < num_clusters; i++ { + histogramClearLiteral(&out[clusters[i]]) + } + + for i = 0; i < in_size; i++ { + histogramAddHistogramLiteral(&out[symbols[i]], &in[i]) + } +} + +/* Reorders elements of the out[0..length) array and changes values in + symbols[0..length) array in the following way: + * when called, symbols[] contains indexes into out[], and has N unique + values (possibly N < length) + * on return, symbols'[i] = f(symbols[i]) and + out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length, + where f is a bijection between the range of symbols[] and [0..N), and + the first occurrences of values in symbols'[i] come in consecutive + increasing order. + Returns N, the number of unique values in symbols[]. */ + +var histogramReindexLiteral_kInvalidIndex uint32 = math.MaxUint32 + +func histogramReindexLiteral(out []histogramLiteral, symbols []uint32, length uint) uint { + var new_index []uint32 = make([]uint32, length) + var next_index uint32 + var tmp []histogramLiteral + var i uint + for i = 0; i < length; i++ { + new_index[i] = histogramReindexLiteral_kInvalidIndex + } + + next_index = 0 + for i = 0; i < length; i++ { + if new_index[symbols[i]] == histogramReindexLiteral_kInvalidIndex { + new_index[symbols[i]] = next_index + next_index++ + } + } + + /* TODO: by using idea of "cycle-sort" we can avoid allocation of + tmp and reduce the number of copying by the factor of 2. */ + tmp = make([]histogramLiteral, next_index) + + next_index = 0 + for i = 0; i < length; i++ { + if new_index[symbols[i]] == next_index { + tmp[next_index] = out[symbols[i]] + next_index++ + } + + symbols[i] = new_index[symbols[i]] + } + + new_index = nil + for i = 0; uint32(i) < next_index; i++ { + out[i] = tmp[i] + } + + tmp = nil + return uint(next_index) +} + +func clusterHistogramsLiteral(in []histogramLiteral, in_size uint, max_histograms uint, out []histogramLiteral, out_size *uint, histogram_symbols []uint32) { + var cluster_size []uint32 = make([]uint32, in_size) + var clusters []uint32 = make([]uint32, in_size) + var num_clusters uint = 0 + var max_input_histograms uint = 64 + var pairs_capacity uint = max_input_histograms * max_input_histograms / 2 + var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1)) + var i uint + + /* For the first pass of clustering, we allow all pairs. */ + for i = 0; i < in_size; i++ { + cluster_size[i] = 1 + } + + for i = 0; i < in_size; i++ { + out[i] = in[i] + out[i].bit_cost_ = populationCostLiteral(&in[i]) + histogram_symbols[i] = uint32(i) + } + + for i = 0; i < in_size; i += max_input_histograms { + var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms) + var num_new_clusters uint + var j uint + for j = 0; j < num_to_combine; j++ { + clusters[num_clusters+j] = uint32(i + j) + } + + num_new_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity) + num_clusters += num_new_clusters + } + { + /* For the second pass, we limit the total number of histogram pairs. + After this limit is reached, we only keep searching for the best pair. */ + var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) + if pairs_capacity < (max_num_pairs + 1) { + var _new_size uint + if pairs_capacity == 0 { + _new_size = max_num_pairs + 1 + } else { + _new_size = pairs_capacity + } + var new_array []histogramPair + for _new_size < (max_num_pairs + 1) { + _new_size *= 2 + } + new_array = make([]histogramPair, _new_size) + if pairs_capacity != 0 { + copy(new_array, pairs[:pairs_capacity]) + } + + pairs = new_array + pairs_capacity = _new_size + } + + /* Collapse similar histograms. */ + num_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs) + } + + pairs = nil + cluster_size = nil + + /* Find the optimal map from original histograms to the final ones. */ + histogramRemapLiteral(in, in_size, clusters, num_clusters, out, histogram_symbols) + + clusters = nil + + /* Convert the context map to a canonical form. */ + *out_size = histogramReindexLiteral(out, histogram_symbols, in_size) +} diff --git a/vendor/github.com/andybalholm/brotli/command.go b/vendor/github.com/andybalholm/brotli/command.go new file mode 100644 index 0000000000..b1662a5555 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/command.go @@ -0,0 +1,254 @@ +package brotli + +var kInsBase = []uint32{ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 8, + 10, + 14, + 18, + 26, + 34, + 50, + 66, + 98, + 130, + 194, + 322, + 578, + 1090, + 2114, + 6210, + 22594, +} + +var kInsExtra = []uint32{ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 7, + 8, + 9, + 10, + 12, + 14, + 24, +} + +var kCopyBase = []uint32{ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 12, + 14, + 18, + 22, + 30, + 38, + 54, + 70, + 102, + 134, + 198, + 326, + 582, + 1094, + 2118, +} + +var kCopyExtra = []uint32{ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 7, + 8, + 9, + 10, + 24, +} + +func getInsertLengthCode(insertlen uint) uint16 { + if insertlen < 6 { + return uint16(insertlen) + } else if insertlen < 130 { + var nbits uint32 = log2FloorNonZero(insertlen-2) - 1 + return uint16((nbits << 1) + uint32((insertlen-2)>>nbits) + 2) + } else if insertlen < 2114 { + return uint16(log2FloorNonZero(insertlen-66) + 10) + } else if insertlen < 6210 { + return 21 + } else if insertlen < 22594 { + return 22 + } else { + return 23 + } +} + +func getCopyLengthCode(copylen uint) uint16 { + if copylen < 10 { + return uint16(copylen - 2) + } else if copylen < 134 { + var nbits uint32 = log2FloorNonZero(copylen-6) - 1 + return uint16((nbits << 1) + uint32((copylen-6)>>nbits) + 4) + } else if copylen < 2118 { + return uint16(log2FloorNonZero(copylen-70) + 12) + } else { + return 23 + } +} + +func combineLengthCodes(inscode uint16, copycode uint16, use_last_distance bool) uint16 { + var bits64 uint16 = uint16(copycode&0x7 | (inscode&0x7)<<3) + if use_last_distance && inscode < 8 && copycode < 16 { + if copycode < 8 { + return bits64 + } else { + return bits64 | 64 + } + } else { + /* Specification: 5 Encoding of ... (last table) */ + /* offset = 2 * index, where index is in range [0..8] */ + var offset uint32 = 2 * ((uint32(copycode) >> 3) + 3*(uint32(inscode)>>3)) + + /* All values in specification are K * 64, + where K = [2, 3, 6, 4, 5, 8, 7, 9, 10], + i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9], + K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D. + All values in D require only 2 bits to encode. + Magic constant is shifted 6 bits left, to avoid final multiplication. */ + offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0) + + return uint16(offset | uint32(bits64)) + } +} + +func getLengthCode(insertlen uint, copylen uint, use_last_distance bool, code *uint16) { + var inscode uint16 = getInsertLengthCode(insertlen) + var copycode uint16 = getCopyLengthCode(copylen) + *code = combineLengthCodes(inscode, copycode, use_last_distance) +} + +func getInsertBase(inscode uint16) uint32 { + return kInsBase[inscode] +} + +func getInsertExtra(inscode uint16) uint32 { + return kInsExtra[inscode] +} + +func getCopyBase(copycode uint16) uint32 { + return kCopyBase[copycode] +} + +func getCopyExtra(copycode uint16) uint32 { + return kCopyExtra[copycode] +} + +type command struct { + insert_len_ uint32 + copy_len_ uint32 + dist_extra_ uint32 + cmd_prefix_ uint16 + dist_prefix_ uint16 +} + +/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ +func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) { + /* Don't rely on signed int representation, use honest casts. */ + var delta uint32 = uint32(byte(int8(copylen_code_delta))) + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = uint32(uint32(copylen) | delta<<25) + + /* The distance prefix and extra bits are stored in this Command as if + npostfix and ndirect were 0, they are only recomputed later after the + clustering if needed. */ + prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) + getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_) + + return cmd +} + +func makeInsertCommand(insertlen uint) (cmd command) { + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = 4 << 25 + cmd.dist_extra_ = 0 + cmd.dist_prefix_ = numDistanceShortCodes + getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_) + return cmd +} + +func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 { + if uint32(self.dist_prefix_&0x3FF) < numDistanceShortCodes+dist.num_direct_distance_codes { + return uint32(self.dist_prefix_) & 0x3FF + } else { + var dcode uint32 = uint32(self.dist_prefix_) & 0x3FF + var nbits uint32 = uint32(self.dist_prefix_) >> 10 + var extra uint32 = self.dist_extra_ + var postfix_mask uint32 = (1 << dist.distance_postfix_bits) - 1 + var hcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) >> dist.distance_postfix_bits + var lcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) & postfix_mask + var offset uint32 = ((2 + (hcode & 1)) << nbits) - 4 + return ((offset + extra) << dist.distance_postfix_bits) + lcode + dist.num_direct_distance_codes + numDistanceShortCodes + } +} + +func commandDistanceContext(self *command) uint32 { + var r uint32 = uint32(self.cmd_prefix_) >> 6 + var c uint32 = uint32(self.cmd_prefix_) & 7 + if (r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2) { + return c + } + + return 3 +} + +func commandCopyLen(self *command) uint32 { + return self.copy_len_ & 0x1FFFFFF +} + +func commandCopyLenCode(self *command) uint32 { + var modifier uint32 = self.copy_len_ >> 25 + var delta int32 = int32(int8(byte(modifier | (modifier&0x40)<<1))) + return uint32(int32(self.copy_len_&0x1FFFFFF) + delta) +} diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment.go b/vendor/github.com/andybalholm/brotli/compress_fragment.go new file mode 100644 index 0000000000..c9bd057705 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/compress_fragment.go @@ -0,0 +1,834 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Function for fast encoding of an input fragment, independently from the input + history. This function uses one-pass processing: when we find a backward + match, we immediately emit the corresponding command and literal codes to + the bit stream. + + Adapted from the CompressFragment() function in + https://github.com/google/snappy/blob/master/snappy.cc */ + +const maxDistance_compress_fragment = 262128 + +func hash5(p []byte, shift uint) uint32 { + var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32) + return uint32(h >> shift) +} + +func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 { + assert(offset >= 0) + assert(offset <= 3) + { + var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32) + return uint32(h >> shift) + } +} + +func isMatch5(p1 []byte, p2 []byte) bool { + return binary.LittleEndian.Uint32(p1) == binary.LittleEndian.Uint32(p2) && + p1[4] == p2[4] +} + +/* Builds a literal prefix code into "depths" and "bits" based on the statistics + of the "input" string and stores it into the bit stream. + Note that the prefix code here is built from the pre-LZ77 input, therefore + we can only approximate the statistics of the actual literal stream. + Moreover, for long inputs we build a histogram from a sample of the input + and thus have to assign a non-zero depth for each literal. + Returns estimated compression ratio millibytes/char for encoding given input + with generated code. */ +func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint { + var histogram = [256]uint32{0} + var histogram_total uint + var i uint + if input_size < 1<<15 { + for i = 0; i < input_size; i++ { + histogram[input[i]]++ + } + + histogram_total = input_size + for i = 0; i < 256; i++ { + /* We weigh the first 11 samples with weight 3 to account for the + balancing effect of the LZ77 phase on the histogram. */ + var adjust uint32 = 2 * brotli_min_uint32_t(histogram[i], 11) + histogram[i] += adjust + histogram_total += uint(adjust) + } + } else { + const kSampleRate uint = 29 + for i = 0; i < input_size; i += kSampleRate { + histogram[input[i]]++ + } + + histogram_total = (input_size + kSampleRate - 1) / kSampleRate + for i = 0; i < 256; i++ { + /* We add 1 to each population count to avoid 0 bit depths (since this is + only a sample and we don't know if the symbol appears or not), and we + weigh the first 11 samples with weight 3 to account for the balancing + effect of the LZ77 phase on the histogram (more frequent symbols are + more likely to be in backward references instead as literals). */ + var adjust uint32 = 1 + 2*brotli_min_uint32_t(histogram[i], 11) + histogram[i] += adjust + histogram_total += uint(adjust) + } + } + + buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */ + 8, depths, bits, storage_ix, storage) + { + var literal_ratio uint = 0 + for i = 0; i < 256; i++ { + if histogram[i] != 0 { + literal_ratio += uint(histogram[i] * uint32(depths[i])) + } + } + + /* Estimated encoding ratio, millibytes per symbol. */ + return (literal_ratio * 125) / histogram_total + } +} + +/* Builds a command and distance prefix code (each 64 symbols) into "depth" and + "bits" based on "histogram" and stores it into the bit stream. */ +func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { + var tree [129]huffmanTree + var cmd_depth = [numCommandSymbols]byte{0} + /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ + + var cmd_bits [64]uint16 + + createHuffmanTree(histogram, 64, 15, tree[:], depth) + createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:]) + + /* We have to jump through a few hoops here in order to compute + the command bits because the symbols are in a different order than in + the full alphabet. This looks complicated, but having the symbols + in this order in the command bits saves a few branches in the Emit* + functions. */ + copy(cmd_depth[:], depth[:24]) + + copy(cmd_depth[24:][:], depth[40:][:8]) + copy(cmd_depth[32:][:], depth[24:][:8]) + copy(cmd_depth[40:][:], depth[48:][:8]) + copy(cmd_depth[48:][:], depth[32:][:8]) + copy(cmd_depth[56:][:], depth[56:][:8]) + convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:]) + copy(bits, cmd_bits[:24]) + copy(bits[24:], cmd_bits[32:][:8]) + copy(bits[32:], cmd_bits[48:][:8]) + copy(bits[40:], cmd_bits[24:][:8]) + copy(bits[48:], cmd_bits[40:][:8]) + copy(bits[56:], cmd_bits[56:][:8]) + convertBitDepthsToSymbols(depth[64:], 64, bits[64:]) + { + /* Create the bit length array for the full command alphabet. */ + var i uint + for i := 0; i < int(64); i++ { + cmd_depth[i] = 0 + } /* only 64 first values were used */ + copy(cmd_depth[:], depth[:8]) + copy(cmd_depth[64:][:], depth[8:][:8]) + copy(cmd_depth[128:][:], depth[16:][:8]) + copy(cmd_depth[192:][:], depth[24:][:8]) + copy(cmd_depth[384:][:], depth[32:][:8]) + for i = 0; i < 8; i++ { + cmd_depth[128+8*i] = depth[40+i] + cmd_depth[256+8*i] = depth[48+i] + cmd_depth[448+8*i] = depth[56+i] + } + + storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) + } + + storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) +} + +/* REQUIRES: insertlen < 6210 */ +func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { + if insertlen < 6 { + var code uint = insertlen + 40 + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + histo[code]++ + } else if insertlen < 130 { + var tail uint = insertlen - 2 + var nbits uint32 = log2FloorNonZero(tail) - 1 + var prefix uint = tail >> nbits + var inscode uint = uint((nbits << 1) + uint32(prefix) + 42) + writeBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage) + writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> nbits + var code uint = uint((nbits << 1) + uint32(prefix) + 20) + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> nbits + var code uint = uint((nbits << 1) + uint32(prefix) + 4) + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> 5) + 30 + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + writeBits(5, uint64(tail)&31, storage_ix, storage) + writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage) + histo[code]++ + histo[64]++ + } else if copylen < 2120 { + var tail uint = copylen - 72 + var nbits uint32 = log2FloorNonZero(tail) + var code uint = uint(nbits + 28) + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<> nbits) & 1 + var offset uint = (2 + prefix) << nbits + var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80) + writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage) + writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage) + histo[distcode]++ +} + +func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { + var j uint + for j = 0; j < len; j++ { + var lit byte = input[j] + writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage) + } +} + +/* REQUIRES: len <= 1 << 24. */ +func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { + var nibbles uint = 6 + + /* ISLAST */ + writeBits(1, 0, storage_ix, storage) + + if len <= 1<<16 { + nibbles = 4 + } else if len <= 1<<20 { + nibbles = 5 + } + + writeBits(2, uint64(nibbles)-4, storage_ix, storage) + writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) + + /* ISUNCOMPRESSED */ + writeSingleBit(is_uncompressed, storage_ix, storage) +} + +func updateBits(n_bits uint, bits uint32, pos uint, array []byte) { + for n_bits > 0 { + var byte_pos uint = pos >> 3 + var n_unchanged_bits uint = pos & 7 + var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits) + var total_bits uint = n_unchanged_bits + n_changed_bits + var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1) + var unchanged_bits uint32 = uint32(array[byte_pos]) & mask + var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1) + array[byte_pos] = byte(changed_bits<>= n_changed_bits + pos += n_changed_bits + } +} + +func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) { + var bitpos uint = new_storage_ix & 7 + var mask uint = (1 << bitpos) - 1 + storage[new_storage_ix>>3] &= byte(mask) + *storage_ix = new_storage_ix +} + +var shouldMergeBlock_kSampleRate uint = 43 + +func shouldMergeBlock(data []byte, len uint, depths []byte) bool { + var histo = [256]uint{0} + var i uint + for i = 0; i < len; i += shouldMergeBlock_kSampleRate { + histo[data[i]]++ + } + { + var total uint = (len + shouldMergeBlock_kSampleRate - 1) / shouldMergeBlock_kSampleRate + var r float64 = (fastLog2(total)+0.5)*float64(total) + 200 + for i = 0; i < 256; i++ { + r -= float64(histo[i]) * (float64(depths[i]) + fastLog2(histo[i])) + } + + return r >= 0.0 + } +} + +func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertlen uint, literal_ratio uint) bool { + var compressed uint = uint(-cap(next_emit) + cap(metablock_start)) + if compressed*50 > insertlen { + return false + } else { + return literal_ratio > 980 + } +} + +func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) { + var len uint = uint(-cap(end) + cap(begin)) + rewindBitPosition1(storage_ix_start, storage_ix, storage) + storeMetaBlockHeader1(uint(len), true, storage_ix, storage) + *storage_ix = (*storage_ix + 7) &^ 7 + copy(storage[*storage_ix>>3:], begin[:len]) + *storage_ix += uint(len << 3) + storage[*storage_ix>>3] = 0 +} + +var kCmdHistoSeed = [128]uint32{ + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, +} + +var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15 +var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16 + +func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { + var cmd_histo [128]uint32 + var ip_end int + var next_emit int = 0 + var base_ip int = 0 + var input int = 0 + const kInputMarginBytes uint = windowGap + const kMinMatchLen uint = 5 + var metablock_start int = input + var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize) + var total_block_size uint = block_size + var mlen_storage_ix uint = *storage_ix + 3 + var lit_depth [256]byte + var lit_bits [256]uint16 + var literal_ratio uint + var ip int + var last_distance int + var shift uint = 64 - table_bits + + /* "next_emit" is a pointer to the first byte that is not covered by a + previous copy. Bytes between "next_emit" and the start of the next copy or + the end of the input will be emitted as literal bytes. */ + + /* Save the start of the first block for position and distance computations. + */ + + /* Save the bit position of the MLEN field of the meta-block header, so that + we can update it later if we decide to extend this meta-block. */ + storeMetaBlockHeader1(block_size, false, storage_ix, storage) + + /* No block splits, no contexts. */ + writeBits(13, 0, storage_ix, storage) + + literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) + { + /* Store the pre-compressed command and distance prefix codes. */ + var i uint + for i = 0; i+7 < *cmd_code_numbits; i += 8 { + writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage) + } + } + + writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage) + + /* Initialize the command and distance histograms. We will gather + statistics of command and distance codes during the processing + of this block and use it to update the command and distance + prefix codes for the next block. */ +emit_commands: + copy(cmd_histo[:], kCmdHistoSeed[:]) + + /* "ip" is the input pointer. */ + ip = input + + last_distance = -1 + ip_end = int(uint(input) + block_size) + + if block_size >= kInputMarginBytes { + var len_limit uint = brotli_min_size_t(block_size-kMinMatchLen, input_size-kInputMarginBytes) + var ip_limit int = int(uint(input) + len_limit) + /* For the last block, we need to keep a 16 bytes margin so that we can be + sure that all distances are at most window size - 16. + For all other blocks, we only need to keep a margin of 5 bytes so that + we don't go over the block size with a copy. */ + + var next_hash uint32 + ip++ + for next_hash = hash5(in[ip:], shift); ; { + var skip uint32 = 32 + var next_ip int = ip + /* Step 1: Scan forward in the input looking for a 5-byte-long match. + If we get close to exhausting the input then goto emit_remainder. + + Heuristic match skipping: If 32 bytes are scanned with no matches + found, start looking only at every other byte. If 32 more bytes are + scanned, look at every third byte, etc.. When a match is found, + immediately go back to looking at every byte. This is a small loss + (~5% performance, ~0.1% density) for compressible data due to more + bookkeeping, but for non-compressible data (such as JPEG) it's a huge + win since the compressor quickly "realizes" the data is incompressible + and doesn't bother looking for matches everywhere. + + The "skip" variable keeps track of how many bytes there are since the + last match; dividing it by 32 (i.e. right-shifting by five) gives the + number of bytes to move ahead for each iteration. */ + + var candidate int + assert(next_emit < ip) + + trawl: + for { + var hash uint32 = next_hash + var bytes_between_hash_lookups uint32 = skip >> 5 + skip++ + assert(hash == hash5(in[next_ip:], shift)) + ip = next_ip + next_ip = int(uint32(ip) + bytes_between_hash_lookups) + if next_ip > ip_limit { + goto emit_remainder + } + + next_hash = hash5(in[next_ip:], shift) + candidate = ip - last_distance + if isMatch5(in[ip:], in[candidate:]) { + if candidate < ip { + table[hash] = int(ip - base_ip) + break + } + } + + candidate = base_ip + table[hash] + assert(candidate >= base_ip) + assert(candidate < ip) + + table[hash] = int(ip - base_ip) + if isMatch5(in[ip:], in[candidate:]) { + break + } + } + + /* Check copy distance. If candidate is not feasible, continue search. + Checking is done outside of hot loop to reduce overhead. */ + if ip-candidate > maxDistance_compress_fragment { + goto trawl + } + + /* Step 2: Emit the found match together with the literal bytes from + "next_emit" to the bit stream, and then see if we can find a next match + immediately afterwards. Repeat until we find no match for the input + without emitting some literal bytes. */ + { + var base int = ip + /* > 0 */ + var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5) + var distance int = int(base - candidate) + /* We have a 5-byte match at ip, and we need to emit bytes in + [next_emit, ip). */ + + var insert uint = uint(base - next_emit) + ip += int(matched) + if insert < 6210 { + emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + } else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { + emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage) + input_size -= uint(base - input) + input = base + next_emit = input + goto next_block + } else { + emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + } + + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) + if distance == last_distance { + writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage) + cmd_histo[64]++ + } else { + emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + last_distance = distance + } + + emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + + next_emit = ip + if ip >= ip_limit { + goto emit_remainder + } + + /* We could immediately start working at ip now, but to improve + compression we first update "table" with the hashes of some positions + within the last copy. */ + { + var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:]) + var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift) + var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift) + table[prev_hash] = int(ip - base_ip - 3) + prev_hash = hashBytesAtOffset5(input_bytes, 1, shift) + table[prev_hash] = int(ip - base_ip - 2) + prev_hash = hashBytesAtOffset5(input_bytes, 2, shift) + table[prev_hash] = int(ip - base_ip - 1) + + candidate = base_ip + table[cur_hash] + table[cur_hash] = int(ip - base_ip) + } + } + + for isMatch5(in[ip:], in[candidate:]) { + var base int = ip + /* We have a 5-byte match at ip, and no need to emit any literal bytes + prior to ip. */ + + var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5) + if ip-candidate > maxDistance_compress_fragment { + break + } + ip += int(matched) + last_distance = int(base - candidate) /* > 0 */ + emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + + next_emit = ip + if ip >= ip_limit { + goto emit_remainder + } + + /* We could immediately start working at ip now, but to improve + compression we first update "table" with the hashes of some positions + within the last copy. */ + { + var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:]) + var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift) + var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift) + table[prev_hash] = int(ip - base_ip - 3) + prev_hash = hashBytesAtOffset5(input_bytes, 1, shift) + table[prev_hash] = int(ip - base_ip - 2) + prev_hash = hashBytesAtOffset5(input_bytes, 2, shift) + table[prev_hash] = int(ip - base_ip - 1) + + candidate = base_ip + table[cur_hash] + table[cur_hash] = int(ip - base_ip) + } + } + + ip++ + next_hash = hash5(in[ip:], shift) + } + } + +emit_remainder: + assert(next_emit <= ip_end) + input += int(block_size) + input_size -= block_size + block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kMergeBlockSize) + + /* Decide if we want to continue this meta-block instead of emitting the + last insert-only command. */ + if input_size > 0 && total_block_size+block_size <= 1<<20 && shouldMergeBlock(in[input:], block_size, lit_depth[:]) { + assert(total_block_size > 1<<16) + + /* Update the size of the current meta-block and continue emitting commands. + We can do this because the current size and the new size both have 5 + nibbles. */ + total_block_size += block_size + + updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage) + goto emit_commands + } + + /* Emit the remaining bytes as literals. */ + if next_emit < ip_end { + var insert uint = uint(ip_end - next_emit) + if insert < 6210 { + emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) + } else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { + emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage) + } else { + emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) + } + } + + next_emit = ip_end + + /* If we have more data, write a new meta-block header and prefix codes and + then continue emitting commands. */ +next_block: + if input_size > 0 { + metablock_start = input + block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize) + total_block_size = block_size + + /* Save the bit position of the MLEN field of the meta-block header, so that + we can update it later if we decide to extend this meta-block. */ + mlen_storage_ix = *storage_ix + 3 + + storeMetaBlockHeader1(block_size, false, storage_ix, storage) + + /* No block splits, no contexts. */ + writeBits(13, 0, storage_ix, storage) + + literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) + buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage) + goto emit_commands + } + + if !is_last { + /* If this is not the last block, update the command and distance prefix + codes for the next block and store the compressed forms. */ + cmd_code[0] = 0 + + *cmd_code_numbits = 0 + buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code) + } +} + +/* Compresses "input" string to the "*storage" buffer as one or more complete + meta-blocks, and updates the "*storage_ix" bit position. + + If "is_last" is 1, emits an additional empty last meta-block. + + "cmd_depth" and "cmd_bits" contain the command and distance prefix codes + (see comment in encode.h) used for the encoding of this input fragment. + If "is_last" is 0, they are updated to reflect the statistics + of this input fragment, to be used for the encoding of the next fragment. + + "*cmd_code_numbits" is the number of bits of the compressed representation + of the command and distance prefix codes, and "cmd_code" is an array of + at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed + command and distance prefix codes. If "is_last" is 0, these are also + updated to represent the updated "cmd_depth" and "cmd_bits". + + REQUIRES: "input_size" is greater than zero, or "is_last" is 1. + REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). + REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. + REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ +func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { + var initial_storage_ix uint = *storage_ix + var table_bits uint = uint(log2FloorNonZero(table_size)) + + if input_size == 0 { + assert(is_last) + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + *storage_ix = (*storage_ix + 7) &^ 7 + return + } + + compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage) + + /* If output is larger than single uncompressed block, rewrite it. */ + if *storage_ix-initial_storage_ix > 31+(input_size<<3) { + emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage) + } + + if is_last { + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + *storage_ix = (*storage_ix + 7) &^ 7 + } +} diff --git a/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go new file mode 100644 index 0000000000..79f9c7fdfc --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/compress_fragment_two_pass.go @@ -0,0 +1,773 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Function for fast encoding of an input fragment, independently from the input + history. This function uses two-pass processing: in the first pass we save + the found backward matches and literal bytes into a buffer, and in the + second pass we emit them into the bit stream using prefix codes built based + on the actual command and literal byte histograms. */ + +const kCompressFragmentTwoPassBlockSize uint = 1 << 17 + +func hash1(p []byte, shift uint, length uint) uint32 { + var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32) + return uint32(h >> shift) +} + +func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 { + assert(offset <= 8-length) + { + var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32) + return uint32(h >> shift) + } +} + +func isMatch1(p1 []byte, p2 []byte, length uint) bool { + if binary.LittleEndian.Uint32(p1) != binary.LittleEndian.Uint32(p2) { + return false + } + if length == 4 { + return true + } + return p1[4] == p2[4] && p1[5] == p2[5] +} + +/* +Builds a command and distance prefix code (each 64 symbols) into "depth" and + + "bits" based on "histogram" and stores it into the bit stream. +*/ +func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { + var tree [129]huffmanTree + var cmd_depth = [numCommandSymbols]byte{0} + /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ + + var cmd_bits [64]uint16 + createHuffmanTree(histogram, 64, 15, tree[:], depth) + createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:]) + + /* We have to jump through a few hoops here in order to compute + the command bits because the symbols are in a different order than in + the full alphabet. This looks complicated, but having the symbols + in this order in the command bits saves a few branches in the Emit* + functions. */ + copy(cmd_depth[:], depth[24:][:24]) + + copy(cmd_depth[24:][:], depth[:8]) + copy(cmd_depth[32:][:], depth[48:][:8]) + copy(cmd_depth[40:][:], depth[8:][:8]) + copy(cmd_depth[48:][:], depth[56:][:8]) + copy(cmd_depth[56:][:], depth[16:][:8]) + convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:]) + copy(bits, cmd_bits[24:][:8]) + copy(bits[8:], cmd_bits[40:][:8]) + copy(bits[16:], cmd_bits[56:][:8]) + copy(bits[24:], cmd_bits[:24]) + copy(bits[48:], cmd_bits[32:][:8]) + copy(bits[56:], cmd_bits[48:][:8]) + convertBitDepthsToSymbols(depth[64:], 64, bits[64:]) + { + /* Create the bit length array for the full command alphabet. */ + var i uint + for i := 0; i < int(64); i++ { + cmd_depth[i] = 0 + } /* only 64 first values were used */ + copy(cmd_depth[:], depth[24:][:8]) + copy(cmd_depth[64:][:], depth[32:][:8]) + copy(cmd_depth[128:][:], depth[40:][:8]) + copy(cmd_depth[192:][:], depth[48:][:8]) + copy(cmd_depth[384:][:], depth[56:][:8]) + for i = 0; i < 8; i++ { + cmd_depth[128+8*i] = depth[i] + cmd_depth[256+8*i] = depth[8+i] + cmd_depth[448+8*i] = depth[16+i] + } + + storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) + } + + storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) +} + +func emitInsertLen(insertlen uint32, commands *[]uint32) { + if insertlen < 6 { + (*commands)[0] = insertlen + } else if insertlen < 130 { + var tail uint32 = insertlen - 2 + var nbits uint32 = log2FloorNonZero(uint(tail)) - 1 + var prefix uint32 = tail >> nbits + var inscode uint32 = (nbits << 1) + prefix + 2 + var extra uint32 = tail - (prefix << nbits) + (*commands)[0] = inscode | extra<<8 + } else if insertlen < 2114 { + var tail uint32 = insertlen - 66 + var nbits uint32 = log2FloorNonZero(uint(tail)) + var code uint32 = nbits + 10 + var extra uint32 = tail - (1 << nbits) + (*commands)[0] = code | extra<<8 + } else if insertlen < 6210 { + var extra uint32 = insertlen - 2114 + (*commands)[0] = 21 | extra<<8 + } else if insertlen < 22594 { + var extra uint32 = insertlen - 6210 + (*commands)[0] = 22 | extra<<8 + } else { + var extra uint32 = insertlen - 22594 + (*commands)[0] = 23 | extra<<8 + } + + *commands = (*commands)[1:] +} + +func emitCopyLen(copylen uint, commands *[]uint32) { + if copylen < 10 { + (*commands)[0] = uint32(copylen + 38) + } else if copylen < 134 { + var tail uint = copylen - 6 + var nbits uint = uint(log2FloorNonZero(tail) - 1) + var prefix uint = tail >> nbits + var code uint = (nbits << 1) + prefix + 44 + var extra uint = tail - (prefix << nbits) + (*commands)[0] = uint32(code | extra<<8) + } else if copylen < 2118 { + var tail uint = copylen - 70 + var nbits uint = uint(log2FloorNonZero(tail)) + var code uint = nbits + 52 + var extra uint = tail - (uint(1) << nbits) + (*commands)[0] = uint32(code | extra<<8) + } else { + var extra uint = copylen - 2118 + (*commands)[0] = uint32(63 | extra<<8) + } + + *commands = (*commands)[1:] +} + +func emitCopyLenLastDistance(copylen uint, commands *[]uint32) { + if copylen < 12 { + (*commands)[0] = uint32(copylen + 20) + *commands = (*commands)[1:] + } else if copylen < 72 { + var tail uint = copylen - 8 + var nbits uint = uint(log2FloorNonZero(tail) - 1) + var prefix uint = tail >> nbits + var code uint = (nbits << 1) + prefix + 28 + var extra uint = tail - (prefix << nbits) + (*commands)[0] = uint32(code | extra<<8) + *commands = (*commands)[1:] + } else if copylen < 136 { + var tail uint = copylen - 8 + var code uint = (tail >> 5) + 54 + var extra uint = tail & 31 + (*commands)[0] = uint32(code | extra<<8) + *commands = (*commands)[1:] + (*commands)[0] = 64 + *commands = (*commands)[1:] + } else if copylen < 2120 { + var tail uint = copylen - 72 + var nbits uint = uint(log2FloorNonZero(tail)) + var code uint = nbits + 52 + var extra uint = tail - (uint(1) << nbits) + (*commands)[0] = uint32(code | extra<<8) + *commands = (*commands)[1:] + (*commands)[0] = 64 + *commands = (*commands)[1:] + } else { + var extra uint = copylen - 2120 + (*commands)[0] = uint32(63 | extra<<8) + *commands = (*commands)[1:] + (*commands)[0] = 64 + *commands = (*commands)[1:] + } +} + +func emitDistance(distance uint32, commands *[]uint32) { + var d uint32 = distance + 3 + var nbits uint32 = log2FloorNonZero(uint(d)) - 1 + var prefix uint32 = (d >> nbits) & 1 + var offset uint32 = (2 + prefix) << nbits + var distcode uint32 = 2*(nbits-1) + prefix + 80 + var extra uint32 = d - offset + (*commands)[0] = distcode | extra<<8 + *commands = (*commands)[1:] +} + +/* REQUIRES: len <= 1 << 24. */ +func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { + var nibbles uint = 6 + + /* ISLAST */ + writeBits(1, 0, storage_ix, storage) + + if len <= 1<<16 { + nibbles = 4 + } else if len <= 1<<20 { + nibbles = 5 + } + + writeBits(2, uint64(nibbles)-4, storage_ix, storage) + writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) + + /* ISUNCOMPRESSED */ + writeSingleBit(is_uncompressed, storage_ix, storage) +} + +func storeMetaBlockHeaderBW(len uint, is_uncompressed bool, bw *bitWriter) { + var nibbles uint = 6 + + /* ISLAST */ + bw.writeBits(1, 0) + + if len <= 1<<16 { + nibbles = 4 + } else if len <= 1<<20 { + nibbles = 5 + } + + bw.writeBits(2, uint64(nibbles)-4) + bw.writeBits(nibbles*4, uint64(len)-1) + + /* ISUNCOMPRESSED */ + bw.writeSingleBit(is_uncompressed) +} + +func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) { + var ip int = 0 + var shift uint = 64 - table_bits + var ip_end int = int(block_size) + var base_ip int = -cap(base_ip_ptr) + cap(input) + var next_emit int = 0 + var last_distance int = -1 + /* "ip" is the input pointer. */ + + const kInputMarginBytes uint = windowGap + + /* "next_emit" is a pointer to the first byte that is not covered by a + previous copy. Bytes between "next_emit" and the start of the next copy or + the end of the input will be emitted as literal bytes. */ + if block_size >= kInputMarginBytes { + var len_limit uint = brotli_min_size_t(block_size-min_match, input_size-kInputMarginBytes) + var ip_limit int = int(len_limit) + /* For the last block, we need to keep a 16 bytes margin so that we can be + sure that all distances are at most window size - 16. + For all other blocks, we only need to keep a margin of 5 bytes so that + we don't go over the block size with a copy. */ + + var next_hash uint32 + ip++ + for next_hash = hash1(input[ip:], shift, min_match); ; { + var skip uint32 = 32 + var next_ip int = ip + /* Step 1: Scan forward in the input looking for a 6-byte-long match. + If we get close to exhausting the input then goto emit_remainder. + + Heuristic match skipping: If 32 bytes are scanned with no matches + found, start looking only at every other byte. If 32 more bytes are + scanned, look at every third byte, etc.. When a match is found, + immediately go back to looking at every byte. This is a small loss + (~5% performance, ~0.1% density) for compressible data due to more + bookkeeping, but for non-compressible data (such as JPEG) it's a huge + win since the compressor quickly "realizes" the data is incompressible + and doesn't bother looking for matches everywhere. + + The "skip" variable keeps track of how many bytes there are since the + last match; dividing it by 32 (ie. right-shifting by five) gives the + number of bytes to move ahead for each iteration. */ + + var candidate int + + assert(next_emit < ip) + + trawl: + for { + var hash uint32 = next_hash + var bytes_between_hash_lookups uint32 = skip >> 5 + skip++ + ip = next_ip + assert(hash == hash1(input[ip:], shift, min_match)) + next_ip = int(uint32(ip) + bytes_between_hash_lookups) + if next_ip > ip_limit { + goto emit_remainder + } + + next_hash = hash1(input[next_ip:], shift, min_match) + candidate = ip - last_distance + if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) { + if candidate < ip { + table[hash] = int(ip - base_ip) + break + } + } + + candidate = base_ip + table[hash] + assert(candidate >= base_ip) + assert(candidate < ip) + + table[hash] = int(ip - base_ip) + if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) { + break + } + } + + /* Check copy distance. If candidate is not feasible, continue search. + Checking is done outside of hot loop to reduce overhead. */ + if ip-candidate > maxDistance_compress_fragment { + goto trawl + } + + /* Step 2: Emit the found match together with the literal bytes from + "next_emit", and then see if we can find a next match immediately + afterwards. Repeat until we find no match for the input + without emitting some literal bytes. */ + { + var base int = ip + /* > 0 */ + var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match) + var distance int = int(base - candidate) + /* We have a 6-byte match at ip, and we need to emit bytes in + [next_emit, ip). */ + + var insert int = int(base - next_emit) + ip += int(matched) + emitInsertLen(uint32(insert), commands) + copy(*literals, input[next_emit:][:uint(insert)]) + *literals = (*literals)[insert:] + if distance == last_distance { + (*commands)[0] = 64 + *commands = (*commands)[1:] + } else { + emitDistance(uint32(distance), commands) + last_distance = distance + } + + emitCopyLenLastDistance(matched, commands) + + next_emit = ip + if ip >= ip_limit { + goto emit_remainder + } + { + var input_bytes uint64 + var cur_hash uint32 + /* We could immediately start working at ip now, but to improve + compression we first update "table" with the hashes of some + positions within the last copy. */ + + var prev_hash uint32 + if min_match == 4 { + input_bytes = binary.LittleEndian.Uint64(input[ip-3:]) + cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 3) + prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) + table[prev_hash] = int(ip - base_ip - 2) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 1) + } else { + input_bytes = binary.LittleEndian.Uint64(input[ip-5:]) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 5) + prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) + table[prev_hash] = int(ip - base_ip - 4) + prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) + table[prev_hash] = int(ip - base_ip - 3) + input_bytes = binary.LittleEndian.Uint64(input[ip-2:]) + cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 2) + prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) + table[prev_hash] = int(ip - base_ip - 1) + } + + candidate = base_ip + table[cur_hash] + table[cur_hash] = int(ip - base_ip) + } + } + + for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) { + var base int = ip + /* We have a 6-byte match at ip, and no need to emit any + literal bytes prior to ip. */ + + var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match) + ip += int(matched) + last_distance = int(base - candidate) /* > 0 */ + emitCopyLen(matched, commands) + emitDistance(uint32(last_distance), commands) + + next_emit = ip + if ip >= ip_limit { + goto emit_remainder + } + { + var input_bytes uint64 + var cur_hash uint32 + /* We could immediately start working at ip now, but to improve + compression we first update "table" with the hashes of some + positions within the last copy. */ + + var prev_hash uint32 + if min_match == 4 { + input_bytes = binary.LittleEndian.Uint64(input[ip-3:]) + cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 3) + prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) + table[prev_hash] = int(ip - base_ip - 2) + prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) + table[prev_hash] = int(ip - base_ip - 1) + } else { + input_bytes = binary.LittleEndian.Uint64(input[ip-5:]) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 5) + prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) + table[prev_hash] = int(ip - base_ip - 4) + prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) + table[prev_hash] = int(ip - base_ip - 3) + input_bytes = binary.LittleEndian.Uint64(input[ip-2:]) + cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) + prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) + table[prev_hash] = int(ip - base_ip - 2) + prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) + table[prev_hash] = int(ip - base_ip - 1) + } + + candidate = base_ip + table[cur_hash] + table[cur_hash] = int(ip - base_ip) + } + } + + ip++ + next_hash = hash1(input[ip:], shift, min_match) + } + } + +emit_remainder: + assert(next_emit <= ip_end) + + /* Emit the remaining bytes as literals. */ + if next_emit < ip_end { + var insert uint32 = uint32(ip_end - next_emit) + emitInsertLen(insert, commands) + copy(*literals, input[next_emit:][:insert]) + *literals = (*literals)[insert:] + } +} + +var storeCommands_kNumExtraBits = [128]uint32{ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 7, + 8, + 9, + 10, + 12, + 14, + 24, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 7, + 8, + 9, + 10, + 24, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 6, + 7, + 7, + 8, + 8, + 9, + 9, + 10, + 10, + 11, + 11, + 12, + 12, + 13, + 13, + 14, + 14, + 15, + 15, + 16, + 16, + 17, + 17, + 18, + 18, + 19, + 19, + 20, + 20, + 21, + 21, + 22, + 22, + 23, + 23, + 24, + 24, +} +var storeCommands_kInsertOffset = [24]uint32{ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 8, + 10, + 14, + 18, + 26, + 34, + 50, + 66, + 98, + 130, + 194, + 322, + 578, + 1090, + 2114, + 6210, + 22594, +} + +func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) { + var lit_depths [256]byte + var lit_bits [256]uint16 + var lit_histo = [256]uint32{0} + var cmd_depths = [128]byte{0} + var cmd_bits = [128]uint16{0} + var cmd_histo = [128]uint32{0} + var i uint + for i = 0; i < num_literals; i++ { + lit_histo[literals[i]]++ + } + + buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */ + 8, lit_depths[:], lit_bits[:], storage_ix, storage) + + for i = 0; i < num_commands; i++ { + var code uint32 = commands[i] & 0xFF + assert(code < 128) + cmd_histo[code]++ + } + + cmd_histo[1] += 1 + cmd_histo[2] += 1 + cmd_histo[64] += 1 + cmd_histo[84] += 1 + buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage) + + for i = 0; i < num_commands; i++ { + var cmd uint32 = commands[i] + var code uint32 = cmd & 0xFF + var extra uint32 = cmd >> 8 + assert(code < 128) + writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage) + writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage) + if code < 24 { + var insert uint32 = storeCommands_kInsertOffset[code] + extra + var j uint32 + for j = 0; j < insert; j++ { + var lit byte = literals[0] + writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage) + literals = literals[1:] + } + } + } +} + +/* Acceptable loss for uncompressible speedup is 2% */ +const minRatio = 0.98 + +const sampleRate = 43 + +func shouldCompress(input []byte, input_size uint, num_literals uint) bool { + var corpus_size float64 = float64(input_size) + if float64(num_literals) < minRatio*corpus_size { + return true + } else { + var literal_histo = [256]uint32{0} + var max_total_bit_cost float64 = corpus_size * 8 * minRatio / sampleRate + var i uint + for i = 0; i < input_size; i += sampleRate { + literal_histo[input[i]]++ + } + + return bitsEntropy(literal_histo[:], 256) < max_total_bit_cost + } +} + +func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) { + var bitpos uint = new_storage_ix & 7 + var mask uint = (1 << bitpos) - 1 + storage[new_storage_ix>>3] &= byte(mask) + *storage_ix = new_storage_ix +} + +func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) { + storeMetaBlockHeader(input_size, true, storage_ix, storage) + *storage_ix = (*storage_ix + 7) &^ 7 + copy(storage[*storage_ix>>3:], input[:input_size]) + *storage_ix += input_size << 3 + storage[*storage_ix>>3] = 0 +} + +func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) { + /* Save the start of the first block for position and distance computations. + */ + var base_ip []byte = input + + for input_size > 0 { + var block_size uint = brotli_min_size_t(input_size, kCompressFragmentTwoPassBlockSize) + var commands []uint32 = command_buf + var literals []byte = literal_buf + var num_literals uint + createCommands(input, block_size, input_size, base_ip, table, table_bits, min_match, &literals, &commands) + num_literals = uint(-cap(literals) + cap(literal_buf)) + if shouldCompress(input, block_size, num_literals) { + var num_commands uint = uint(-cap(commands) + cap(command_buf)) + storeMetaBlockHeader(block_size, false, storage_ix, storage) + + /* No block splits, no contexts. */ + writeBits(13, 0, storage_ix, storage) + + storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage) + } else { + /* Since we did not find many backward references and the entropy of + the data is close to 8 bits, we can simply emit an uncompressed block. + This makes compression speed of uncompressible data about 3x faster. */ + emitUncompressedMetaBlock(input, block_size, storage_ix, storage) + } + + input = input[block_size:] + input_size -= block_size + } +} + +/* +Compresses "input" string to the "*storage" buffer as one or more complete + + meta-blocks, and updates the "*storage_ix" bit position. + + If "is_last" is 1, emits an additional empty last meta-block. + + REQUIRES: "input_size" is greater than zero, or "is_last" is 1. + REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). + REQUIRES: "command_buf" and "literal_buf" point to at least + kCompressFragmentTwoPassBlockSize long arrays. + REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. + REQUIRES: "table_size" is a power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) +*/ +func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { + var initial_storage_ix uint = *storage_ix + var table_bits uint = uint(log2FloorNonZero(table_size)) + var min_match uint + if table_bits <= 15 { + min_match = 4 + } else { + min_match = 6 + } + compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage) + + /* If output is larger than single uncompressed block, rewrite it. */ + if *storage_ix-initial_storage_ix > 31+(input_size<<3) { + rewindBitPosition(initial_storage_ix, storage_ix, storage) + emitUncompressedMetaBlock(input, input_size, storage_ix, storage) + } + + if is_last { + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + *storage_ix = (*storage_ix + 7) &^ 7 + } +} diff --git a/vendor/github.com/andybalholm/brotli/constants.go b/vendor/github.com/andybalholm/brotli/constants.go new file mode 100644 index 0000000000..a880dff789 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/constants.go @@ -0,0 +1,77 @@ +package brotli + +/* Copyright 2016 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Specification: 7.3. Encoding of the context map */ +const contextMapMaxRle = 16 + +/* Specification: 2. Compressed representation overview */ +const maxNumberOfBlockTypes = 256 + +/* Specification: 3.3. Alphabet sizes: insert-and-copy length */ +const numLiteralSymbols = 256 + +const numCommandSymbols = 704 + +const numBlockLenSymbols = 26 + +const maxContextMapSymbols = (maxNumberOfBlockTypes + contextMapMaxRle) + +const maxBlockTypeSymbols = (maxNumberOfBlockTypes + 2) + +/* Specification: 3.5. Complex prefix codes */ +const repeatPreviousCodeLength = 16 + +const repeatZeroCodeLength = 17 + +const codeLengthCodes = (repeatZeroCodeLength + 1) + +/* "code length of 8 is repeated" */ +const initialRepeatedCodeLength = 8 + +/* "Large Window Brotli" */ +const largeMaxDistanceBits = 62 + +const largeMinWbits = 10 + +const largeMaxWbits = 30 + +/* Specification: 4. Encoding of distances */ +const numDistanceShortCodes = 16 + +const maxNpostfix = 3 + +const maxNdirect = 120 + +const maxDistanceBits = 24 + +func distanceAlphabetSize(NPOSTFIX uint, NDIRECT uint, MAXNBITS uint) uint { + return numDistanceShortCodes + NDIRECT + uint(MAXNBITS<<(NPOSTFIX+1)) +} + +/* numDistanceSymbols == 1128 */ +const numDistanceSymbols = 1128 + +const maxDistance = 0x3FFFFFC + +const maxAllowedDistance = 0x7FFFFFFC + +/* 7.1. Context modes and context ID lookup for literals */ +/* "context IDs for literals are in the range of 0..63" */ +const literalContextBits = 6 + +/* 7.2. Context ID for distances */ +const distanceContextBits = 2 + +/* 9.1. Format of the Stream Header */ +/* Number of slack bytes for window size. Don't confuse + with BROTLI_NUM_DISTANCE_SHORT_CODES. */ +const windowGap = 16 + +func maxBackwardLimit(W uint) uint { + return (uint(1) << W) - windowGap +} diff --git a/vendor/github.com/andybalholm/brotli/context.go b/vendor/github.com/andybalholm/brotli/context.go new file mode 100644 index 0000000000..884ff8a2d6 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/context.go @@ -0,0 +1,2176 @@ +package brotli + +/* Lookup table to map the previous two bytes to a context id. + +There are four different context modeling modes defined here: + contextLSB6: context id is the least significant 6 bits of the last byte, + contextMSB6: context id is the most significant 6 bits of the last byte, + contextUTF8: second-order context model tuned for UTF8-encoded text, + contextSigned: second-order context model tuned for signed integers. + +If |p1| and |p2| are the previous two bytes, and |mode| is current context +mode, we calculate the context as: + + context = ContextLut(mode)[p1] | ContextLut(mode)[p2 + 256]. + +For contextUTF8 mode, if the previous two bytes are ASCII characters +(i.e. < 128), this will be equivalent to + + context = 4 * context1(p1) + context2(p2), + +where context1 is based on the previous byte in the following way: + + 0 : non-ASCII control + 1 : \t, \n, \r + 2 : space + 3 : other punctuation + 4 : " ' + 5 : % + 6 : ( < [ { + 7 : ) > ] } + 8 : , ; : + 9 : . + 10 : = + 11 : number + 12 : upper-case vowel + 13 : upper-case consonant + 14 : lower-case vowel + 15 : lower-case consonant + +and context2 is based on the second last byte: + + 0 : control, space + 1 : punctuation + 2 : upper-case letter, number + 3 : lower-case letter + +If the last byte is ASCII, and the second last byte is not (in a valid UTF8 +stream it will be a continuation byte, value between 128 and 191), the +context is the same as if the second last byte was an ASCII control or space. + +If the last byte is a UTF8 lead byte (value >= 192), then the next byte will +be a continuation byte and the context id is 2 or 3 depending on the LSB of +the last byte and to a lesser extent on the second last byte if it is ASCII. + +If the last byte is a UTF8 continuation byte, the second last byte can be: + - continuation byte: the next byte is probably ASCII or lead byte (assuming + 4-byte UTF8 characters are rare) and the context id is 0 or 1. + - lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1 + - lead byte (208 - 255): next byte is continuation byte, context is 2 or 3 + +The possible value combinations of the previous two bytes, the range of +context ids and the type of the next byte is summarized in the table below: + +|--------\-----------------------------------------------------------------| +| \ Last byte | +| Second \---------------------------------------------------------------| +| last byte \ ASCII | cont. byte | lead byte | +| \ (0-127) | (128-191) | (192-) | +|=============|===================|=====================|==================| +| ASCII | next: ASCII/lead | not valid | next: cont. | +| (0-127) | context: 4 - 63 | | context: 2 - 3 | +|-------------|-------------------|---------------------|------------------| +| cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. | +| (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 | +|-------------|-------------------|---------------------|------------------| +| lead byte | not valid | next: ASCII/lead | not valid | +| (192-207) | | context: 0 - 1 | | +|-------------|-------------------|---------------------|------------------| +| lead byte | not valid | next: cont. | not valid | +| (208-) | | context: 2 - 3 | | +|-------------|-------------------|---------------------|------------------| +*/ + +const ( + contextLSB6 = 0 + contextMSB6 = 1 + contextUTF8 = 2 + contextSigned = 3 +) + +/* Common context lookup table for all context modes. */ +var kContextLookup = [2048]byte{ + /* CONTEXT_LSB6, last byte. */ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + + /* CONTEXT_LSB6, second last byte, */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + + /* CONTEXT_MSB6, last byte. */ + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 7, + 7, + 7, + 7, + 8, + 8, + 8, + 8, + 9, + 9, + 9, + 9, + 10, + 10, + 10, + 10, + 11, + 11, + 11, + 11, + 12, + 12, + 12, + 12, + 13, + 13, + 13, + 13, + 14, + 14, + 14, + 14, + 15, + 15, + 15, + 15, + 16, + 16, + 16, + 16, + 17, + 17, + 17, + 17, + 18, + 18, + 18, + 18, + 19, + 19, + 19, + 19, + 20, + 20, + 20, + 20, + 21, + 21, + 21, + 21, + 22, + 22, + 22, + 22, + 23, + 23, + 23, + 23, + 24, + 24, + 24, + 24, + 25, + 25, + 25, + 25, + 26, + 26, + 26, + 26, + 27, + 27, + 27, + 27, + 28, + 28, + 28, + 28, + 29, + 29, + 29, + 29, + 30, + 30, + 30, + 30, + 31, + 31, + 31, + 31, + 32, + 32, + 32, + 32, + 33, + 33, + 33, + 33, + 34, + 34, + 34, + 34, + 35, + 35, + 35, + 35, + 36, + 36, + 36, + 36, + 37, + 37, + 37, + 37, + 38, + 38, + 38, + 38, + 39, + 39, + 39, + 39, + 40, + 40, + 40, + 40, + 41, + 41, + 41, + 41, + 42, + 42, + 42, + 42, + 43, + 43, + 43, + 43, + 44, + 44, + 44, + 44, + 45, + 45, + 45, + 45, + 46, + 46, + 46, + 46, + 47, + 47, + 47, + 47, + 48, + 48, + 48, + 48, + 49, + 49, + 49, + 49, + 50, + 50, + 50, + 50, + 51, + 51, + 51, + 51, + 52, + 52, + 52, + 52, + 53, + 53, + 53, + 53, + 54, + 54, + 54, + 54, + 55, + 55, + 55, + 55, + 56, + 56, + 56, + 56, + 57, + 57, + 57, + 57, + 58, + 58, + 58, + 58, + 59, + 59, + 59, + 59, + 60, + 60, + 60, + 60, + 61, + 61, + 61, + 61, + 62, + 62, + 62, + 62, + 63, + 63, + 63, + 63, + + /* CONTEXT_MSB6, second last byte, */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + + /* CONTEXT_UTF8, last byte. */ + /* ASCII range. */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 4, + 0, + 0, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8, + 12, + 16, + 12, + 12, + 20, + 12, + 16, + 24, + 28, + 12, + 12, + 32, + 12, + 36, + 12, + 44, + 44, + 44, + 44, + 44, + 44, + 44, + 44, + 44, + 44, + 32, + 32, + 24, + 40, + 28, + 12, + 12, + 48, + 52, + 52, + 52, + 48, + 52, + 52, + 52, + 48, + 52, + 52, + 52, + 52, + 52, + 48, + 52, + 52, + 52, + 52, + 52, + 48, + 52, + 52, + 52, + 52, + 52, + 24, + 12, + 28, + 12, + 12, + 12, + 56, + 60, + 60, + 60, + 56, + 60, + 60, + 60, + 56, + 60, + 60, + 60, + 60, + 60, + 56, + 60, + 60, + 60, + 60, + 60, + 56, + 60, + 60, + 60, + 60, + 60, + 24, + 12, + 28, + 12, + 0, + + /* UTF8 continuation byte range. */ + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + 0, + 1, + + /* UTF8 lead byte range. */ + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 3, + + /* CONTEXT_UTF8 second last byte. */ + /* ASCII range. */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 1, + 1, + 1, + 1, + 0, + + /* UTF8 continuation byte range. */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + + /* UTF8 lead byte range. */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + + /* CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits. */ + 0, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 8, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 16, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 32, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 48, + 56, + + /* CONTEXT_SIGNED, second last byte. */ + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 7, +} + +type contextLUT []byte + +func getContextLUT(mode int) contextLUT { + return kContextLookup[mode<<9:] +} + +func getContext(p1 byte, p2 byte, lut contextLUT) byte { + return lut[p1] | lut[256+int(p2)] +} diff --git a/vendor/github.com/andybalholm/brotli/decode.go b/vendor/github.com/andybalholm/brotli/decode.go new file mode 100644 index 0000000000..9d9513b7cf --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/decode.go @@ -0,0 +1,2581 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +const ( + decoderResultError = 0 + decoderResultSuccess = 1 + decoderResultNeedsMoreInput = 2 + decoderResultNeedsMoreOutput = 3 +) + +/** + * Error code for detailed logging / production debugging. + * + * See ::BrotliDecoderGetErrorCode and ::BROTLI_LAST_ERROR_CODE. + */ +const ( + decoderNoError = 0 + decoderSuccess = 1 + decoderNeedsMoreInput = 2 + decoderNeedsMoreOutput = 3 + decoderErrorFormatExuberantNibble = -1 + decoderErrorFormatReserved = -2 + decoderErrorFormatExuberantMetaNibble = -3 + decoderErrorFormatSimpleHuffmanAlphabet = -4 + decoderErrorFormatSimpleHuffmanSame = -5 + decoderErrorFormatClSpace = -6 + decoderErrorFormatHuffmanSpace = -7 + decoderErrorFormatContextMapRepeat = -8 + decoderErrorFormatBlockLength1 = -9 + decoderErrorFormatBlockLength2 = -10 + decoderErrorFormatTransform = -11 + decoderErrorFormatDictionary = -12 + decoderErrorFormatWindowBits = -13 + decoderErrorFormatPadding1 = -14 + decoderErrorFormatPadding2 = -15 + decoderErrorFormatDistance = -16 + decoderErrorDictionaryNotSet = -19 + decoderErrorInvalidArguments = -20 + decoderErrorAllocContextModes = -21 + decoderErrorAllocTreeGroups = -22 + decoderErrorAllocContextMap = -25 + decoderErrorAllocRingBuffer1 = -26 + decoderErrorAllocRingBuffer2 = -27 + decoderErrorAllocBlockTypeTrees = -30 + decoderErrorUnreachable = -31 +) + +const huffmanTableBits = 8 + +const huffmanTableMask = 0xFF + +/* We need the slack region for the following reasons: + - doing up to two 16-byte copies for fast backward copying + - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */ +const kRingBufferWriteAheadSlack uint32 = 42 + +var kCodeLengthCodeOrder = [codeLengthCodes]byte{1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15} + +/* Static prefix code for the complex code length code lengths. */ +var kCodeLengthPrefixLength = [16]byte{2, 2, 2, 3, 2, 2, 2, 4, 2, 2, 2, 3, 2, 2, 2, 4} + +var kCodeLengthPrefixValue = [16]byte{0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5} + +/* Saves error code and converts it to BrotliDecoderResult. */ +func saveErrorCode(s *Reader, e int) int { + s.error_code = int(e) + switch e { + case decoderSuccess: + return decoderResultSuccess + + case decoderNeedsMoreInput: + return decoderResultNeedsMoreInput + + case decoderNeedsMoreOutput: + return decoderResultNeedsMoreOutput + + default: + return decoderResultError + } +} + +/* Decodes WBITS by reading 1 - 7 bits, or 0x11 for "Large Window Brotli". + Precondition: bit-reader accumulator has at least 8 bits. */ +func decodeWindowBits(s *Reader, br *bitReader) int { + var n uint32 + var large_window bool = s.large_window + s.large_window = false + takeBits(br, 1, &n) + if n == 0 { + s.window_bits = 16 + return decoderSuccess + } + + takeBits(br, 3, &n) + if n != 0 { + s.window_bits = 17 + n + return decoderSuccess + } + + takeBits(br, 3, &n) + if n == 1 { + if large_window { + takeBits(br, 1, &n) + if n == 1 { + return decoderErrorFormatWindowBits + } + + s.large_window = true + return decoderSuccess + } else { + return decoderErrorFormatWindowBits + } + } + + if n != 0 { + s.window_bits = 8 + n + return decoderSuccess + } + + s.window_bits = 17 + return decoderSuccess +} + +/* Decodes a number in the range [0..255], by reading 1 - 11 bits. */ +func decodeVarLenUint8(s *Reader, br *bitReader, value *uint32) int { + var bits uint32 + switch s.substate_decode_uint8 { + case stateDecodeUint8None: + if !safeReadBits(br, 1, &bits) { + return decoderNeedsMoreInput + } + + if bits == 0 { + *value = 0 + return decoderSuccess + } + fallthrough + + /* Fall through. */ + case stateDecodeUint8Short: + if !safeReadBits(br, 3, &bits) { + s.substate_decode_uint8 = stateDecodeUint8Short + return decoderNeedsMoreInput + } + + if bits == 0 { + *value = 1 + s.substate_decode_uint8 = stateDecodeUint8None + return decoderSuccess + } + + /* Use output value as a temporary storage. It MUST be persisted. */ + *value = bits + fallthrough + + /* Fall through. */ + case stateDecodeUint8Long: + if !safeReadBits(br, *value, &bits) { + s.substate_decode_uint8 = stateDecodeUint8Long + return decoderNeedsMoreInput + } + + *value = (1 << *value) + bits + s.substate_decode_uint8 = stateDecodeUint8None + return decoderSuccess + + default: + return decoderErrorUnreachable + } +} + +/* Decodes a metablock length and flags by reading 2 - 31 bits. */ +func decodeMetaBlockLength(s *Reader, br *bitReader) int { + var bits uint32 + var i int + for { + switch s.substate_metablock_header { + case stateMetablockHeaderNone: + if !safeReadBits(br, 1, &bits) { + return decoderNeedsMoreInput + } + + if bits != 0 { + s.is_last_metablock = 1 + } else { + s.is_last_metablock = 0 + } + s.meta_block_remaining_len = 0 + s.is_uncompressed = 0 + s.is_metadata = 0 + if s.is_last_metablock == 0 { + s.substate_metablock_header = stateMetablockHeaderNibbles + break + } + + s.substate_metablock_header = stateMetablockHeaderEmpty + fallthrough + + /* Fall through. */ + case stateMetablockHeaderEmpty: + if !safeReadBits(br, 1, &bits) { + return decoderNeedsMoreInput + } + + if bits != 0 { + s.substate_metablock_header = stateMetablockHeaderNone + return decoderSuccess + } + + s.substate_metablock_header = stateMetablockHeaderNibbles + fallthrough + + /* Fall through. */ + case stateMetablockHeaderNibbles: + if !safeReadBits(br, 2, &bits) { + return decoderNeedsMoreInput + } + + s.size_nibbles = uint(byte(bits + 4)) + s.loop_counter = 0 + if bits == 3 { + s.is_metadata = 1 + s.substate_metablock_header = stateMetablockHeaderReserved + break + } + + s.substate_metablock_header = stateMetablockHeaderSize + fallthrough + + /* Fall through. */ + case stateMetablockHeaderSize: + i = s.loop_counter + + for ; i < int(s.size_nibbles); i++ { + if !safeReadBits(br, 4, &bits) { + s.loop_counter = i + return decoderNeedsMoreInput + } + + if uint(i+1) == s.size_nibbles && s.size_nibbles > 4 && bits == 0 { + return decoderErrorFormatExuberantNibble + } + + s.meta_block_remaining_len |= int(bits << uint(i*4)) + } + + s.substate_metablock_header = stateMetablockHeaderUncompressed + fallthrough + + /* Fall through. */ + case stateMetablockHeaderUncompressed: + if s.is_last_metablock == 0 { + if !safeReadBits(br, 1, &bits) { + return decoderNeedsMoreInput + } + + if bits != 0 { + s.is_uncompressed = 1 + } else { + s.is_uncompressed = 0 + } + } + + s.meta_block_remaining_len++ + s.substate_metablock_header = stateMetablockHeaderNone + return decoderSuccess + + case stateMetablockHeaderReserved: + if !safeReadBits(br, 1, &bits) { + return decoderNeedsMoreInput + } + + if bits != 0 { + return decoderErrorFormatReserved + } + + s.substate_metablock_header = stateMetablockHeaderBytes + fallthrough + + /* Fall through. */ + case stateMetablockHeaderBytes: + if !safeReadBits(br, 2, &bits) { + return decoderNeedsMoreInput + } + + if bits == 0 { + s.substate_metablock_header = stateMetablockHeaderNone + return decoderSuccess + } + + s.size_nibbles = uint(byte(bits)) + s.substate_metablock_header = stateMetablockHeaderMetadata + fallthrough + + /* Fall through. */ + case stateMetablockHeaderMetadata: + i = s.loop_counter + + for ; i < int(s.size_nibbles); i++ { + if !safeReadBits(br, 8, &bits) { + s.loop_counter = i + return decoderNeedsMoreInput + } + + if uint(i+1) == s.size_nibbles && s.size_nibbles > 1 && bits == 0 { + return decoderErrorFormatExuberantMetaNibble + } + + s.meta_block_remaining_len |= int(bits << uint(i*8)) + } + + s.meta_block_remaining_len++ + s.substate_metablock_header = stateMetablockHeaderNone + return decoderSuccess + + default: + return decoderErrorUnreachable + } + } +} + +/* Decodes the Huffman code. + This method doesn't read data from the bit reader, BUT drops the amount of + bits that correspond to the decoded symbol. + bits MUST contain at least 15 (BROTLI_HUFFMAN_MAX_CODE_LENGTH) valid bits. */ +func decodeSymbol(bits uint32, table []huffmanCode, br *bitReader) uint32 { + table = table[bits&huffmanTableMask:] + if table[0].bits > huffmanTableBits { + var nbits uint32 = uint32(table[0].bits) - huffmanTableBits + dropBits(br, huffmanTableBits) + table = table[uint32(table[0].value)+((bits>>huffmanTableBits)&bitMask(nbits)):] + } + + dropBits(br, uint32(table[0].bits)) + return uint32(table[0].value) +} + +/* Reads and decodes the next Huffman code from bit-stream. + This method peeks 16 bits of input and drops 0 - 15 of them. */ +func readSymbol(table []huffmanCode, br *bitReader) uint32 { + return decodeSymbol(get16BitsUnmasked(br), table, br) +} + +/* Same as DecodeSymbol, but it is known that there is less than 15 bits of + input are currently available. */ +func safeDecodeSymbol(table []huffmanCode, br *bitReader, result *uint32) bool { + var val uint32 + var available_bits uint32 = getAvailableBits(br) + if available_bits == 0 { + if table[0].bits == 0 { + *result = uint32(table[0].value) + return true + } + + return false /* No valid bits at all. */ + } + + val = uint32(getBitsUnmasked(br)) + table = table[val&huffmanTableMask:] + if table[0].bits <= huffmanTableBits { + if uint32(table[0].bits) <= available_bits { + dropBits(br, uint32(table[0].bits)) + *result = uint32(table[0].value) + return true + } else { + return false /* Not enough bits for the first level. */ + } + } + + if available_bits <= huffmanTableBits { + return false /* Not enough bits to move to the second level. */ + } + + /* Speculatively drop HUFFMAN_TABLE_BITS. */ + val = (val & bitMask(uint32(table[0].bits))) >> huffmanTableBits + + available_bits -= huffmanTableBits + table = table[uint32(table[0].value)+val:] + if available_bits < uint32(table[0].bits) { + return false /* Not enough bits for the second level. */ + } + + dropBits(br, huffmanTableBits+uint32(table[0].bits)) + *result = uint32(table[0].value) + return true +} + +func safeReadSymbol(table []huffmanCode, br *bitReader, result *uint32) bool { + var val uint32 + if safeGetBits(br, 15, &val) { + *result = decodeSymbol(val, table, br) + return true + } + + return safeDecodeSymbol(table, br, result) +} + +/* Makes a look-up in first level Huffman table. Peeks 8 bits. */ +func preloadSymbol(safe int, table []huffmanCode, br *bitReader, bits *uint32, value *uint32) { + if safe != 0 { + return + } + + table = table[getBits(br, huffmanTableBits):] + *bits = uint32(table[0].bits) + *value = uint32(table[0].value) +} + +/* Decodes the next Huffman code using data prepared by PreloadSymbol. + Reads 0 - 15 bits. Also peeks 8 following bits. */ +func readPreloadedSymbol(table []huffmanCode, br *bitReader, bits *uint32, value *uint32) uint32 { + var result uint32 = *value + var ext []huffmanCode + if *bits > huffmanTableBits { + var val uint32 = get16BitsUnmasked(br) + ext = table[val&huffmanTableMask:][*value:] + var mask uint32 = bitMask((*bits - huffmanTableBits)) + dropBits(br, huffmanTableBits) + ext = ext[(val>>huffmanTableBits)&mask:] + dropBits(br, uint32(ext[0].bits)) + result = uint32(ext[0].value) + } else { + dropBits(br, *bits) + } + + preloadSymbol(0, table, br, bits, value) + return result +} + +func log2Floor(x uint32) uint32 { + var result uint32 = 0 + for x != 0 { + x >>= 1 + result++ + } + + return result +} + +/* Reads (s->symbol + 1) symbols. + Totally 1..4 symbols are read, 1..11 bits each. + The list of symbols MUST NOT contain duplicates. */ +func readSimpleHuffmanSymbols(alphabet_size uint32, max_symbol uint32, s *Reader) int { + var br *bitReader = &s.br + var max_bits uint32 = log2Floor(alphabet_size - 1) + var i uint32 = s.sub_loop_counter + /* max_bits == 1..11; symbol == 0..3; 1..44 bits will be read. */ + + var num_symbols uint32 = s.symbol + for i <= num_symbols { + var v uint32 + if !safeReadBits(br, max_bits, &v) { + s.sub_loop_counter = i + s.substate_huffman = stateHuffmanSimpleRead + return decoderNeedsMoreInput + } + + if v >= max_symbol { + return decoderErrorFormatSimpleHuffmanAlphabet + } + + s.symbols_lists_array[i] = uint16(v) + i++ + } + + for i = 0; i < num_symbols; i++ { + var k uint32 = i + 1 + for ; k <= num_symbols; k++ { + if s.symbols_lists_array[i] == s.symbols_lists_array[k] { + return decoderErrorFormatSimpleHuffmanSame + } + } + } + + return decoderSuccess +} + +/* Process single decoded symbol code length: + A) reset the repeat variable + B) remember code length (if it is not 0) + C) extend corresponding index-chain + D) reduce the Huffman space + E) update the histogram */ +func processSingleCodeLength(code_len uint32, symbol *uint32, repeat *uint32, space *uint32, prev_code_len *uint32, symbol_lists symbolList, code_length_histo []uint16, next_symbol []int) { + *repeat = 0 + if code_len != 0 { /* code_len == 1..15 */ + symbolListPut(symbol_lists, next_symbol[code_len], uint16(*symbol)) + next_symbol[code_len] = int(*symbol) + *prev_code_len = code_len + *space -= 32768 >> code_len + code_length_histo[code_len]++ + } + + (*symbol)++ +} + +/* Process repeated symbol code length. + A) Check if it is the extension of previous repeat sequence; if the decoded + value is not BROTLI_REPEAT_PREVIOUS_CODE_LENGTH, then it is a new + symbol-skip + B) Update repeat variable + C) Check if operation is feasible (fits alphabet) + D) For each symbol do the same operations as in ProcessSingleCodeLength + + PRECONDITION: code_len == BROTLI_REPEAT_PREVIOUS_CODE_LENGTH or + code_len == BROTLI_REPEAT_ZERO_CODE_LENGTH */ +func processRepeatedCodeLength(code_len uint32, repeat_delta uint32, alphabet_size uint32, symbol *uint32, repeat *uint32, space *uint32, prev_code_len *uint32, repeat_code_len *uint32, symbol_lists symbolList, code_length_histo []uint16, next_symbol []int) { + var old_repeat uint32 /* for BROTLI_REPEAT_ZERO_CODE_LENGTH */ /* for BROTLI_REPEAT_ZERO_CODE_LENGTH */ + var extra_bits uint32 = 3 + var new_len uint32 = 0 + if code_len == repeatPreviousCodeLength { + new_len = *prev_code_len + extra_bits = 2 + } + + if *repeat_code_len != new_len { + *repeat = 0 + *repeat_code_len = new_len + } + + old_repeat = *repeat + if *repeat > 0 { + *repeat -= 2 + *repeat <<= extra_bits + } + + *repeat += repeat_delta + 3 + repeat_delta = *repeat - old_repeat + if *symbol+repeat_delta > alphabet_size { + *symbol = alphabet_size + *space = 0xFFFFF + return + } + + if *repeat_code_len != 0 { + var last uint = uint(*symbol + repeat_delta) + var next int = next_symbol[*repeat_code_len] + for { + symbolListPut(symbol_lists, next, uint16(*symbol)) + next = int(*symbol) + (*symbol)++ + if (*symbol) == uint32(last) { + break + } + } + + next_symbol[*repeat_code_len] = next + *space -= repeat_delta << (15 - *repeat_code_len) + code_length_histo[*repeat_code_len] = uint16(uint32(code_length_histo[*repeat_code_len]) + repeat_delta) + } else { + *symbol += repeat_delta + } +} + +/* Reads and decodes symbol codelengths. */ +func readSymbolCodeLengths(alphabet_size uint32, s *Reader) int { + var br *bitReader = &s.br + var symbol uint32 = s.symbol + var repeat uint32 = s.repeat + var space uint32 = s.space + var prev_code_len uint32 = s.prev_code_len + var repeat_code_len uint32 = s.repeat_code_len + var symbol_lists symbolList = s.symbol_lists + var code_length_histo []uint16 = s.code_length_histo[:] + var next_symbol []int = s.next_symbol[:] + if !warmupBitReader(br) { + return decoderNeedsMoreInput + } + var p []huffmanCode + for symbol < alphabet_size && space > 0 { + p = s.table[:] + var code_len uint32 + if !checkInputAmount(br, shortFillBitWindowRead) { + s.symbol = symbol + s.repeat = repeat + s.prev_code_len = prev_code_len + s.repeat_code_len = repeat_code_len + s.space = space + return decoderNeedsMoreInput + } + + fillBitWindow16(br) + p = p[getBitsUnmasked(br)&uint64(bitMask(huffmanMaxCodeLengthCodeLength)):] + dropBits(br, uint32(p[0].bits)) /* Use 1..5 bits. */ + code_len = uint32(p[0].value) /* code_len == 0..17 */ + if code_len < repeatPreviousCodeLength { + processSingleCodeLength(code_len, &symbol, &repeat, &space, &prev_code_len, symbol_lists, code_length_histo, next_symbol) /* code_len == 16..17, extra_bits == 2..3 */ + } else { + var extra_bits uint32 + if code_len == repeatPreviousCodeLength { + extra_bits = 2 + } else { + extra_bits = 3 + } + var repeat_delta uint32 = uint32(getBitsUnmasked(br)) & bitMask(extra_bits) + dropBits(br, extra_bits) + processRepeatedCodeLength(code_len, repeat_delta, alphabet_size, &symbol, &repeat, &space, &prev_code_len, &repeat_code_len, symbol_lists, code_length_histo, next_symbol) + } + } + + s.space = space + return decoderSuccess +} + +func safeReadSymbolCodeLengths(alphabet_size uint32, s *Reader) int { + var br *bitReader = &s.br + var get_byte bool = false + var p []huffmanCode + for s.symbol < alphabet_size && s.space > 0 { + p = s.table[:] + var code_len uint32 + var available_bits uint32 + var bits uint32 = 0 + if get_byte && !pullByte(br) { + return decoderNeedsMoreInput + } + get_byte = false + available_bits = getAvailableBits(br) + if available_bits != 0 { + bits = uint32(getBitsUnmasked(br)) + } + + p = p[bits&bitMask(huffmanMaxCodeLengthCodeLength):] + if uint32(p[0].bits) > available_bits { + get_byte = true + continue + } + + code_len = uint32(p[0].value) /* code_len == 0..17 */ + if code_len < repeatPreviousCodeLength { + dropBits(br, uint32(p[0].bits)) + processSingleCodeLength(code_len, &s.symbol, &s.repeat, &s.space, &s.prev_code_len, s.symbol_lists, s.code_length_histo[:], s.next_symbol[:]) /* code_len == 16..17, extra_bits == 2..3 */ + } else { + var extra_bits uint32 = code_len - 14 + var repeat_delta uint32 = (bits >> p[0].bits) & bitMask(extra_bits) + if available_bits < uint32(p[0].bits)+extra_bits { + get_byte = true + continue + } + + dropBits(br, uint32(p[0].bits)+extra_bits) + processRepeatedCodeLength(code_len, repeat_delta, alphabet_size, &s.symbol, &s.repeat, &s.space, &s.prev_code_len, &s.repeat_code_len, s.symbol_lists, s.code_length_histo[:], s.next_symbol[:]) + } + } + + return decoderSuccess +} + +/* Reads and decodes 15..18 codes using static prefix code. + Each code is 2..4 bits long. In total 30..72 bits are used. */ +func readCodeLengthCodeLengths(s *Reader) int { + var br *bitReader = &s.br + var num_codes uint32 = s.repeat + var space uint32 = s.space + var i uint32 = s.sub_loop_counter + for ; i < codeLengthCodes; i++ { + var code_len_idx byte = kCodeLengthCodeOrder[i] + var ix uint32 + var v uint32 + if !safeGetBits(br, 4, &ix) { + var available_bits uint32 = getAvailableBits(br) + if available_bits != 0 { + ix = uint32(getBitsUnmasked(br) & 0xF) + } else { + ix = 0 + } + + if uint32(kCodeLengthPrefixLength[ix]) > available_bits { + s.sub_loop_counter = i + s.repeat = num_codes + s.space = space + s.substate_huffman = stateHuffmanComplex + return decoderNeedsMoreInput + } + } + + v = uint32(kCodeLengthPrefixValue[ix]) + dropBits(br, uint32(kCodeLengthPrefixLength[ix])) + s.code_length_code_lengths[code_len_idx] = byte(v) + if v != 0 { + space = space - (32 >> v) + num_codes++ + s.code_length_histo[v]++ + if space-1 >= 32 { + /* space is 0 or wrapped around. */ + break + } + } + } + + if num_codes != 1 && space != 0 { + return decoderErrorFormatClSpace + } + + return decoderSuccess +} + +/* Decodes the Huffman tables. + There are 2 scenarios: + A) Huffman code contains only few symbols (1..4). Those symbols are read + directly; their code lengths are defined by the number of symbols. + For this scenario 4 - 49 bits will be read. + + B) 2-phase decoding: + B.1) Small Huffman table is decoded; it is specified with code lengths + encoded with predefined entropy code. 32 - 74 bits are used. + B.2) Decoded table is used to decode code lengths of symbols in resulting + Huffman table. In worst case 3520 bits are read. */ +func readHuffmanCode(alphabet_size uint32, max_symbol uint32, table []huffmanCode, opt_table_size *uint32, s *Reader) int { + var br *bitReader = &s.br + + /* Unnecessary masking, but might be good for safety. */ + alphabet_size &= 0x7FF + + /* State machine. */ + for { + switch s.substate_huffman { + case stateHuffmanNone: + if !safeReadBits(br, 2, &s.sub_loop_counter) { + return decoderNeedsMoreInput + } + + /* The value is used as follows: + 1 for simple code; + 0 for no skipping, 2 skips 2 code lengths, 3 skips 3 code lengths */ + if s.sub_loop_counter != 1 { + s.space = 32 + s.repeat = 0 /* num_codes */ + var i int + for i = 0; i <= huffmanMaxCodeLengthCodeLength; i++ { + s.code_length_histo[i] = 0 + } + + for i = 0; i < codeLengthCodes; i++ { + s.code_length_code_lengths[i] = 0 + } + + s.substate_huffman = stateHuffmanComplex + continue + } + fallthrough + + /* Read symbols, codes & code lengths directly. */ + case stateHuffmanSimpleSize: + if !safeReadBits(br, 2, &s.symbol) { /* num_symbols */ + s.substate_huffman = stateHuffmanSimpleSize + return decoderNeedsMoreInput + } + + s.sub_loop_counter = 0 + fallthrough + + case stateHuffmanSimpleRead: + { + var result int = readSimpleHuffmanSymbols(alphabet_size, max_symbol, s) + if result != decoderSuccess { + return result + } + } + fallthrough + + case stateHuffmanSimpleBuild: + var table_size uint32 + if s.symbol == 3 { + var bits uint32 + if !safeReadBits(br, 1, &bits) { + s.substate_huffman = stateHuffmanSimpleBuild + return decoderNeedsMoreInput + } + + s.symbol += bits + } + + table_size = buildSimpleHuffmanTable(table, huffmanTableBits, s.symbols_lists_array[:], s.symbol) + if opt_table_size != nil { + *opt_table_size = table_size + } + + s.substate_huffman = stateHuffmanNone + return decoderSuccess + + /* Decode Huffman-coded code lengths. */ + case stateHuffmanComplex: + { + var i uint32 + var result int = readCodeLengthCodeLengths(s) + if result != decoderSuccess { + return result + } + + buildCodeLengthsHuffmanTable(s.table[:], s.code_length_code_lengths[:], s.code_length_histo[:]) + for i = 0; i < 16; i++ { + s.code_length_histo[i] = 0 + } + + for i = 0; i <= huffmanMaxCodeLength; i++ { + s.next_symbol[i] = int(i) - (huffmanMaxCodeLength + 1) + symbolListPut(s.symbol_lists, s.next_symbol[i], 0xFFFF) + } + + s.symbol = 0 + s.prev_code_len = initialRepeatedCodeLength + s.repeat = 0 + s.repeat_code_len = 0 + s.space = 32768 + s.substate_huffman = stateHuffmanLengthSymbols + } + fallthrough + + case stateHuffmanLengthSymbols: + var table_size uint32 + var result int = readSymbolCodeLengths(max_symbol, s) + if result == decoderNeedsMoreInput { + result = safeReadSymbolCodeLengths(max_symbol, s) + } + + if result != decoderSuccess { + return result + } + + if s.space != 0 { + return decoderErrorFormatHuffmanSpace + } + + table_size = buildHuffmanTable(table, huffmanTableBits, s.symbol_lists, s.code_length_histo[:]) + if opt_table_size != nil { + *opt_table_size = table_size + } + + s.substate_huffman = stateHuffmanNone + return decoderSuccess + + default: + return decoderErrorUnreachable + } + } +} + +/* Decodes a block length by reading 3..39 bits. */ +func readBlockLength(table []huffmanCode, br *bitReader) uint32 { + var code uint32 + var nbits uint32 + code = readSymbol(table, br) + nbits = kBlockLengthPrefixCode[code].nbits /* nbits == 2..24 */ + return kBlockLengthPrefixCode[code].offset + readBits(br, nbits) +} + +/* WARNING: if state is not BROTLI_STATE_READ_BLOCK_LENGTH_NONE, then + reading can't be continued with ReadBlockLength. */ +func safeReadBlockLength(s *Reader, result *uint32, table []huffmanCode, br *bitReader) bool { + var index uint32 + if s.substate_read_block_length == stateReadBlockLengthNone { + if !safeReadSymbol(table, br, &index) { + return false + } + } else { + index = s.block_length_index + } + { + var bits uint32 /* nbits == 2..24 */ + var nbits uint32 = kBlockLengthPrefixCode[index].nbits + if !safeReadBits(br, nbits, &bits) { + s.block_length_index = index + s.substate_read_block_length = stateReadBlockLengthSuffix + return false + } + + *result = kBlockLengthPrefixCode[index].offset + bits + s.substate_read_block_length = stateReadBlockLengthNone + return true + } +} + +/* Transform: + 1) initialize list L with values 0, 1,... 255 + 2) For each input element X: + 2.1) let Y = L[X] + 2.2) remove X-th element from L + 2.3) prepend Y to L + 2.4) append Y to output + + In most cases max(Y) <= 7, so most of L remains intact. + To reduce the cost of initialization, we reuse L, remember the upper bound + of Y values, and reinitialize only first elements in L. + + Most of input values are 0 and 1. To reduce number of branches, we replace + inner for loop with do-while. */ +func inverseMoveToFrontTransform(v []byte, v_len uint32, state *Reader) { + var mtf [256]byte + var i int + for i = 1; i < 256; i++ { + mtf[i] = byte(i) + } + var mtf_1 byte + + /* Transform the input. */ + for i = 0; uint32(i) < v_len; i++ { + var index int = int(v[i]) + var value byte = mtf[index] + v[i] = value + mtf_1 = value + for index >= 1 { + index-- + mtf[index+1] = mtf[index] + } + + mtf[0] = mtf_1 + } +} + +/* Decodes a series of Huffman table using ReadHuffmanCode function. */ +func huffmanTreeGroupDecode(group *huffmanTreeGroup, s *Reader) int { + if s.substate_tree_group != stateTreeGroupLoop { + s.next = group.codes + s.htree_index = 0 + s.substate_tree_group = stateTreeGroupLoop + } + + for s.htree_index < int(group.num_htrees) { + var table_size uint32 + var result int = readHuffmanCode(uint32(group.alphabet_size), uint32(group.max_symbol), s.next, &table_size, s) + if result != decoderSuccess { + return result + } + group.htrees[s.htree_index] = s.next + s.next = s.next[table_size:] + s.htree_index++ + } + + s.substate_tree_group = stateTreeGroupNone + return decoderSuccess +} + +/* Decodes a context map. + Decoding is done in 4 phases: + 1) Read auxiliary information (6..16 bits) and allocate memory. + In case of trivial context map, decoding is finished at this phase. + 2) Decode Huffman table using ReadHuffmanCode function. + This table will be used for reading context map items. + 3) Read context map items; "0" values could be run-length encoded. + 4) Optionally, apply InverseMoveToFront transform to the resulting map. */ +func decodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_arg *[]byte, s *Reader) int { + var br *bitReader = &s.br + var result int = decoderSuccess + + switch int(s.substate_context_map) { + case stateContextMapNone: + result = decodeVarLenUint8(s, br, num_htrees) + if result != decoderSuccess { + return result + } + + (*num_htrees)++ + s.context_index = 0 + *context_map_arg = make([]byte, uint(context_map_size)) + if *context_map_arg == nil { + return decoderErrorAllocContextMap + } + + if *num_htrees <= 1 { + for i := 0; i < int(context_map_size); i++ { + (*context_map_arg)[i] = 0 + } + return decoderSuccess + } + + s.substate_context_map = stateContextMapReadPrefix + fallthrough + /* Fall through. */ + case stateContextMapReadPrefix: + { + var bits uint32 + + /* In next stage ReadHuffmanCode uses at least 4 bits, so it is safe + to peek 4 bits ahead. */ + if !safeGetBits(br, 5, &bits) { + return decoderNeedsMoreInput + } + + if bits&1 != 0 { /* Use RLE for zeros. */ + s.max_run_length_prefix = (bits >> 1) + 1 + dropBits(br, 5) + } else { + s.max_run_length_prefix = 0 + dropBits(br, 1) + } + + s.substate_context_map = stateContextMapHuffman + } + fallthrough + + /* Fall through. */ + case stateContextMapHuffman: + { + var alphabet_size uint32 = *num_htrees + s.max_run_length_prefix + result = readHuffmanCode(alphabet_size, alphabet_size, s.context_map_table[:], nil, s) + if result != decoderSuccess { + return result + } + s.code = 0xFFFF + s.substate_context_map = stateContextMapDecode + } + fallthrough + + /* Fall through. */ + case stateContextMapDecode: + { + var context_index uint32 = s.context_index + var max_run_length_prefix uint32 = s.max_run_length_prefix + var context_map []byte = *context_map_arg + var code uint32 = s.code + var skip_preamble bool = (code != 0xFFFF) + for context_index < context_map_size || skip_preamble { + if !skip_preamble { + if !safeReadSymbol(s.context_map_table[:], br, &code) { + s.code = 0xFFFF + s.context_index = context_index + return decoderNeedsMoreInput + } + + if code == 0 { + context_map[context_index] = 0 + context_index++ + continue + } + + if code > max_run_length_prefix { + context_map[context_index] = byte(code - max_run_length_prefix) + context_index++ + continue + } + } else { + skip_preamble = false + } + + /* RLE sub-stage. */ + { + var reps uint32 + if !safeReadBits(br, code, &reps) { + s.code = code + s.context_index = context_index + return decoderNeedsMoreInput + } + + reps += 1 << code + if context_index+reps > context_map_size { + return decoderErrorFormatContextMapRepeat + } + + for { + context_map[context_index] = 0 + context_index++ + reps-- + if reps == 0 { + break + } + } + } + } + } + fallthrough + + case stateContextMapTransform: + var bits uint32 + if !safeReadBits(br, 1, &bits) { + s.substate_context_map = stateContextMapTransform + return decoderNeedsMoreInput + } + + if bits != 0 { + inverseMoveToFrontTransform(*context_map_arg, context_map_size, s) + } + + s.substate_context_map = stateContextMapNone + return decoderSuccess + + default: + return decoderErrorUnreachable + } +} + +/* Decodes a command or literal and updates block type ring-buffer. + Reads 3..54 bits. */ +func decodeBlockTypeAndLength(safe int, s *Reader, tree_type int) bool { + var max_block_type uint32 = s.num_block_types[tree_type] + type_tree := s.block_type_trees[tree_type*huffmanMaxSize258:] + len_tree := s.block_len_trees[tree_type*huffmanMaxSize26:] + var br *bitReader = &s.br + var ringbuffer []uint32 = s.block_type_rb[tree_type*2:] + var block_type uint32 + if max_block_type <= 1 { + return false + } + + /* Read 0..15 + 3..39 bits. */ + if safe == 0 { + block_type = readSymbol(type_tree, br) + s.block_length[tree_type] = readBlockLength(len_tree, br) + } else { + var memento bitReaderState + bitReaderSaveState(br, &memento) + if !safeReadSymbol(type_tree, br, &block_type) { + return false + } + if !safeReadBlockLength(s, &s.block_length[tree_type], len_tree, br) { + s.substate_read_block_length = stateReadBlockLengthNone + bitReaderRestoreState(br, &memento) + return false + } + } + + if block_type == 1 { + block_type = ringbuffer[1] + 1 + } else if block_type == 0 { + block_type = ringbuffer[0] + } else { + block_type -= 2 + } + + if block_type >= max_block_type { + block_type -= max_block_type + } + + ringbuffer[0] = ringbuffer[1] + ringbuffer[1] = block_type + return true +} + +func detectTrivialLiteralBlockTypes(s *Reader) { + var i uint + for i = 0; i < 8; i++ { + s.trivial_literal_contexts[i] = 0 + } + for i = 0; uint32(i) < s.num_block_types[0]; i++ { + var offset uint = i << literalContextBits + var error uint = 0 + var sample uint = uint(s.context_map[offset]) + var j uint + for j = 0; j < 1<>5] |= 1 << (i & 31) + } + } +} + +func prepareLiteralDecoding(s *Reader) { + var context_mode byte + var trivial uint + var block_type uint32 = s.block_type_rb[1] + var context_offset uint32 = block_type << literalContextBits + s.context_map_slice = s.context_map[context_offset:] + trivial = uint(s.trivial_literal_contexts[block_type>>5]) + s.trivial_literal_context = int((trivial >> (block_type & 31)) & 1) + s.literal_htree = []huffmanCode(s.literal_hgroup.htrees[s.context_map_slice[0]]) + context_mode = s.context_modes[block_type] & 3 + s.context_lookup = getContextLUT(int(context_mode)) +} + +/* Decodes the block type and updates the state for literal context. + Reads 3..54 bits. */ +func decodeLiteralBlockSwitchInternal(safe int, s *Reader) bool { + if !decodeBlockTypeAndLength(safe, s, 0) { + return false + } + + prepareLiteralDecoding(s) + return true +} + +func decodeLiteralBlockSwitch(s *Reader) { + decodeLiteralBlockSwitchInternal(0, s) +} + +func safeDecodeLiteralBlockSwitch(s *Reader) bool { + return decodeLiteralBlockSwitchInternal(1, s) +} + +/* Block switch for insert/copy length. + Reads 3..54 bits. */ +func decodeCommandBlockSwitchInternal(safe int, s *Reader) bool { + if !decodeBlockTypeAndLength(safe, s, 1) { + return false + } + + s.htree_command = []huffmanCode(s.insert_copy_hgroup.htrees[s.block_type_rb[3]]) + return true +} + +func decodeCommandBlockSwitch(s *Reader) { + decodeCommandBlockSwitchInternal(0, s) +} + +func safeDecodeCommandBlockSwitch(s *Reader) bool { + return decodeCommandBlockSwitchInternal(1, s) +} + +/* Block switch for distance codes. + Reads 3..54 bits. */ +func decodeDistanceBlockSwitchInternal(safe int, s *Reader) bool { + if !decodeBlockTypeAndLength(safe, s, 2) { + return false + } + + s.dist_context_map_slice = s.dist_context_map[s.block_type_rb[5]< s.ringbuffer_size { + pos = uint(s.ringbuffer_size) + } else { + pos = uint(s.pos) + } + var partial_pos_rb uint = (s.rb_roundtrips * uint(s.ringbuffer_size)) + pos + return partial_pos_rb - s.partial_pos_out +} + +/* Dumps output. + Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push + and either ring-buffer is as big as window size, or |force| is true. */ +func writeRingBuffer(s *Reader, available_out *uint, next_out *[]byte, total_out *uint, force bool) int { + start := s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):] + var to_write uint = unwrittenBytes(s, true) + var num_written uint = *available_out + if num_written > to_write { + num_written = to_write + } + + if s.meta_block_remaining_len < 0 { + return decoderErrorFormatBlockLength1 + } + + if next_out != nil && *next_out == nil { + *next_out = start + } else { + if next_out != nil { + copy(*next_out, start[:num_written]) + *next_out = (*next_out)[num_written:] + } + } + + *available_out -= num_written + s.partial_pos_out += num_written + if total_out != nil { + *total_out = s.partial_pos_out + } + + if num_written < to_write { + if s.ringbuffer_size == 1<= s.ringbuffer_size { + s.pos -= s.ringbuffer_size + s.rb_roundtrips++ + if uint(s.pos) != 0 { + s.should_wrap_ringbuffer = 1 + } else { + s.should_wrap_ringbuffer = 0 + } + } + + return decoderSuccess +} + +func wrapRingBuffer(s *Reader) { + if s.should_wrap_ringbuffer != 0 { + copy(s.ringbuffer, s.ringbuffer_end[:uint(s.pos)]) + s.should_wrap_ringbuffer = 0 + } +} + +/* Allocates ring-buffer. + + s->ringbuffer_size MUST be updated by BrotliCalculateRingBufferSize before + this function is called. + + Last two bytes of ring-buffer are initialized to 0, so context calculation + could be done uniformly for the first two and all other positions. */ +func ensureRingBuffer(s *Reader) bool { + var old_ringbuffer []byte + if s.ringbuffer_size == s.new_ringbuffer_size { + return true + } + spaceNeeded := int(s.new_ringbuffer_size) + int(kRingBufferWriteAheadSlack) + if len(s.ringbuffer) < spaceNeeded { + old_ringbuffer = s.ringbuffer + s.ringbuffer = make([]byte, spaceNeeded) + } + + s.ringbuffer[s.new_ringbuffer_size-2] = 0 + s.ringbuffer[s.new_ringbuffer_size-1] = 0 + + if old_ringbuffer != nil { + copy(s.ringbuffer, old_ringbuffer[:uint(s.pos)]) + } + + s.ringbuffer_size = s.new_ringbuffer_size + s.ringbuffer_mask = s.new_ringbuffer_size - 1 + s.ringbuffer_end = s.ringbuffer[s.ringbuffer_size:] + + return true +} + +func copyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_out *uint, s *Reader) int { + /* TODO: avoid allocation for single uncompressed block. */ + if !ensureRingBuffer(s) { + return decoderErrorAllocRingBuffer1 + } + + /* State machine */ + for { + switch s.substate_uncompressed { + case stateUncompressedNone: + { + var nbytes int = int(getRemainingBytes(&s.br)) + if nbytes > s.meta_block_remaining_len { + nbytes = s.meta_block_remaining_len + } + + if s.pos+nbytes > s.ringbuffer_size { + nbytes = s.ringbuffer_size - s.pos + } + + /* Copy remaining bytes from s->br.buf_ to ring-buffer. */ + copyBytes(s.ringbuffer[s.pos:], &s.br, uint(nbytes)) + + s.pos += nbytes + s.meta_block_remaining_len -= nbytes + if s.pos < 1<>1 >= min_size { + new_ringbuffer_size >>= 1 + } + } + + s.new_ringbuffer_size = new_ringbuffer_size +} + +/* Reads 1..256 2-bit context modes. */ +func readContextModes(s *Reader) int { + var br *bitReader = &s.br + var i int = s.loop_counter + + for i < int(s.num_block_types[0]) { + var bits uint32 + if !safeReadBits(br, 2, &bits) { + s.loop_counter = i + return decoderNeedsMoreInput + } + + s.context_modes[i] = byte(bits) + i++ + } + + return decoderSuccess +} + +func takeDistanceFromRingBuffer(s *Reader) { + if s.distance_code == 0 { + s.dist_rb_idx-- + s.distance_code = s.dist_rb[s.dist_rb_idx&3] + + /* Compensate double distance-ring-buffer roll for dictionary items. */ + s.distance_context = 1 + } else { + var distance_code int = s.distance_code << 1 + const kDistanceShortCodeIndexOffset uint32 = 0xAAAFFF1B + const kDistanceShortCodeValueOffset uint32 = 0xFA5FA500 + var v int = (s.dist_rb_idx + int(kDistanceShortCodeIndexOffset>>uint(distance_code))) & 0x3 + /* kDistanceShortCodeIndexOffset has 2-bit values from LSB: + 3, 2, 1, 0, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2 */ + + /* kDistanceShortCodeValueOffset has 2-bit values from LSB: + -0, 0,-0, 0,-1, 1,-2, 2,-3, 3,-1, 1,-2, 2,-3, 3 */ + s.distance_code = s.dist_rb[v] + + v = int(kDistanceShortCodeValueOffset>>uint(distance_code)) & 0x3 + if distance_code&0x3 != 0 { + s.distance_code += v + } else { + s.distance_code -= v + if s.distance_code <= 0 { + /* A huge distance will cause a () soon. + This is a little faster than failing here. */ + s.distance_code = 0x7FFFFFFF + } + } + } +} + +func safeReadBitsMaybeZero(br *bitReader, n_bits uint32, val *uint32) bool { + if n_bits != 0 { + return safeReadBits(br, n_bits, val) + } else { + *val = 0 + return true + } +} + +/* Precondition: s->distance_code < 0. */ +func readDistanceInternal(safe int, s *Reader, br *bitReader) bool { + var distval int + var memento bitReaderState + var distance_tree []huffmanCode = []huffmanCode(s.distance_hgroup.htrees[s.dist_htree_index]) + if safe == 0 { + s.distance_code = int(readSymbol(distance_tree, br)) + } else { + var code uint32 + bitReaderSaveState(br, &memento) + if !safeReadSymbol(distance_tree, br, &code) { + return false + } + + s.distance_code = int(code) + } + + /* Convert the distance code to the actual distance by possibly + looking up past distances from the s->ringbuffer. */ + s.distance_context = 0 + + if s.distance_code&^0xF == 0 { + takeDistanceFromRingBuffer(s) + s.block_length[2]-- + return true + } + + distval = s.distance_code - int(s.num_direct_distance_codes) + if distval >= 0 { + var nbits uint32 + var postfix int + var offset int + if safe == 0 && (s.distance_postfix_bits == 0) { + nbits = (uint32(distval) >> 1) + 1 + offset = ((2 + (distval & 1)) << nbits) - 4 + s.distance_code = int(s.num_direct_distance_codes) + offset + int(readBits(br, nbits)) + } else { + /* This branch also works well when s->distance_postfix_bits == 0. */ + var bits uint32 + postfix = distval & s.distance_postfix_mask + distval >>= s.distance_postfix_bits + nbits = (uint32(distval) >> 1) + 1 + if safe != 0 { + if !safeReadBitsMaybeZero(br, nbits, &bits) { + s.distance_code = -1 /* Restore precondition. */ + bitReaderRestoreState(br, &memento) + return false + } + } else { + bits = readBits(br, nbits) + } + + offset = ((2 + (distval & 1)) << nbits) - 4 + s.distance_code = int(s.num_direct_distance_codes) + ((offset + int(bits)) << s.distance_postfix_bits) + postfix + } + } + + s.distance_code = s.distance_code - numDistanceShortCodes + 1 + s.block_length[2]-- + return true +} + +func readDistance(s *Reader, br *bitReader) { + readDistanceInternal(0, s, br) +} + +func safeReadDistance(s *Reader, br *bitReader) bool { + return readDistanceInternal(1, s, br) +} + +func readCommandInternal(safe int, s *Reader, br *bitReader, insert_length *int) bool { + var cmd_code uint32 + var insert_len_extra uint32 = 0 + var copy_length uint32 + var v cmdLutElement + var memento bitReaderState + if safe == 0 { + cmd_code = readSymbol(s.htree_command, br) + } else { + bitReaderSaveState(br, &memento) + if !safeReadSymbol(s.htree_command, br, &cmd_code) { + return false + } + } + + v = kCmdLut[cmd_code] + s.distance_code = int(v.distance_code) + s.distance_context = int(v.context) + s.dist_htree_index = s.dist_context_map_slice[s.distance_context] + *insert_length = int(v.insert_len_offset) + if safe == 0 { + if v.insert_len_extra_bits != 0 { + insert_len_extra = readBits(br, uint32(v.insert_len_extra_bits)) + } + + copy_length = readBits(br, uint32(v.copy_len_extra_bits)) + } else { + if !safeReadBitsMaybeZero(br, uint32(v.insert_len_extra_bits), &insert_len_extra) || !safeReadBitsMaybeZero(br, uint32(v.copy_len_extra_bits), ©_length) { + bitReaderRestoreState(br, &memento) + return false + } + } + + s.copy_length = int(copy_length) + int(v.copy_len_offset) + s.block_length[1]-- + *insert_length += int(insert_len_extra) + return true +} + +func readCommand(s *Reader, br *bitReader, insert_length *int) { + readCommandInternal(0, s, br, insert_length) +} + +func safeReadCommand(s *Reader, br *bitReader, insert_length *int) bool { + return readCommandInternal(1, s, br, insert_length) +} + +func checkInputAmountMaybeSafe(safe int, br *bitReader, num uint) bool { + if safe != 0 { + return true + } + + return checkInputAmount(br, num) +} + +func processCommandsInternal(safe int, s *Reader) int { + var pos int = s.pos + var i int = s.loop_counter + var result int = decoderSuccess + var br *bitReader = &s.br + var hc []huffmanCode + + if !checkInputAmountMaybeSafe(safe, br, 28) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + + if safe == 0 { + warmupBitReader(br) + } + + /* Jump into state machine. */ + if s.state == stateCommandBegin { + goto CommandBegin + } else if s.state == stateCommandInner { + goto CommandInner + } else if s.state == stateCommandPostDecodeLiterals { + goto CommandPostDecodeLiterals + } else if s.state == stateCommandPostWrapCopy { + goto CommandPostWrapCopy + } else { + return decoderErrorUnreachable + } + +CommandBegin: + if safe != 0 { + s.state = stateCommandBegin + } + + if !checkInputAmountMaybeSafe(safe, br, 28) { /* 156 bits + 7 bytes */ + s.state = stateCommandBegin + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + + if s.block_length[1] == 0 { + if safe != 0 { + if !safeDecodeCommandBlockSwitch(s) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + } else { + decodeCommandBlockSwitch(s) + } + + goto CommandBegin + } + + /* Read the insert/copy length in the command. */ + if safe != 0 { + if !safeReadCommand(s, br, &i) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + } else { + readCommand(s, br, &i) + } + + if i == 0 { + goto CommandPostDecodeLiterals + } + + s.meta_block_remaining_len -= i + +CommandInner: + if safe != 0 { + s.state = stateCommandInner + } + + /* Read the literals in the command. */ + if s.trivial_literal_context != 0 { + var bits uint32 + var value uint32 + preloadSymbol(safe, s.literal_htree, br, &bits, &value) + for { + if !checkInputAmountMaybeSafe(safe, br, 28) { /* 162 bits + 7 bytes */ + s.state = stateCommandInner + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + + if s.block_length[0] == 0 { + if safe != 0 { + if !safeDecodeLiteralBlockSwitch(s) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + } else { + decodeLiteralBlockSwitch(s) + } + + preloadSymbol(safe, s.literal_htree, br, &bits, &value) + if s.trivial_literal_context == 0 { + goto CommandInner + } + } + + if safe == 0 { + s.ringbuffer[pos] = byte(readPreloadedSymbol(s.literal_htree, br, &bits, &value)) + } else { + var literal uint32 + if !safeReadSymbol(s.literal_htree, br, &literal) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + + s.ringbuffer[pos] = byte(literal) + } + + s.block_length[0]-- + pos++ + if pos == s.ringbuffer_size { + s.state = stateCommandInnerWrite + i-- + goto saveStateAndReturn + } + i-- + if i == 0 { + break + } + } + } else { + var p1 byte = s.ringbuffer[(pos-1)&s.ringbuffer_mask] + var p2 byte = s.ringbuffer[(pos-2)&s.ringbuffer_mask] + for { + var context byte + if !checkInputAmountMaybeSafe(safe, br, 28) { /* 162 bits + 7 bytes */ + s.state = stateCommandInner + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + + if s.block_length[0] == 0 { + if safe != 0 { + if !safeDecodeLiteralBlockSwitch(s) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + } else { + decodeLiteralBlockSwitch(s) + } + + if s.trivial_literal_context != 0 { + goto CommandInner + } + } + + context = getContext(p1, p2, s.context_lookup) + hc = []huffmanCode(s.literal_hgroup.htrees[s.context_map_slice[context]]) + p2 = p1 + if safe == 0 { + p1 = byte(readSymbol(hc, br)) + } else { + var literal uint32 + if !safeReadSymbol(hc, br, &literal) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + + p1 = byte(literal) + } + + s.ringbuffer[pos] = p1 + s.block_length[0]-- + pos++ + if pos == s.ringbuffer_size { + s.state = stateCommandInnerWrite + i-- + goto saveStateAndReturn + } + i-- + if i == 0 { + break + } + } + } + + if s.meta_block_remaining_len <= 0 { + s.state = stateMetablockDone + goto saveStateAndReturn + } + +CommandPostDecodeLiterals: + if safe != 0 { + s.state = stateCommandPostDecodeLiterals + } + + if s.distance_code >= 0 { + /* Implicit distance case. */ + if s.distance_code != 0 { + s.distance_context = 0 + } else { + s.distance_context = 1 + } + + s.dist_rb_idx-- + s.distance_code = s.dist_rb[s.dist_rb_idx&3] + } else { + /* Read distance code in the command, unless it was implicitly zero. */ + if s.block_length[2] == 0 { + if safe != 0 { + if !safeDecodeDistanceBlockSwitch(s) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + } else { + decodeDistanceBlockSwitch(s) + } + } + + if safe != 0 { + if !safeReadDistance(s, br) { + result = decoderNeedsMoreInput + goto saveStateAndReturn + } + } else { + readDistance(s, br) + } + } + + if s.max_distance != s.max_backward_distance { + if pos < s.max_backward_distance { + s.max_distance = pos + } else { + s.max_distance = s.max_backward_distance + } + } + + i = s.copy_length + + /* Apply copy of LZ77 back-reference, or static dictionary reference if + the distance is larger than the max LZ77 distance */ + if s.distance_code > s.max_distance { + /* The maximum allowed distance is BROTLI_MAX_ALLOWED_DISTANCE = 0x7FFFFFFC. + With this choice, no signed overflow can occur after decoding + a special distance code (e.g., after adding 3 to the last distance). */ + if s.distance_code > maxAllowedDistance { + return decoderErrorFormatDistance + } + + if i >= minDictionaryWordLength && i <= maxDictionaryWordLength { + var address int = s.distance_code - s.max_distance - 1 + var words *dictionary = s.dictionary + var trans *transforms = s.transforms + var offset int = int(s.dictionary.offsets_by_length[i]) + var shift uint32 = uint32(s.dictionary.size_bits_by_length[i]) + var mask int = int(bitMask(shift)) + var word_idx int = address & mask + var transform_idx int = address >> shift + + /* Compensate double distance-ring-buffer roll. */ + s.dist_rb_idx += s.distance_context + + offset += word_idx * i + if words.data == nil { + return decoderErrorDictionaryNotSet + } + + if transform_idx < int(trans.num_transforms) { + word := words.data[offset:] + var len int = i + if transform_idx == int(trans.cutOffTransforms[0]) { + copy(s.ringbuffer[pos:], word[:uint(len)]) + } else { + len = transformDictionaryWord(s.ringbuffer[pos:], word, int(len), trans, transform_idx) + } + + pos += int(len) + s.meta_block_remaining_len -= int(len) + if pos >= s.ringbuffer_size { + s.state = stateCommandPostWrite1 + goto saveStateAndReturn + } + } else { + return decoderErrorFormatTransform + } + } else { + return decoderErrorFormatDictionary + } + } else { + var src_start int = (pos - s.distance_code) & s.ringbuffer_mask + copy_dst := s.ringbuffer[pos:] + copy_src := s.ringbuffer[src_start:] + var dst_end int = pos + i + var src_end int = src_start + i + + /* Update the recent distances cache. */ + s.dist_rb[s.dist_rb_idx&3] = s.distance_code + + s.dist_rb_idx++ + s.meta_block_remaining_len -= i + + /* There are 32+ bytes of slack in the ring-buffer allocation. + Also, we have 16 short codes, that make these 16 bytes irrelevant + in the ring-buffer. Let's copy over them as a first guess. */ + copy(copy_dst, copy_src[:16]) + + if src_end > pos && dst_end > src_start { + /* Regions intersect. */ + goto CommandPostWrapCopy + } + + if dst_end >= s.ringbuffer_size || src_end >= s.ringbuffer_size { + /* At least one region wraps. */ + goto CommandPostWrapCopy + } + + pos += i + if i > 16 { + if i > 32 { + copy(copy_dst[16:], copy_src[16:][:uint(i-16)]) + } else { + /* This branch covers about 45% cases. + Fixed size short copy allows more compiler optimizations. */ + copy(copy_dst[16:], copy_src[16:][:16]) + } + } + } + + if s.meta_block_remaining_len <= 0 { + /* Next metablock, if any. */ + s.state = stateMetablockDone + + goto saveStateAndReturn + } else { + goto CommandBegin + } +CommandPostWrapCopy: + { + var wrap_guard int = s.ringbuffer_size - pos + for { + i-- + if i < 0 { + break + } + s.ringbuffer[pos] = s.ringbuffer[(pos-s.distance_code)&s.ringbuffer_mask] + pos++ + wrap_guard-- + if wrap_guard == 0 { + s.state = stateCommandPostWrite2 + goto saveStateAndReturn + } + } + } + + if s.meta_block_remaining_len <= 0 { + /* Next metablock, if any. */ + s.state = stateMetablockDone + + goto saveStateAndReturn + } else { + goto CommandBegin + } + +saveStateAndReturn: + s.pos = pos + s.loop_counter = i + return result +} + +func processCommands(s *Reader) int { + return processCommandsInternal(0, s) +} + +func safeProcessCommands(s *Reader) int { + return processCommandsInternal(1, s) +} + +/* Returns the maximum number of distance symbols which can only represent + distances not exceeding BROTLI_MAX_ALLOWED_DISTANCE. */ + +var maxDistanceSymbol_bound = [maxNpostfix + 1]uint32{0, 4, 12, 28} +var maxDistanceSymbol_diff = [maxNpostfix + 1]uint32{73, 126, 228, 424} + +func maxDistanceSymbol(ndirect uint32, npostfix uint32) uint32 { + var postfix uint32 = 1 << npostfix + if ndirect < maxDistanceSymbol_bound[npostfix] { + return ndirect + maxDistanceSymbol_diff[npostfix] + postfix + } else if ndirect > maxDistanceSymbol_bound[npostfix]+postfix { + return ndirect + maxDistanceSymbol_diff[npostfix] + } else { + return maxDistanceSymbol_bound[npostfix] + maxDistanceSymbol_diff[npostfix] + postfix + } +} + +/* Invariant: input stream is never overconsumed: + - invalid input implies that the whole stream is invalid -> any amount of + input could be read and discarded + - when result is "needs more input", then at least one more byte is REQUIRED + to complete decoding; all input data MUST be consumed by decoder, so + client could swap the input buffer + - when result is "needs more output" decoder MUST ensure that it doesn't + hold more than 7 bits in bit reader; this saves client from swapping input + buffer ahead of time + - when result is "success" decoder MUST return all unused data back to input + buffer; this is possible because the invariant is held on enter */ +func decoderDecompressStream(s *Reader, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte) int { + var result int = decoderSuccess + var br *bitReader = &s.br + + /* Do not try to process further in a case of unrecoverable error. */ + if int(s.error_code) < 0 { + return decoderResultError + } + + if *available_out != 0 && (next_out == nil || *next_out == nil) { + return saveErrorCode(s, decoderErrorInvalidArguments) + } + + if *available_out == 0 { + next_out = nil + } + if s.buffer_length == 0 { /* Just connect bit reader to input stream. */ + br.input_len = *available_in + br.input = *next_in + br.byte_pos = 0 + } else { + /* At least one byte of input is required. More than one byte of input may + be required to complete the transaction -> reading more data must be + done in a loop -> do it in a main loop. */ + result = decoderNeedsMoreInput + + br.input = s.buffer.u8[:] + br.byte_pos = 0 + } + + /* State machine */ + for { + if result != decoderSuccess { + /* Error, needs more input/output. */ + if result == decoderNeedsMoreInput { + if s.ringbuffer != nil { /* Pro-actively push output. */ + var intermediate_result int = writeRingBuffer(s, available_out, next_out, nil, true) + + /* WriteRingBuffer checks s->meta_block_remaining_len validity. */ + if int(intermediate_result) < 0 { + result = intermediate_result + break + } + } + + if s.buffer_length != 0 { /* Used with internal buffer. */ + if br.byte_pos == br.input_len { + /* Successfully finished read transaction. + Accumulator contains less than 8 bits, because internal buffer + is expanded byte-by-byte until it is enough to complete read. */ + s.buffer_length = 0 + + /* Switch to input stream and restart. */ + result = decoderSuccess + + br.input_len = *available_in + br.input = *next_in + br.byte_pos = 0 + continue + } else if *available_in != 0 { + /* Not enough data in buffer, but can take one more byte from + input stream. */ + result = decoderSuccess + + s.buffer.u8[s.buffer_length] = (*next_in)[0] + s.buffer_length++ + br.input_len = uint(s.buffer_length) + *next_in = (*next_in)[1:] + (*available_in)-- + + /* Retry with more data in buffer. */ + continue + } + + /* Can't finish reading and no more input. */ + break + /* Input stream doesn't contain enough input. */ + } else { + /* Copy tail to internal buffer and return. */ + *next_in = br.input[br.byte_pos:] + + *available_in = br.input_len - br.byte_pos + for *available_in != 0 { + s.buffer.u8[s.buffer_length] = (*next_in)[0] + s.buffer_length++ + *next_in = (*next_in)[1:] + (*available_in)-- + } + + break + } + } + + /* Unreachable. */ + + /* Fail or needs more output. */ + if s.buffer_length != 0 { + /* Just consumed the buffered input and produced some output. Otherwise + it would result in "needs more input". Reset internal buffer. */ + s.buffer_length = 0 + } else { + /* Using input stream in last iteration. When decoder switches to input + stream it has less than 8 bits in accumulator, so it is safe to + return unused accumulator bits there. */ + bitReaderUnload(br) + + *available_in = br.input_len - br.byte_pos + *next_in = br.input[br.byte_pos:] + } + + break + } + + switch s.state { + /* Prepare to the first read. */ + case stateUninited: + if !warmupBitReader(br) { + result = decoderNeedsMoreInput + break + } + + /* Decode window size. */ + result = decodeWindowBits(s, br) /* Reads 1..8 bits. */ + if result != decoderSuccess { + break + } + + if s.large_window { + s.state = stateLargeWindowBits + break + } + + s.state = stateInitialize + + case stateLargeWindowBits: + if !safeReadBits(br, 6, &s.window_bits) { + result = decoderNeedsMoreInput + break + } + + if s.window_bits < largeMinWbits || s.window_bits > largeMaxWbits { + result = decoderErrorFormatWindowBits + break + } + + s.state = stateInitialize + fallthrough + + /* Maximum distance, see section 9.1. of the spec. */ + /* Fall through. */ + case stateInitialize: + s.max_backward_distance = (1 << s.window_bits) - windowGap + + /* Allocate memory for both block_type_trees and block_len_trees. */ + s.block_type_trees = make([]huffmanCode, (3 * (huffmanMaxSize258 + huffmanMaxSize26))) + + if s.block_type_trees == nil { + result = decoderErrorAllocBlockTypeTrees + break + } + + s.block_len_trees = s.block_type_trees[3*huffmanMaxSize258:] + + s.state = stateMetablockBegin + fallthrough + + /* Fall through. */ + case stateMetablockBegin: + decoderStateMetablockBegin(s) + + s.state = stateMetablockHeader + fallthrough + + /* Fall through. */ + case stateMetablockHeader: + result = decodeMetaBlockLength(s, br) + /* Reads 2 - 31 bits. */ + if result != decoderSuccess { + break + } + + if s.is_metadata != 0 || s.is_uncompressed != 0 { + if !bitReaderJumpToByteBoundary(br) { + result = decoderErrorFormatPadding1 + break + } + } + + if s.is_metadata != 0 { + s.state = stateMetadata + break + } + + if s.meta_block_remaining_len == 0 { + s.state = stateMetablockDone + break + } + + calculateRingBufferSize(s) + if s.is_uncompressed != 0 { + s.state = stateUncompressed + break + } + + s.loop_counter = 0 + s.state = stateHuffmanCode0 + + case stateUncompressed: + result = copyUncompressedBlockToOutput(available_out, next_out, nil, s) + if result == decoderSuccess { + s.state = stateMetablockDone + } + + case stateMetadata: + for ; s.meta_block_remaining_len > 0; s.meta_block_remaining_len-- { + var bits uint32 + + /* Read one byte and ignore it. */ + if !safeReadBits(br, 8, &bits) { + result = decoderNeedsMoreInput + break + } + } + + if result == decoderSuccess { + s.state = stateMetablockDone + } + + case stateHuffmanCode0: + if s.loop_counter >= 3 { + s.state = stateMetablockHeader2 + break + } + + /* Reads 1..11 bits. */ + result = decodeVarLenUint8(s, br, &s.num_block_types[s.loop_counter]) + + if result != decoderSuccess { + break + } + + s.num_block_types[s.loop_counter]++ + if s.num_block_types[s.loop_counter] < 2 { + s.loop_counter++ + break + } + + s.state = stateHuffmanCode1 + fallthrough + + case stateHuffmanCode1: + { + var alphabet_size uint32 = s.num_block_types[s.loop_counter] + 2 + var tree_offset int = s.loop_counter * huffmanMaxSize258 + result = readHuffmanCode(alphabet_size, alphabet_size, s.block_type_trees[tree_offset:], nil, s) + if result != decoderSuccess { + break + } + s.state = stateHuffmanCode2 + } + fallthrough + + case stateHuffmanCode2: + { + var alphabet_size uint32 = numBlockLenSymbols + var tree_offset int = s.loop_counter * huffmanMaxSize26 + result = readHuffmanCode(alphabet_size, alphabet_size, s.block_len_trees[tree_offset:], nil, s) + if result != decoderSuccess { + break + } + s.state = stateHuffmanCode3 + } + fallthrough + + case stateHuffmanCode3: + var tree_offset int = s.loop_counter * huffmanMaxSize26 + if !safeReadBlockLength(s, &s.block_length[s.loop_counter], s.block_len_trees[tree_offset:], br) { + result = decoderNeedsMoreInput + break + } + + s.loop_counter++ + s.state = stateHuffmanCode0 + + case stateMetablockHeader2: + { + var bits uint32 + if !safeReadBits(br, 6, &bits) { + result = decoderNeedsMoreInput + break + } + + s.distance_postfix_bits = bits & bitMask(2) + bits >>= 2 + s.num_direct_distance_codes = numDistanceShortCodes + (bits << s.distance_postfix_bits) + s.distance_postfix_mask = int(bitMask(s.distance_postfix_bits)) + s.context_modes = make([]byte, uint(s.num_block_types[0])) + if s.context_modes == nil { + result = decoderErrorAllocContextModes + break + } + + s.loop_counter = 0 + s.state = stateContextModes + } + fallthrough + + case stateContextModes: + result = readContextModes(s) + + if result != decoderSuccess { + break + } + + s.state = stateContextMap1 + fallthrough + + case stateContextMap1: + result = decodeContextMap(s.num_block_types[0]<= 3 { + prepareLiteralDecoding(s) + s.dist_context_map_slice = s.dist_context_map + s.htree_command = []huffmanCode(s.insert_copy_hgroup.htrees[0]) + if !ensureRingBuffer(s) { + result = decoderErrorAllocRingBuffer2 + break + } + + s.state = stateCommandBegin + } + + case stateCommandBegin, stateCommandInner, stateCommandPostDecodeLiterals, stateCommandPostWrapCopy: + result = processCommands(s) + + if result == decoderNeedsMoreInput { + result = safeProcessCommands(s) + } + + case stateCommandInnerWrite, stateCommandPostWrite1, stateCommandPostWrite2: + result = writeRingBuffer(s, available_out, next_out, nil, false) + + if result != decoderSuccess { + break + } + + wrapRingBuffer(s) + if s.ringbuffer_size == 1<= uint64(block_size) { + return 0 + } + return block_size - uint(delta) +} + +/* Wraps 64-bit input position to 32-bit ring-buffer position preserving + "not-a-first-lap" feature. */ +func wrapPosition(position uint64) uint32 { + var result uint32 = uint32(position) + var gb uint64 = position >> 30 + if gb > 2 { + /* Wrap every 2GiB; The first 3GB are continuous. */ + result = result&((1<<30)-1) | (uint32((gb-1)&1)+1)<<30 + } + + return result +} + +func (s *Writer) getStorage(size int) []byte { + if len(s.storage) < size { + s.storage = make([]byte, size) + } + + return s.storage +} + +func hashTableSize(max_table_size uint, input_size uint) uint { + var htsize uint = 256 + for htsize < max_table_size && htsize < input_size { + htsize <<= 1 + } + + return htsize +} + +func getHashTable(s *Writer, quality int, input_size uint, table_size *uint) []int { + var max_table_size uint = maxHashTableSize(quality) + var htsize uint = hashTableSize(max_table_size, input_size) + /* Use smaller hash table when input.size() is smaller, since we + fill the table, incurring O(hash table size) overhead for + compression, and if the input is short, we won't need that + many hash table entries anyway. */ + + var table []int + assert(max_table_size >= 256) + if quality == fastOnePassCompressionQuality { + /* Only odd shifts are supported by fast-one-pass. */ + if htsize&0xAAAAA == 0 { + htsize <<= 1 + } + } + + if htsize <= uint(len(s.small_table_)) { + table = s.small_table_[:] + } else { + if htsize > s.large_table_size_ { + s.large_table_size_ = htsize + s.large_table_ = nil + s.large_table_ = make([]int, htsize) + } + + table = s.large_table_ + } + + *table_size = htsize + for i := 0; i < int(htsize); i++ { + table[i] = 0 + } + return table +} + +func encodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_bytes_bits *byte) { + if large_window { + *last_bytes = uint16((lgwin&0x3F)<<8 | 0x11) + *last_bytes_bits = 14 + } else { + if lgwin == 16 { + *last_bytes = 0 + *last_bytes_bits = 1 + } else if lgwin == 17 { + *last_bytes = 1 + *last_bytes_bits = 7 + } else if lgwin > 17 { + *last_bytes = uint16((lgwin-17)<<1 | 0x01) + *last_bytes_bits = 4 + } else { + *last_bytes = uint16((lgwin-8)<<4 | 0x01) + *last_bytes_bits = 7 + } + } +} + +/* Decide about the context map based on the ability of the prediction + ability of the previous byte UTF8-prefix on the next byte. The + prediction ability is calculated as Shannon entropy. Here we need + Shannon entropy instead of 'BitsEntropy' since the prefix will be + encoded with the remaining 6 bits of the following byte, and + BitsEntropy will assume that symbol to be stored alone using Huffman + coding. */ + +var kStaticContextMapContinuation = [64]uint32{ + 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +} +var kStaticContextMapSimpleUTF8 = [64]uint32{ + 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +} + +func chooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts *uint, literal_context_map *[]uint32) { + var monogram_histo = [3]uint32{0} + var two_prefix_histo = [6]uint32{0} + var total uint + var i uint + var dummy uint + var entropy [4]float64 + for i = 0; i < 9; i++ { + monogram_histo[i%3] += bigram_histo[i] + two_prefix_histo[i%6] += bigram_histo[i] + } + + entropy[1] = shannonEntropy(monogram_histo[:], 3, &dummy) + entropy[2] = (shannonEntropy(two_prefix_histo[:], 3, &dummy) + shannonEntropy(two_prefix_histo[3:], 3, &dummy)) + entropy[3] = 0 + for i = 0; i < 3; i++ { + entropy[3] += shannonEntropy(bigram_histo[3*i:], 3, &dummy) + } + + total = uint(monogram_histo[0] + monogram_histo[1] + monogram_histo[2]) + assert(total != 0) + entropy[0] = 1.0 / float64(total) + entropy[1] *= entropy[0] + entropy[2] *= entropy[0] + entropy[3] *= entropy[0] + + if quality < minQualityForHqContextModeling { + /* 3 context models is a bit slower, don't use it at lower qualities. */ + entropy[3] = entropy[1] * 10 + } + + /* If expected savings by symbol are less than 0.2 bits, skip the + context modeling -- in exchange for faster decoding speed. */ + if entropy[1]-entropy[2] < 0.2 && entropy[1]-entropy[3] < 0.2 { + *num_literal_contexts = 1 + } else if entropy[2]-entropy[3] < 0.02 { + *num_literal_contexts = 2 + *literal_context_map = kStaticContextMapSimpleUTF8[:] + } else { + *num_literal_contexts = 3 + *literal_context_map = kStaticContextMapContinuation[:] + } +} + +/* Decide if we want to use a more complex static context map containing 13 + context values, based on the entropy reduction of histograms over the + first 5 bits of literals. */ + +var kStaticContextMapComplexUTF8 = [64]uint32{ + 11, 11, 12, 12, /* 0 special */ + 0, 0, 0, 0, /* 4 lf */ + 1, 1, 9, 9, /* 8 space */ + 2, 2, 2, 2, /* !, first after space/lf and after something else. */ + 1, 1, 1, 1, /* " */ + 8, 3, 3, 3, /* % */ + 1, 1, 1, 1, /* ({[ */ + 2, 2, 2, 2, /* }]) */ + 8, 4, 4, 4, /* :; */ + 8, 7, 4, 4, /* . */ + 8, 0, 0, 0, /* > */ + 3, 3, 3, 3, /* [0..9] */ + 5, 5, 10, 5, /* [A-Z] */ + 5, 5, 10, 5, + 6, 6, 6, 6, /* [a-z] */ + 6, 6, 6, 6, +} + +func shouldUseComplexStaticContextMap(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) bool { + /* Try the more complex static context map only for long data. */ + if size_hint < 1<<20 { + return false + } else { + var end_pos uint = start_pos + length + var combined_histo = [32]uint32{0} + var context_histo = [13][32]uint32{[32]uint32{0}} + var total uint32 = 0 + var entropy [3]float64 + var dummy uint + var i uint + var utf8_lut contextLUT = getContextLUT(contextUTF8) + /* To make entropy calculations faster and to fit on the stack, we collect + histograms over the 5 most significant bits of literals. One histogram + without context and 13 additional histograms for each context value. */ + for ; start_pos+64 <= end_pos; start_pos += 4096 { + var stride_end_pos uint = start_pos + 64 + var prev2 byte = input[start_pos&mask] + var prev1 byte = input[(start_pos+1)&mask] + var pos uint + + /* To make the analysis of the data faster we only examine 64 byte long + strides at every 4kB intervals. */ + for pos = start_pos + 2; pos < stride_end_pos; pos++ { + var literal byte = input[pos&mask] + var context byte = byte(kStaticContextMapComplexUTF8[getContext(prev1, prev2, utf8_lut)]) + total++ + combined_histo[literal>>3]++ + context_histo[context][literal>>3]++ + prev2 = prev1 + prev1 = literal + } + } + + entropy[1] = shannonEntropy(combined_histo[:], 32, &dummy) + entropy[2] = 0 + for i = 0; i < 13; i++ { + entropy[2] += shannonEntropy(context_histo[i][0:], 32, &dummy) + } + + entropy[0] = 1.0 / float64(total) + entropy[1] *= entropy[0] + entropy[2] *= entropy[0] + + /* The triggering heuristics below were tuned by compressing the individual + files of the silesia corpus. If we skip this kind of context modeling + for not very well compressible input (i.e. entropy using context modeling + is 60% of maximal entropy) or if expected savings by symbol are less + than 0.2 bits, then in every case when it triggers, the final compression + ratio is improved. Note however that this heuristics might be too strict + for some cases and could be tuned further. */ + if entropy[2] > 3.0 || entropy[1]-entropy[2] < 0.2 { + return false + } else { + *num_literal_contexts = 13 + *literal_context_map = kStaticContextMapComplexUTF8[:] + return true + } + } +} + +func decideOverLiteralContextModeling(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) { + if quality < minQualityForContextModeling || length < 64 { + return + } else if shouldUseComplexStaticContextMap(input, start_pos, length, mask, quality, size_hint, num_literal_contexts, literal_context_map) { + } else /* Context map was already set, nothing else to do. */ + { + var end_pos uint = start_pos + length + /* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of + UTF8 data faster we only examine 64 byte long strides at every 4kB + intervals. */ + + var bigram_prefix_histo = [9]uint32{0} + for ; start_pos+64 <= end_pos; start_pos += 4096 { + var lut = [4]int{0, 0, 1, 2} + var stride_end_pos uint = start_pos + 64 + var prev int = lut[input[start_pos&mask]>>6] * 3 + var pos uint + for pos = start_pos + 1; pos < stride_end_pos; pos++ { + var literal byte = input[pos&mask] + bigram_prefix_histo[prev+lut[literal>>6]]++ + prev = lut[literal>>6] * 3 + } + } + + chooseContextMap(quality, bigram_prefix_histo[0:], num_literal_contexts, literal_context_map) + } +} + +func shouldCompress_encode(data []byte, mask uint, last_flush_pos uint64, bytes uint, num_literals uint, num_commands uint) bool { + /* TODO: find more precise minimal block overhead. */ + if bytes <= 2 { + return false + } + if num_commands < (bytes>>8)+2 { + if float64(num_literals) > 0.99*float64(bytes) { + var literal_histo = [256]uint32{0} + const kSampleRate uint32 = 13 + const kMinEntropy float64 = 7.92 + var bit_cost_threshold float64 = float64(bytes) * kMinEntropy / float64(kSampleRate) + var t uint = uint((uint32(bytes) + kSampleRate - 1) / kSampleRate) + var pos uint32 = uint32(last_flush_pos) + var i uint + for i = 0; i < t; i++ { + literal_histo[data[pos&uint32(mask)]]++ + pos += kSampleRate + } + + if bitsEntropy(literal_histo[:], 256) > bit_cost_threshold { + return false + } + } + } + + return true +} + +/* Chooses the literal context mode for a metablock */ +func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, length uint) int { + /* We only do the computation for the option of something else than + CONTEXT_UTF8 for the highest qualities */ + if params.quality >= minQualityForHqBlockSplitting && !isMostlyUTF8(data, pos, mask, length, kMinUTF8Ratio) { + return contextSigned + } + + return contextUTF8 +} + +func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { + var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos) + var last_bytes uint16 + var last_bytes_bits byte + var literal_context_lut contextLUT = getContextLUT(literal_context_mode) + var block_params encoderParams = *params + + if bytes == 0 { + /* Write the ISLAST and ISEMPTY bits. */ + writeBits(2, 3, storage_ix, storage) + + *storage_ix = (*storage_ix + 7) &^ 7 + return + } + + if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) { + /* Restore the distance cache, as its last update by + CreateBackwardReferences is now unused. */ + copy(dist_cache, saved_dist_cache[:4]) + + storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage) + return + } + + assert(*storage_ix <= 14) + last_bytes = uint16(storage[1])<<8 | uint16(storage[0]) + last_bytes_bits = byte(*storage_ix) + if params.quality <= maxQualityForStaticEntropyCodes { + storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) + } else if params.quality < minQualityForBlockSplit { + storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) + } else { + mb := getMetaBlockSplit() + if params.quality < minQualityForHqBlockSplitting { + var num_literal_contexts uint = 1 + var literal_context_map []uint32 = nil + if !params.disable_literal_context_modeling { + decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map) + } + + buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, mb) + } else { + buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, mb) + } + + if params.quality >= minQualityForOptimizeHistograms { + /* The number of distance symbols effectively used for distance + histograms. It might be less than distance alphabet size + for "Large Window Brotli" (32-bit). */ + var num_effective_dist_codes uint32 = block_params.dist.alphabet_size + if num_effective_dist_codes > numHistogramDistanceSymbols { + num_effective_dist_codes = numHistogramDistanceSymbols + } + + optimizeHistograms(num_effective_dist_codes, mb) + } + + storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, storage_ix, storage) + freeMetaBlockSplit(mb) + } + + if bytes+4 < *storage_ix>>3 { + /* Restore the distance cache and last byte. */ + copy(dist_cache, saved_dist_cache[:4]) + + storage[0] = byte(last_bytes) + storage[1] = byte(last_bytes >> 8) + *storage_ix = uint(last_bytes_bits) + storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage) + } +} + +func chooseDistanceParams(params *encoderParams) { + var distance_postfix_bits uint32 = 0 + var num_direct_distance_codes uint32 = 0 + + if params.quality >= minQualityForNonzeroDistanceParams { + var ndirect_msb uint32 + if params.mode == modeFont { + distance_postfix_bits = 1 + num_direct_distance_codes = 12 + } else { + distance_postfix_bits = params.dist.distance_postfix_bits + num_direct_distance_codes = params.dist.num_direct_distance_codes + } + + ndirect_msb = (num_direct_distance_codes >> distance_postfix_bits) & 0x0F + if distance_postfix_bits > maxNpostfix || num_direct_distance_codes > maxNdirect || ndirect_msb<>25)), (last_command.dist_prefix_&0x3FF == 0), &last_command.cmd_prefix_) + } +} + +/* + Processes the accumulated input data and writes + the new output meta-block to s.dest, if one has been + created (otherwise the processed input data is buffered internally). + If |is_last| or |force_flush| is true, an output meta-block is + always created. However, until |is_last| is true encoder may retain up + to 7 bits of the last byte of output. To force encoder to dump the remaining + bits use WriteMetadata() to append an empty meta-data block. + Returns false if the size of the input data is larger than + input_block_size(). +*/ +func encodeData(s *Writer, is_last bool, force_flush bool) bool { + var delta uint64 = unprocessedInputSize(s) + var bytes uint32 = uint32(delta) + var wrapped_last_processed_pos uint32 = wrapPosition(s.last_processed_pos_) + var data []byte + var mask uint32 + var literal_context_mode int + + data = s.ringbuffer_.buffer_ + mask = s.ringbuffer_.mask_ + + /* Adding more blocks after "last" block is forbidden. */ + if s.is_last_block_emitted_ { + return false + } + if is_last { + s.is_last_block_emitted_ = true + } + + if delta > uint64(inputBlockSize(s)) { + return false + } + + if s.params.quality == fastTwoPassCompressionQuality { + if s.command_buf_ == nil || cap(s.command_buf_) < int(kCompressFragmentTwoPassBlockSize) { + s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize) + s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize) + } else { + s.command_buf_ = s.command_buf_[:kCompressFragmentTwoPassBlockSize] + s.literal_buf_ = s.literal_buf_[:kCompressFragmentTwoPassBlockSize] + } + } + + if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality { + var storage []byte + var storage_ix uint = uint(s.last_bytes_bits_) + var table_size uint + var table []int + + if delta == 0 && !is_last { + /* We have no new input data and we don't have to finish the stream, so + nothing to do. */ + return true + } + + storage = s.getStorage(int(2*bytes + 503)) + storage[0] = byte(s.last_bytes_) + storage[1] = byte(s.last_bytes_ >> 8) + table = getHashTable(s, s.params.quality, uint(bytes), &table_size) + if s.params.quality == fastOnePassCompressionQuality { + compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage) + } else { + compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &storage_ix, storage) + } + + s.last_bytes_ = uint16(storage[storage_ix>>3]) + s.last_bytes_bits_ = byte(storage_ix & 7) + updateLastProcessedPos(s) + s.writeOutput(storage[:storage_ix>>3]) + return true + } + { + /* Theoretical max number of commands is 1 per 2 bytes. */ + newsize := len(s.commands) + int(bytes)/2 + 1 + if newsize > cap(s.commands) { + /* Reserve a bit more memory to allow merging with a next block + without reallocation: that would impact speed. */ + newsize += int(bytes/4) + 16 + + new_commands := make([]command, len(s.commands), newsize) + if s.commands != nil { + copy(new_commands, s.commands) + } + + s.commands = new_commands + } + } + + initOrStitchToPreviousBlock(&s.hasher_, data, uint(mask), &s.params, uint(wrapped_last_processed_pos), uint(bytes), is_last) + + literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_)) + + if len(s.commands) != 0 && s.last_insert_len_ == 0 { + extendLastCommand(s, &bytes, &wrapped_last_processed_pos) + } + + if s.params.quality == zopflificationQuality { + assert(s.params.hasher.type_ == 10) + createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) + } else if s.params.quality == hqZopflificationQuality { + assert(s.params.hasher.type_ == 10) + createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) + } else { + createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) + } + { + var max_length uint = maxMetablockSize(&s.params) + var max_literals uint = max_length / 8 + max_commands := int(max_length / 8) + var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_) + var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length) + var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols) + /* If maximal possible additional block doesn't fit metablock, flush now. */ + /* TODO: Postpone decision until next block arrives? */ + + /* If block splitting is not used, then flush as soon as there is some + amount of commands / literals produced. */ + if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands { + /* Merge with next input block. Everything will happen later. */ + if updateLastProcessedPos(s) { + hasherReset(s.hasher_) + } + + return true + } + } + + /* Create the last insert-only command. */ + if s.last_insert_len_ > 0 { + s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_)) + s.num_literals_ += s.last_insert_len_ + s.last_insert_len_ = 0 + } + + if !is_last && s.input_pos_ == s.last_flush_pos_ { + /* We have no new input data and we don't have to finish the stream, so + nothing to do. */ + return true + } + + assert(s.input_pos_ >= s.last_flush_pos_) + assert(s.input_pos_ > s.last_flush_pos_ || is_last) + assert(s.input_pos_-s.last_flush_pos_ <= 1<<24) + { + var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_) + var storage []byte = s.getStorage(int(2*metablock_size + 503)) + var storage_ix uint = uint(s.last_bytes_bits_) + storage[0] = byte(s.last_bytes_) + storage[1] = byte(s.last_bytes_ >> 8) + writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) + s.last_bytes_ = uint16(storage[storage_ix>>3]) + s.last_bytes_bits_ = byte(storage_ix & 7) + s.last_flush_pos_ = s.input_pos_ + if updateLastProcessedPos(s) { + hasherReset(s.hasher_) + } + + if s.last_flush_pos_ > 0 { + s.prev_byte_ = data[(uint32(s.last_flush_pos_)-1)&mask] + } + + if s.last_flush_pos_ > 1 { + s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask] + } + + s.commands = s.commands[:0] + s.num_literals_ = 0 + + /* Save the state of the distance cache in case we need to restore it for + emitting an uncompressed block. */ + copy(s.saved_dist_cache_[:], s.dist_cache_[:]) + + s.writeOutput(storage[:storage_ix>>3]) + return true + } +} + +/* Dumps remaining output bits and metadata header to |header|. + Returns number of produced bytes. + REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. + REQUIRED: |block_size| <= (1 << 24). */ +func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { + storage_ix := uint(s.last_bytes_bits_) + header[0] = byte(s.last_bytes_) + header[1] = byte(s.last_bytes_ >> 8) + s.last_bytes_ = 0 + s.last_bytes_bits_ = 0 + + writeBits(1, 0, &storage_ix, header) + writeBits(2, 3, &storage_ix, header) + writeBits(1, 0, &storage_ix, header) + if block_size == 0 { + writeBits(2, 0, &storage_ix, header) + } else { + var nbits uint32 + if block_size == 1 { + nbits = 0 + } else { + nbits = log2FloorNonZero(uint(uint32(block_size)-1)) + 1 + } + var nbytes uint32 = (nbits + 7) / 8 + writeBits(2, uint64(nbytes), &storage_ix, header) + writeBits(uint(8*nbytes), uint64(block_size)-1, &storage_ix, header) + } + + return (storage_ix + 7) >> 3 +} + +func injectBytePaddingBlock(s *Writer) { + var seal uint32 = uint32(s.last_bytes_) + var seal_bits uint = uint(s.last_bytes_bits_) + s.last_bytes_ = 0 + s.last_bytes_bits_ = 0 + + /* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */ + seal |= 0x6 << seal_bits + + seal_bits += 6 + + destination := s.tiny_buf_.u8[:] + + destination[0] = byte(seal) + if seal_bits > 8 { + destination[1] = byte(seal >> 8) + } + if seal_bits > 16 { + destination[2] = byte(seal >> 16) + } + s.writeOutput(destination[:(seal_bits+7)>>3]) +} + +func checkFlushComplete(s *Writer) { + if s.stream_state_ == streamFlushRequested && s.err == nil { + s.stream_state_ = streamProcessing + } +} + +func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[]byte) bool { + var block_size_limit uint = uint(1) << s.params.lgwin + var buf_size uint = brotli_min_size_t(kCompressFragmentTwoPassBlockSize, brotli_min_size_t(*available_in, block_size_limit)) + var command_buf []uint32 = nil + var literal_buf []byte = nil + if s.params.quality != fastOnePassCompressionQuality && s.params.quality != fastTwoPassCompressionQuality { + return false + } + + if s.params.quality == fastTwoPassCompressionQuality { + if s.command_buf_ == nil || cap(s.command_buf_) < int(buf_size) { + s.command_buf_ = make([]uint32, buf_size) + s.literal_buf_ = make([]byte, buf_size) + } else { + s.command_buf_ = s.command_buf_[:buf_size] + s.literal_buf_ = s.literal_buf_[:buf_size] + } + + command_buf = s.command_buf_ + literal_buf = s.literal_buf_ + } + + for { + if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { + injectBytePaddingBlock(s) + continue + } + + /* Compress block only when stream is not + finished, there is no pending flush request, and there is either + additional input or pending operation. */ + if s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) { + var block_size uint = brotli_min_size_t(block_size_limit, *available_in) + var is_last bool = (*available_in == block_size) && (op == int(operationFinish)) + var force_flush bool = (*available_in == block_size) && (op == int(operationFlush)) + var max_out_size uint = 2*block_size + 503 + var storage []byte = nil + var storage_ix uint = uint(s.last_bytes_bits_) + var table_size uint + var table []int + + if force_flush && block_size == 0 { + s.stream_state_ = streamFlushRequested + continue + } + + storage = s.getStorage(int(max_out_size)) + + storage[0] = byte(s.last_bytes_) + storage[1] = byte(s.last_bytes_ >> 8) + table = getHashTable(s, s.params.quality, block_size, &table_size) + + if s.params.quality == fastOnePassCompressionQuality { + compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage) + } else { + compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &storage_ix, storage) + } + + *next_in = (*next_in)[block_size:] + *available_in -= block_size + var out_bytes uint = storage_ix >> 3 + s.writeOutput(storage[:out_bytes]) + + s.last_bytes_ = uint16(storage[storage_ix>>3]) + s.last_bytes_bits_ = byte(storage_ix & 7) + + if force_flush { + s.stream_state_ = streamFlushRequested + } + if is_last { + s.stream_state_ = streamFinished + } + continue + } + + break + } + + checkFlushComplete(s) + return true +} + +func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { + if *available_in > 1<<24 { + return false + } + + /* Switch to metadata block workflow, if required. */ + if s.stream_state_ == streamProcessing { + s.remaining_metadata_bytes_ = uint32(*available_in) + s.stream_state_ = streamMetadataHead + } + + if s.stream_state_ != streamMetadataHead && s.stream_state_ != streamMetadataBody { + return false + } + + for { + if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { + injectBytePaddingBlock(s) + continue + } + + if s.input_pos_ != s.last_flush_pos_ { + var result bool = encodeData(s, false, true) + if !result { + return false + } + continue + } + + if s.stream_state_ == streamMetadataHead { + n := writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.tiny_buf_.u8[:]) + s.writeOutput(s.tiny_buf_.u8[:n]) + s.stream_state_ = streamMetadataBody + continue + } else { + /* Exit workflow only when there is no more input and no more output. + Otherwise client may continue producing empty metadata blocks. */ + if s.remaining_metadata_bytes_ == 0 { + s.remaining_metadata_bytes_ = math.MaxUint32 + s.stream_state_ = streamProcessing + break + } + + /* This guarantees progress in "TakeOutput" workflow. */ + var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16) + copy(s.tiny_buf_.u8[:], (*next_in)[:c]) + *next_in = (*next_in)[c:] + *available_in -= uint(c) + s.remaining_metadata_bytes_ -= c + s.writeOutput(s.tiny_buf_.u8[:c]) + + continue + } + } + + return true +} + +func updateSizeHint(s *Writer, available_in uint) { + if s.params.size_hint == 0 { + var delta uint64 = unprocessedInputSize(s) + var tail uint64 = uint64(available_in) + var limit uint32 = 1 << 30 + var total uint32 + if (delta >= uint64(limit)) || (tail >= uint64(limit)) || ((delta + tail) >= uint64(limit)) { + total = limit + } else { + total = uint32(delta + tail) + } + + s.params.size_hint = uint(total) + } +} + +func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byte) bool { + if !ensureInitialized(s) { + return false + } + + /* Unfinished metadata block; check requirements. */ + if s.remaining_metadata_bytes_ != math.MaxUint32 { + if uint32(*available_in) != s.remaining_metadata_bytes_ { + return false + } + if op != int(operationEmitMetadata) { + return false + } + } + + if op == int(operationEmitMetadata) { + updateSizeHint(s, 0) /* First data metablock might be emitted here. */ + return processMetadata(s, available_in, next_in) + } + + if s.stream_state_ == streamMetadataHead || s.stream_state_ == streamMetadataBody { + return false + } + + if s.stream_state_ != streamProcessing && *available_in != 0 { + return false + } + + if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality { + return encoderCompressStreamFast(s, op, available_in, next_in) + } + + for { + var remaining_block_size uint = remainingInputBlockSize(s) + + if remaining_block_size != 0 && *available_in != 0 { + var copy_input_size uint = brotli_min_size_t(remaining_block_size, *available_in) + copyInputToRingBuffer(s, copy_input_size, *next_in) + *next_in = (*next_in)[copy_input_size:] + *available_in -= copy_input_size + continue + } + + if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { + injectBytePaddingBlock(s) + continue + } + + /* Compress data only when stream is not + finished and there is no pending flush request. */ + if s.stream_state_ == streamProcessing { + if remaining_block_size == 0 || op != int(operationProcess) { + var is_last bool = ((*available_in == 0) && op == int(operationFinish)) + var force_flush bool = ((*available_in == 0) && op == int(operationFlush)) + var result bool + updateSizeHint(s, *available_in) + result = encodeData(s, is_last, force_flush) + if !result { + return false + } + if force_flush { + s.stream_state_ = streamFlushRequested + } + if is_last { + s.stream_state_ = streamFinished + } + continue + } + } + + break + } + + checkFlushComplete(s) + return true +} + +func (w *Writer) writeOutput(data []byte) { + if w.err != nil { + return + } + + _, w.err = w.dst.Write(data) + if w.err == nil { + checkFlushComplete(w) + } +} diff --git a/vendor/github.com/andybalholm/brotli/encoder.go b/vendor/github.com/andybalholm/brotli/encoder.go new file mode 100644 index 0000000000..650d1e42b4 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/encoder.go @@ -0,0 +1,168 @@ +package brotli + +import "github.com/andybalholm/brotli/matchfinder" + +// An Encoder implements the matchfinder.Encoder interface, writing in Brotli format. +type Encoder struct { + wroteHeader bool + bw bitWriter + distCache []distanceCode +} + +func (e *Encoder) Reset() { + e.wroteHeader = false + e.bw = bitWriter{} +} + +func (e *Encoder) Encode(dst []byte, src []byte, matches []matchfinder.Match, lastBlock bool) []byte { + e.bw.dst = dst + if !e.wroteHeader { + e.bw.writeBits(4, 15) + e.wroteHeader = true + } + + var literalHisto [256]uint32 + var commandHisto [704]uint32 + var distanceHisto [64]uint32 + literalCount := 0 + commandCount := 0 + distanceCount := 0 + + if len(e.distCache) < len(matches) { + e.distCache = make([]distanceCode, len(matches)) + } + + // first pass: build the histograms + pos := 0 + + // d is the ring buffer of the last 4 distances. + d := [4]int{-10, -10, -10, -10} + for i, m := range matches { + if m.Unmatched > 0 { + for _, c := range src[pos : pos+m.Unmatched] { + literalHisto[c]++ + } + literalCount += m.Unmatched + } + + insertCode := getInsertLengthCode(uint(m.Unmatched)) + copyCode := getCopyLengthCode(uint(m.Length)) + if m.Length == 0 { + // If the stream ends with unmatched bytes, we need a dummy copy length. + copyCode = 2 + } + command := combineLengthCodes(insertCode, copyCode, false) + commandHisto[command]++ + commandCount++ + + if command >= 128 && m.Length != 0 { + var distCode distanceCode + switch m.Distance { + case d[3]: + distCode.code = 0 + case d[2]: + distCode.code = 1 + case d[1]: + distCode.code = 2 + case d[0]: + distCode.code = 3 + case d[3] - 1: + distCode.code = 4 + case d[3] + 1: + distCode.code = 5 + case d[3] - 2: + distCode.code = 6 + case d[3] + 2: + distCode.code = 7 + case d[3] - 3: + distCode.code = 8 + case d[3] + 3: + distCode.code = 9 + + // In my testing, codes 10โ€“15 actually reduced the compression ratio. + + default: + distCode = getDistanceCode(m.Distance) + } + e.distCache[i] = distCode + distanceHisto[distCode.code]++ + distanceCount++ + if distCode.code != 0 { + d[0], d[1], d[2], d[3] = d[1], d[2], d[3], m.Distance + } + } + + pos += m.Unmatched + m.Length + } + + storeMetaBlockHeaderBW(uint(len(src)), false, &e.bw) + e.bw.writeBits(13, 0) + + var literalDepths [256]byte + var literalBits [256]uint16 + buildAndStoreHuffmanTreeFastBW(literalHisto[:], uint(literalCount), 8, literalDepths[:], literalBits[:], &e.bw) + + var commandDepths [704]byte + var commandBits [704]uint16 + buildAndStoreHuffmanTreeFastBW(commandHisto[:], uint(commandCount), 10, commandDepths[:], commandBits[:], &e.bw) + + var distanceDepths [64]byte + var distanceBits [64]uint16 + buildAndStoreHuffmanTreeFastBW(distanceHisto[:], uint(distanceCount), 6, distanceDepths[:], distanceBits[:], &e.bw) + + pos = 0 + for i, m := range matches { + insertCode := getInsertLengthCode(uint(m.Unmatched)) + copyCode := getCopyLengthCode(uint(m.Length)) + if m.Length == 0 { + // If the stream ends with unmatched bytes, we need a dummy copy length. + copyCode = 2 + } + command := combineLengthCodes(insertCode, copyCode, false) + e.bw.writeBits(uint(commandDepths[command]), uint64(commandBits[command])) + if kInsExtra[insertCode] > 0 { + e.bw.writeBits(uint(kInsExtra[insertCode]), uint64(m.Unmatched)-uint64(kInsBase[insertCode])) + } + if kCopyExtra[copyCode] > 0 { + e.bw.writeBits(uint(kCopyExtra[copyCode]), uint64(m.Length)-uint64(kCopyBase[copyCode])) + } + + if m.Unmatched > 0 { + for _, c := range src[pos : pos+m.Unmatched] { + e.bw.writeBits(uint(literalDepths[c]), uint64(literalBits[c])) + } + } + + if command >= 128 && m.Length != 0 { + distCode := e.distCache[i] + e.bw.writeBits(uint(distanceDepths[distCode.code]), uint64(distanceBits[distCode.code])) + if distCode.nExtra > 0 { + e.bw.writeBits(distCode.nExtra, distCode.extraBits) + } + } + + pos += m.Unmatched + m.Length + } + + if lastBlock { + e.bw.writeBits(2, 3) // islast + isempty + e.bw.jumpToByteBoundary() + } + return e.bw.dst +} + +type distanceCode struct { + code int + nExtra uint + extraBits uint64 +} + +func getDistanceCode(distance int) distanceCode { + d := distance + 3 + nbits := log2FloorNonZero(uint(d)) - 1 + prefix := (d >> nbits) & 1 + offset := (2 + prefix) << nbits + distcode := int(2*(nbits-1)) + prefix + 16 + extra := d - offset + return distanceCode{distcode, uint(nbits), uint64(extra)} +} diff --git a/vendor/github.com/andybalholm/brotli/encoder_dict.go b/vendor/github.com/andybalholm/brotli/encoder_dict.go new file mode 100644 index 0000000000..55c051c623 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/encoder_dict.go @@ -0,0 +1,22 @@ +package brotli + +/* Dictionary data (words and transforms) for 1 possible context */ +type encoderDictionary struct { + words *dictionary + cutoffTransformsCount uint32 + cutoffTransforms uint64 + hash_table []uint16 + buckets []uint16 + dict_words []dictWord +} + +func initEncoderDictionary(dict *encoderDictionary) { + dict.words = getDictionary() + + dict.hash_table = kStaticDictionaryHash[:] + dict.buckets = kStaticDictionaryBuckets[:] + dict.dict_words = kStaticDictionaryWords[:] + + dict.cutoffTransformsCount = kCutoffTransformsCount + dict.cutoffTransforms = kCutoffTransforms +} diff --git a/vendor/github.com/andybalholm/brotli/entropy_encode.go b/vendor/github.com/andybalholm/brotli/entropy_encode.go new file mode 100644 index 0000000000..3f469a3dd9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/entropy_encode.go @@ -0,0 +1,592 @@ +package brotli + +import "math" + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Entropy encoding (Huffman) utilities. */ + +/* A node of a Huffman tree. */ +type huffmanTree struct { + total_count_ uint32 + index_left_ int16 + index_right_or_value_ int16 +} + +func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) { + self.total_count_ = count + self.index_left_ = left + self.index_right_or_value_ = right +} + +/* Input size optimized Shell sort. */ +type huffmanTreeComparator func(huffmanTree, huffmanTree) bool + +var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1} + +func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeComparator) { + if n < 13 { + /* Insertion sort. */ + var i uint + for i = 1; i < n; i++ { + var tmp huffmanTree = items[i] + var k uint = i + var j uint = i - 1 + for comparator(tmp, items[j]) { + items[k] = items[j] + k = j + if j == 0 { + break + } + j-- + } + + items[k] = tmp + } + + return + } else { + var g int + if n < 57 { + g = 2 + } else { + g = 0 + } + for ; g < 6; g++ { + var gap uint = sortHuffmanTreeItems_gaps[g] + var i uint + for i = gap; i < n; i++ { + var j uint = i + var tmp huffmanTree = items[i] + for ; j >= gap && comparator(tmp, items[j-gap]); j -= gap { + items[j] = items[j-gap] + } + + items[j] = tmp + } + } + } +} + +/* Returns 1 if assignment of depths succeeded, otherwise 0. */ +func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool { + var stack [16]int + var level int = 0 + var p int = p0 + assert(max_depth <= 15) + stack[0] = -1 + for { + if pool[p].index_left_ >= 0 { + level++ + if level > max_depth { + return false + } + stack[level] = int(pool[p].index_right_or_value_) + p = int(pool[p].index_left_) + continue + } else { + depth[pool[p].index_right_or_value_] = byte(level) + } + + for level >= 0 && stack[level] == -1 { + level-- + } + if level < 0 { + return true + } + p = stack[level] + stack[level] = -1 + } +} + +/* Sort the root nodes, least popular first. */ +func sortHuffmanTree(v0 huffmanTree, v1 huffmanTree) bool { + if v0.total_count_ != v1.total_count_ { + return v0.total_count_ < v1.total_count_ + } + + return v0.index_right_or_value_ > v1.index_right_or_value_ +} + +/* This function will create a Huffman tree. + + The catch here is that the tree cannot be arbitrarily deep. + Brotli specifies a maximum depth of 15 bits for "code trees" + and 7 bits for "code length code trees." + + count_limit is the value that is to be faked as the minimum value + and this minimum value is raised until the tree matches the + maximum length requirement. + + This algorithm is not of excellent performance for very long data blocks, + especially when population counts are longer than 2**tree_limit, but + we are not planning to use this with extremely long blocks. + + See http://en.wikipedia.org/wiki/Huffman_coding */ +func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) { + var count_limit uint32 + var sentinel huffmanTree + initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) + + /* For block sizes below 64 kB, we never need to do a second iteration + of this loop. Probably all of our block sizes will be smaller than + that, so this loop is mostly of academic interest. If we actually + would need this, we would be better off with the Katajainen algorithm. */ + for count_limit = 1; ; count_limit *= 2 { + var n uint = 0 + var i uint + var j uint + var k uint + for i = length; i != 0; { + i-- + if data[i] != 0 { + var count uint32 = brotli_max_uint32_t(data[i], count_limit) + initHuffmanTree(&tree[n], count, -1, int16(i)) + n++ + } + } + + if n == 1 { + depth[tree[0].index_right_or_value_] = 1 /* Only one element. */ + break + } + + sortHuffmanTreeItems(tree, n, huffmanTreeComparator(sortHuffmanTree)) + + /* The nodes are: + [0, n): the sorted leaf nodes that we start with. + [n]: we add a sentinel here. + [n + 1, 2n): new parent nodes are added here, starting from + (n+1). These are naturally in ascending order. + [2n]: we add a sentinel at the end as well. + There will be (2n+1) elements at the end. */ + tree[n] = sentinel + + tree[n+1] = sentinel + + i = 0 /* Points to the next leaf node. */ + j = n + 1 /* Points to the next non-leaf node. */ + for k = n - 1; k != 0; k-- { + var left uint + var right uint + if tree[i].total_count_ <= tree[j].total_count_ { + left = i + i++ + } else { + left = j + j++ + } + + if tree[i].total_count_ <= tree[j].total_count_ { + right = i + i++ + } else { + right = j + j++ + } + { + /* The sentinel node becomes the parent node. */ + var j_end uint = 2*n - k + tree[j_end].total_count_ = tree[left].total_count_ + tree[right].total_count_ + tree[j_end].index_left_ = int16(left) + tree[j_end].index_right_or_value_ = int16(right) + + /* Add back the last sentinel node. */ + tree[j_end+1] = sentinel + } + } + + if setDepth(int(2*n-1), tree[0:], depth, tree_limit) { + /* We need to pack the Huffman tree in tree_limit bits. If this was not + successful, add fake entities to the lowest values and retry. */ + break + } + } +} + +func reverse(v []byte, start uint, end uint) { + end-- + for start < end { + var tmp byte = v[start] + v[start] = v[end] + v[end] = tmp + start++ + end-- + } +} + +func writeHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) { + assert(repetitions > 0) + if previous_value != value { + tree[*tree_size] = value + extra_bits_data[*tree_size] = 0 + (*tree_size)++ + repetitions-- + } + + if repetitions == 7 { + tree[*tree_size] = value + extra_bits_data[*tree_size] = 0 + (*tree_size)++ + repetitions-- + } + + if repetitions < 3 { + var i uint + for i = 0; i < repetitions; i++ { + tree[*tree_size] = value + extra_bits_data[*tree_size] = 0 + (*tree_size)++ + } + } else { + var start uint = *tree_size + repetitions -= 3 + for { + tree[*tree_size] = repeatPreviousCodeLength + extra_bits_data[*tree_size] = byte(repetitions & 0x3) + (*tree_size)++ + repetitions >>= 2 + if repetitions == 0 { + break + } + + repetitions-- + } + + reverse(tree, start, *tree_size) + reverse(extra_bits_data, start, *tree_size) + } +} + +func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) { + if repetitions == 11 { + tree[*tree_size] = 0 + extra_bits_data[*tree_size] = 0 + (*tree_size)++ + repetitions-- + } + + if repetitions < 3 { + var i uint + for i = 0; i < repetitions; i++ { + tree[*tree_size] = 0 + extra_bits_data[*tree_size] = 0 + (*tree_size)++ + } + } else { + var start uint = *tree_size + repetitions -= 3 + for { + tree[*tree_size] = repeatZeroCodeLength + extra_bits_data[*tree_size] = byte(repetitions & 0x7) + (*tree_size)++ + repetitions >>= 3 + if repetitions == 0 { + break + } + + repetitions-- + } + + reverse(tree, start, *tree_size) + reverse(extra_bits_data, start, *tree_size) + } +} + +/* Change the population counts in a way that the consequent + Huffman tree compression, especially its RLE-part will be more + likely to compress this data more efficiently. + + length contains the size of the histogram. + counts contains the population counts. + good_for_rle is a buffer of at least length size */ +func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) { + var nonzero_count uint = 0 + var stride uint + var limit uint + var sum uint + var streak_limit uint = 1240 + var i uint + /* Let's make the Huffman code more compatible with RLE encoding. */ + for i = 0; i < length; i++ { + if counts[i] != 0 { + nonzero_count++ + } + } + + if nonzero_count < 16 { + return + } + + for length != 0 && counts[length-1] == 0 { + length-- + } + + if length == 0 { + return /* All zeros. */ + } + + /* Now counts[0..length - 1] does not have trailing zeros. */ + { + var nonzeros uint = 0 + var smallest_nonzero uint32 = 1 << 30 + for i = 0; i < length; i++ { + if counts[i] != 0 { + nonzeros++ + if smallest_nonzero > counts[i] { + smallest_nonzero = counts[i] + } + } + } + + if nonzeros < 5 { + /* Small histogram will model it well. */ + return + } + + if smallest_nonzero < 4 { + var zeros uint = length - nonzeros + if zeros < 6 { + for i = 1; i < length-1; i++ { + if counts[i-1] != 0 && counts[i] == 0 && counts[i+1] != 0 { + counts[i] = 1 + } + } + } + } + + if nonzeros < 28 { + return + } + } + + /* 2) Let's mark all population counts that already can be encoded + with an RLE code. */ + for i := 0; i < int(length); i++ { + good_for_rle[i] = 0 + } + { + var symbol uint32 = counts[0] + /* Let's not spoil any of the existing good RLE codes. + Mark any seq of 0's that is longer as 5 as a good_for_rle. + Mark any seq of non-0's that is longer as 7 as a good_for_rle. */ + + var step uint = 0 + for i = 0; i <= length; i++ { + if i == length || counts[i] != symbol { + if (symbol == 0 && step >= 5) || (symbol != 0 && step >= 7) { + var k uint + for k = 0; k < step; k++ { + good_for_rle[i-k-1] = 1 + } + } + + step = 1 + if i != length { + symbol = counts[i] + } + } else { + step++ + } + } + } + + /* 3) Let's replace those population counts that lead to more RLE codes. + Math here is in 24.8 fixed point representation. */ + stride = 0 + + limit = uint(256*(counts[0]+counts[1]+counts[2])/3 + 420) + sum = 0 + for i = 0; i <= length; i++ { + if i == length || good_for_rle[i] != 0 || (i != 0 && good_for_rle[i-1] != 0) || (256*counts[i]-uint32(limit)+uint32(streak_limit)) >= uint32(2*streak_limit) { + if stride >= 4 || (stride >= 3 && sum == 0) { + var k uint + var count uint = (sum + stride/2) / stride + /* The stride must end, collapse what we have, if we have enough (4). */ + if count == 0 { + count = 1 + } + + if sum == 0 { + /* Don't make an all zeros stride to be upgraded to ones. */ + count = 0 + } + + for k = 0; k < stride; k++ { + /* We don't want to change value at counts[i], + that is already belonging to the next stride. Thus - 1. */ + counts[i-k-1] = uint32(count) + } + } + + stride = 0 + sum = 0 + if i < length-2 { + /* All interesting strides have a count of at least 4, */ + /* at least when non-zeros. */ + limit = uint(256*(counts[i]+counts[i+1]+counts[i+2])/3 + 420) + } else if i < length { + limit = uint(256 * counts[i]) + } else { + limit = 0 + } + } + + stride++ + if i != length { + sum += uint(counts[i]) + if stride >= 4 { + limit = (256*sum + stride/2) / stride + } + + if stride == 4 { + limit += 120 + } + } + } +} + +func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) { + var total_reps_zero uint = 0 + var total_reps_non_zero uint = 0 + var count_reps_zero uint = 1 + var count_reps_non_zero uint = 1 + var i uint + for i = 0; i < length; { + var value byte = depth[i] + var reps uint = 1 + var k uint + for k = i + 1; k < length && depth[k] == value; k++ { + reps++ + } + + if reps >= 3 && value == 0 { + total_reps_zero += reps + count_reps_zero++ + } + + if reps >= 4 && value != 0 { + total_reps_non_zero += reps + count_reps_non_zero++ + } + + i += reps + } + + *use_rle_for_non_zero = total_reps_non_zero > count_reps_non_zero*2 + *use_rle_for_zero = total_reps_zero > count_reps_zero*2 +} + +/* Write a Huffman tree from bit depths into the bit-stream representation + of a Huffman tree. The generated Huffman tree is to be compressed once + more using a Huffman tree */ +func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) { + var previous_value byte = initialRepeatedCodeLength + var i uint + var use_rle_for_non_zero bool = false + var use_rle_for_zero bool = false + var new_length uint = length + /* Throw away trailing zeros. */ + for i = 0; i < length; i++ { + if depth[length-i-1] == 0 { + new_length-- + } else { + break + } + } + + /* First gather statistics on if it is a good idea to do RLE. */ + if length > 50 { + /* Find RLE coding for longer codes. + Shorter codes seem not to benefit from RLE. */ + decideOverRLEUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero) + } + + /* Actual RLE coding. */ + for i = 0; i < new_length; { + var value byte = depth[i] + var reps uint = 1 + if (value != 0 && use_rle_for_non_zero) || (value == 0 && use_rle_for_zero) { + var k uint + for k = i + 1; k < new_length && depth[k] == value; k++ { + reps++ + } + } + + if value == 0 { + writeHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data) + } else { + writeHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data) + previous_value = value + } + + i += reps + } +} + +var reverseBits_kLut = [16]uint{ + 0x00, + 0x08, + 0x04, + 0x0C, + 0x02, + 0x0A, + 0x06, + 0x0E, + 0x01, + 0x09, + 0x05, + 0x0D, + 0x03, + 0x0B, + 0x07, + 0x0F, +} + +func reverseBits(num_bits uint, bits uint16) uint16 { + var retval uint = reverseBits_kLut[bits&0x0F] + var i uint + for i = 4; i < num_bits; i += 4 { + retval <<= 4 + bits = uint16(bits >> 4) + retval |= reverseBits_kLut[bits&0x0F] + } + + retval >>= ((0 - num_bits) & 0x03) + return uint16(retval) +} + +/* 0..15 are values for bits */ +const maxHuffmanBits = 16 + +/* Get the actual bit values for a tree of bit depths. */ +func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) { + var bl_count = [maxHuffmanBits]uint16{0} + var next_code [maxHuffmanBits]uint16 + var i uint + /* In Brotli, all bit depths are [1..15] + 0 bit depth means that the symbol does not exist. */ + + var code int = 0 + for i = 0; i < len; i++ { + bl_count[depth[i]]++ + } + + bl_count[0] = 0 + next_code[0] = 0 + for i = 1; i < maxHuffmanBits; i++ { + code = (code + int(bl_count[i-1])) << 1 + next_code[i] = uint16(code) + } + + for i = 0; i < len; i++ { + if depth[i] != 0 { + bits[i] = reverseBits(uint(depth[i]), next_code[depth[i]]) + next_code[depth[i]]++ + } + } +} diff --git a/vendor/github.com/andybalholm/brotli/entropy_encode_static.go b/vendor/github.com/andybalholm/brotli/entropy_encode_static.go new file mode 100644 index 0000000000..294aff4f4e --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/entropy_encode_static.go @@ -0,0 +1,4399 @@ +package brotli + +var kCodeLengthDepth = [18]byte{4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 0, 4, 4} + +var kStaticCommandCodeDepth = [numCommandSymbols]byte{ + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 9, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, + 11, +} + +var kStaticDistanceCodeDepth = [64]byte{ + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, + 6, +} + +var kCodeLengthBits = [18]uint32{0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7} + +func storeStaticCodeLengthCode(storage_ix *uint, storage []byte) { + writeBits(40, 0x0000FF55555554, storage_ix, storage) +} + +func storeStaticCodeLengthCodeBW(bw *bitWriter) { + bw.writeBits(32, 0x55555554) + bw.writeBits(8, 0xFF) +} + +var kZeroRepsBits = [numCommandSymbols]uint64{ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000007, + 0x00000017, + 0x00000027, + 0x00000037, + 0x00000047, + 0x00000057, + 0x00000067, + 0x00000077, + 0x00000770, + 0x00000b87, + 0x00001387, + 0x00001b87, + 0x00002387, + 0x00002b87, + 0x00003387, + 0x00003b87, + 0x00000397, + 0x00000b97, + 0x00001397, + 0x00001b97, + 0x00002397, + 0x00002b97, + 0x00003397, + 0x00003b97, + 0x000003a7, + 0x00000ba7, + 0x000013a7, + 0x00001ba7, + 0x000023a7, + 0x00002ba7, + 0x000033a7, + 0x00003ba7, + 0x000003b7, + 0x00000bb7, + 0x000013b7, + 0x00001bb7, + 0x000023b7, + 0x00002bb7, + 0x000033b7, + 0x00003bb7, + 0x000003c7, + 0x00000bc7, + 0x000013c7, + 0x00001bc7, + 0x000023c7, + 0x00002bc7, + 0x000033c7, + 0x00003bc7, + 0x000003d7, + 0x00000bd7, + 0x000013d7, + 0x00001bd7, + 0x000023d7, + 0x00002bd7, + 0x000033d7, + 0x00003bd7, + 0x000003e7, + 0x00000be7, + 0x000013e7, + 0x00001be7, + 0x000023e7, + 0x00002be7, + 0x000033e7, + 0x00003be7, + 0x000003f7, + 0x00000bf7, + 0x000013f7, + 0x00001bf7, + 0x000023f7, + 0x00002bf7, + 0x000033f7, + 0x00003bf7, + 0x0001c387, + 0x0005c387, + 0x0009c387, + 0x000dc387, + 0x0011c387, + 0x0015c387, + 0x0019c387, + 0x001dc387, + 0x0001cb87, + 0x0005cb87, + 0x0009cb87, + 0x000dcb87, + 0x0011cb87, + 0x0015cb87, + 0x0019cb87, + 0x001dcb87, + 0x0001d387, + 0x0005d387, + 0x0009d387, + 0x000dd387, + 0x0011d387, + 0x0015d387, + 0x0019d387, + 0x001dd387, + 0x0001db87, + 0x0005db87, + 0x0009db87, + 0x000ddb87, + 0x0011db87, + 0x0015db87, + 0x0019db87, + 0x001ddb87, + 0x0001e387, + 0x0005e387, + 0x0009e387, + 0x000de387, + 0x0011e387, + 0x0015e387, + 0x0019e387, + 0x001de387, + 0x0001eb87, + 0x0005eb87, + 0x0009eb87, + 0x000deb87, + 0x0011eb87, + 0x0015eb87, + 0x0019eb87, + 0x001deb87, + 0x0001f387, + 0x0005f387, + 0x0009f387, + 0x000df387, + 0x0011f387, + 0x0015f387, + 0x0019f387, + 0x001df387, + 0x0001fb87, + 0x0005fb87, + 0x0009fb87, + 0x000dfb87, + 0x0011fb87, + 0x0015fb87, + 0x0019fb87, + 0x001dfb87, + 0x0001c397, + 0x0005c397, + 0x0009c397, + 0x000dc397, + 0x0011c397, + 0x0015c397, + 0x0019c397, + 0x001dc397, + 0x0001cb97, + 0x0005cb97, + 0x0009cb97, + 0x000dcb97, + 0x0011cb97, + 0x0015cb97, + 0x0019cb97, + 0x001dcb97, + 0x0001d397, + 0x0005d397, + 0x0009d397, + 0x000dd397, + 0x0011d397, + 0x0015d397, + 0x0019d397, + 0x001dd397, + 0x0001db97, + 0x0005db97, + 0x0009db97, + 0x000ddb97, + 0x0011db97, + 0x0015db97, + 0x0019db97, + 0x001ddb97, + 0x0001e397, + 0x0005e397, + 0x0009e397, + 0x000de397, + 0x0011e397, + 0x0015e397, + 0x0019e397, + 0x001de397, + 0x0001eb97, + 0x0005eb97, + 0x0009eb97, + 0x000deb97, + 0x0011eb97, + 0x0015eb97, + 0x0019eb97, + 0x001deb97, + 0x0001f397, + 0x0005f397, + 0x0009f397, + 0x000df397, + 0x0011f397, + 0x0015f397, + 0x0019f397, + 0x001df397, + 0x0001fb97, + 0x0005fb97, + 0x0009fb97, + 0x000dfb97, + 0x0011fb97, + 0x0015fb97, + 0x0019fb97, + 0x001dfb97, + 0x0001c3a7, + 0x0005c3a7, + 0x0009c3a7, + 0x000dc3a7, + 0x0011c3a7, + 0x0015c3a7, + 0x0019c3a7, + 0x001dc3a7, + 0x0001cba7, + 0x0005cba7, + 0x0009cba7, + 0x000dcba7, + 0x0011cba7, + 0x0015cba7, + 0x0019cba7, + 0x001dcba7, + 0x0001d3a7, + 0x0005d3a7, + 0x0009d3a7, + 0x000dd3a7, + 0x0011d3a7, + 0x0015d3a7, + 0x0019d3a7, + 0x001dd3a7, + 0x0001dba7, + 0x0005dba7, + 0x0009dba7, + 0x000ddba7, + 0x0011dba7, + 0x0015dba7, + 0x0019dba7, + 0x001ddba7, + 0x0001e3a7, + 0x0005e3a7, + 0x0009e3a7, + 0x000de3a7, + 0x0011e3a7, + 0x0015e3a7, + 0x0019e3a7, + 0x001de3a7, + 0x0001eba7, + 0x0005eba7, + 0x0009eba7, + 0x000deba7, + 0x0011eba7, + 0x0015eba7, + 0x0019eba7, + 0x001deba7, + 0x0001f3a7, + 0x0005f3a7, + 0x0009f3a7, + 0x000df3a7, + 0x0011f3a7, + 0x0015f3a7, + 0x0019f3a7, + 0x001df3a7, + 0x0001fba7, + 0x0005fba7, + 0x0009fba7, + 0x000dfba7, + 0x0011fba7, + 0x0015fba7, + 0x0019fba7, + 0x001dfba7, + 0x0001c3b7, + 0x0005c3b7, + 0x0009c3b7, + 0x000dc3b7, + 0x0011c3b7, + 0x0015c3b7, + 0x0019c3b7, + 0x001dc3b7, + 0x0001cbb7, + 0x0005cbb7, + 0x0009cbb7, + 0x000dcbb7, + 0x0011cbb7, + 0x0015cbb7, + 0x0019cbb7, + 0x001dcbb7, + 0x0001d3b7, + 0x0005d3b7, + 0x0009d3b7, + 0x000dd3b7, + 0x0011d3b7, + 0x0015d3b7, + 0x0019d3b7, + 0x001dd3b7, + 0x0001dbb7, + 0x0005dbb7, + 0x0009dbb7, + 0x000ddbb7, + 0x0011dbb7, + 0x0015dbb7, + 0x0019dbb7, + 0x001ddbb7, + 0x0001e3b7, + 0x0005e3b7, + 0x0009e3b7, + 0x000de3b7, + 0x0011e3b7, + 0x0015e3b7, + 0x0019e3b7, + 0x001de3b7, + 0x0001ebb7, + 0x0005ebb7, + 0x0009ebb7, + 0x000debb7, + 0x0011ebb7, + 0x0015ebb7, + 0x0019ebb7, + 0x001debb7, + 0x0001f3b7, + 0x0005f3b7, + 0x0009f3b7, + 0x000df3b7, + 0x0011f3b7, + 0x0015f3b7, + 0x0019f3b7, + 0x001df3b7, + 0x0001fbb7, + 0x0005fbb7, + 0x0009fbb7, + 0x000dfbb7, + 0x0011fbb7, + 0x0015fbb7, + 0x0019fbb7, + 0x001dfbb7, + 0x0001c3c7, + 0x0005c3c7, + 0x0009c3c7, + 0x000dc3c7, + 0x0011c3c7, + 0x0015c3c7, + 0x0019c3c7, + 0x001dc3c7, + 0x0001cbc7, + 0x0005cbc7, + 0x0009cbc7, + 0x000dcbc7, + 0x0011cbc7, + 0x0015cbc7, + 0x0019cbc7, + 0x001dcbc7, + 0x0001d3c7, + 0x0005d3c7, + 0x0009d3c7, + 0x000dd3c7, + 0x0011d3c7, + 0x0015d3c7, + 0x0019d3c7, + 0x001dd3c7, + 0x0001dbc7, + 0x0005dbc7, + 0x0009dbc7, + 0x000ddbc7, + 0x0011dbc7, + 0x0015dbc7, + 0x0019dbc7, + 0x001ddbc7, + 0x0001e3c7, + 0x0005e3c7, + 0x0009e3c7, + 0x000de3c7, + 0x0011e3c7, + 0x0015e3c7, + 0x0019e3c7, + 0x001de3c7, + 0x0001ebc7, + 0x0005ebc7, + 0x0009ebc7, + 0x000debc7, + 0x0011ebc7, + 0x0015ebc7, + 0x0019ebc7, + 0x001debc7, + 0x0001f3c7, + 0x0005f3c7, + 0x0009f3c7, + 0x000df3c7, + 0x0011f3c7, + 0x0015f3c7, + 0x0019f3c7, + 0x001df3c7, + 0x0001fbc7, + 0x0005fbc7, + 0x0009fbc7, + 0x000dfbc7, + 0x0011fbc7, + 0x0015fbc7, + 0x0019fbc7, + 0x001dfbc7, + 0x0001c3d7, + 0x0005c3d7, + 0x0009c3d7, + 0x000dc3d7, + 0x0011c3d7, + 0x0015c3d7, + 0x0019c3d7, + 0x001dc3d7, + 0x0001cbd7, + 0x0005cbd7, + 0x0009cbd7, + 0x000dcbd7, + 0x0011cbd7, + 0x0015cbd7, + 0x0019cbd7, + 0x001dcbd7, + 0x0001d3d7, + 0x0005d3d7, + 0x0009d3d7, + 0x000dd3d7, + 0x0011d3d7, + 0x0015d3d7, + 0x0019d3d7, + 0x001dd3d7, + 0x0001dbd7, + 0x0005dbd7, + 0x0009dbd7, + 0x000ddbd7, + 0x0011dbd7, + 0x0015dbd7, + 0x0019dbd7, + 0x001ddbd7, + 0x0001e3d7, + 0x0005e3d7, + 0x0009e3d7, + 0x000de3d7, + 0x0011e3d7, + 0x0015e3d7, + 0x0019e3d7, + 0x001de3d7, + 0x0001ebd7, + 0x0005ebd7, + 0x0009ebd7, + 0x000debd7, + 0x0011ebd7, + 0x0015ebd7, + 0x0019ebd7, + 0x001debd7, + 0x0001f3d7, + 0x0005f3d7, + 0x0009f3d7, + 0x000df3d7, + 0x0011f3d7, + 0x0015f3d7, + 0x0019f3d7, + 0x001df3d7, + 0x0001fbd7, + 0x0005fbd7, + 0x0009fbd7, + 0x000dfbd7, + 0x0011fbd7, + 0x0015fbd7, + 0x0019fbd7, + 0x001dfbd7, + 0x0001c3e7, + 0x0005c3e7, + 0x0009c3e7, + 0x000dc3e7, + 0x0011c3e7, + 0x0015c3e7, + 0x0019c3e7, + 0x001dc3e7, + 0x0001cbe7, + 0x0005cbe7, + 0x0009cbe7, + 0x000dcbe7, + 0x0011cbe7, + 0x0015cbe7, + 0x0019cbe7, + 0x001dcbe7, + 0x0001d3e7, + 0x0005d3e7, + 0x0009d3e7, + 0x000dd3e7, + 0x0011d3e7, + 0x0015d3e7, + 0x0019d3e7, + 0x001dd3e7, + 0x0001dbe7, + 0x0005dbe7, + 0x0009dbe7, + 0x000ddbe7, + 0x0011dbe7, + 0x0015dbe7, + 0x0019dbe7, + 0x001ddbe7, + 0x0001e3e7, + 0x0005e3e7, + 0x0009e3e7, + 0x000de3e7, + 0x0011e3e7, + 0x0015e3e7, + 0x0019e3e7, + 0x001de3e7, + 0x0001ebe7, + 0x0005ebe7, + 0x0009ebe7, + 0x000debe7, + 0x0011ebe7, + 0x0015ebe7, + 0x0019ebe7, + 0x001debe7, + 0x0001f3e7, + 0x0005f3e7, + 0x0009f3e7, + 0x000df3e7, + 0x0011f3e7, + 0x0015f3e7, + 0x0019f3e7, + 0x001df3e7, + 0x0001fbe7, + 0x0005fbe7, + 0x0009fbe7, + 0x000dfbe7, + 0x0011fbe7, + 0x0015fbe7, + 0x0019fbe7, + 0x001dfbe7, + 0x0001c3f7, + 0x0005c3f7, + 0x0009c3f7, + 0x000dc3f7, + 0x0011c3f7, + 0x0015c3f7, + 0x0019c3f7, + 0x001dc3f7, + 0x0001cbf7, + 0x0005cbf7, + 0x0009cbf7, + 0x000dcbf7, + 0x0011cbf7, + 0x0015cbf7, + 0x0019cbf7, + 0x001dcbf7, + 0x0001d3f7, + 0x0005d3f7, + 0x0009d3f7, + 0x000dd3f7, + 0x0011d3f7, + 0x0015d3f7, + 0x0019d3f7, + 0x001dd3f7, + 0x0001dbf7, + 0x0005dbf7, + 0x0009dbf7, + 0x000ddbf7, + 0x0011dbf7, + 0x0015dbf7, + 0x0019dbf7, + 0x001ddbf7, + 0x0001e3f7, + 0x0005e3f7, + 0x0009e3f7, + 0x000de3f7, + 0x0011e3f7, + 0x0015e3f7, + 0x0019e3f7, + 0x001de3f7, + 0x0001ebf7, + 0x0005ebf7, + 0x0009ebf7, + 0x000debf7, + 0x0011ebf7, + 0x0015ebf7, + 0x0019ebf7, + 0x001debf7, + 0x0001f3f7, + 0x0005f3f7, + 0x0009f3f7, + 0x000df3f7, + 0x0011f3f7, + 0x0015f3f7, + 0x0019f3f7, + 0x001df3f7, + 0x0001fbf7, + 0x0005fbf7, + 0x0009fbf7, + 0x000dfbf7, + 0x0011fbf7, + 0x0015fbf7, + 0x0019fbf7, + 0x001dfbf7, + 0x00e1c387, + 0x02e1c387, + 0x04e1c387, + 0x06e1c387, + 0x08e1c387, + 0x0ae1c387, + 0x0ce1c387, + 0x0ee1c387, + 0x00e5c387, + 0x02e5c387, + 0x04e5c387, + 0x06e5c387, + 0x08e5c387, + 0x0ae5c387, + 0x0ce5c387, + 0x0ee5c387, + 0x00e9c387, + 0x02e9c387, + 0x04e9c387, + 0x06e9c387, + 0x08e9c387, + 0x0ae9c387, + 0x0ce9c387, + 0x0ee9c387, + 0x00edc387, + 0x02edc387, + 0x04edc387, + 0x06edc387, + 0x08edc387, + 0x0aedc387, + 0x0cedc387, + 0x0eedc387, + 0x00f1c387, + 0x02f1c387, + 0x04f1c387, + 0x06f1c387, + 0x08f1c387, + 0x0af1c387, + 0x0cf1c387, + 0x0ef1c387, + 0x00f5c387, + 0x02f5c387, + 0x04f5c387, + 0x06f5c387, + 0x08f5c387, + 0x0af5c387, + 0x0cf5c387, + 0x0ef5c387, + 0x00f9c387, + 0x02f9c387, + 0x04f9c387, + 0x06f9c387, + 0x08f9c387, + 0x0af9c387, + 0x0cf9c387, + 0x0ef9c387, + 0x00fdc387, + 0x02fdc387, + 0x04fdc387, + 0x06fdc387, + 0x08fdc387, + 0x0afdc387, + 0x0cfdc387, + 0x0efdc387, + 0x00e1cb87, + 0x02e1cb87, + 0x04e1cb87, + 0x06e1cb87, + 0x08e1cb87, + 0x0ae1cb87, + 0x0ce1cb87, + 0x0ee1cb87, + 0x00e5cb87, + 0x02e5cb87, + 0x04e5cb87, + 0x06e5cb87, + 0x08e5cb87, + 0x0ae5cb87, + 0x0ce5cb87, + 0x0ee5cb87, + 0x00e9cb87, + 0x02e9cb87, + 0x04e9cb87, + 0x06e9cb87, + 0x08e9cb87, + 0x0ae9cb87, + 0x0ce9cb87, + 0x0ee9cb87, + 0x00edcb87, + 0x02edcb87, + 0x04edcb87, + 0x06edcb87, + 0x08edcb87, + 0x0aedcb87, + 0x0cedcb87, + 0x0eedcb87, + 0x00f1cb87, + 0x02f1cb87, + 0x04f1cb87, + 0x06f1cb87, + 0x08f1cb87, + 0x0af1cb87, + 0x0cf1cb87, + 0x0ef1cb87, + 0x00f5cb87, + 0x02f5cb87, + 0x04f5cb87, + 0x06f5cb87, + 0x08f5cb87, + 0x0af5cb87, + 0x0cf5cb87, + 0x0ef5cb87, + 0x00f9cb87, + 0x02f9cb87, + 0x04f9cb87, + 0x06f9cb87, + 0x08f9cb87, +} + +var kZeroRepsDepth = [numCommandSymbols]uint32{ + 0, + 4, + 8, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 7, + 11, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 14, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 21, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, + 28, +} + +var kNonZeroRepsBits = [numCommandSymbols]uint64{ + 0x0000000b, + 0x0000001b, + 0x0000002b, + 0x0000003b, + 0x000002cb, + 0x000006cb, + 0x00000acb, + 0x00000ecb, + 0x000002db, + 0x000006db, + 0x00000adb, + 0x00000edb, + 0x000002eb, + 0x000006eb, + 0x00000aeb, + 0x00000eeb, + 0x000002fb, + 0x000006fb, + 0x00000afb, + 0x00000efb, + 0x0000b2cb, + 0x0001b2cb, + 0x0002b2cb, + 0x0003b2cb, + 0x0000b6cb, + 0x0001b6cb, + 0x0002b6cb, + 0x0003b6cb, + 0x0000bacb, + 0x0001bacb, + 0x0002bacb, + 0x0003bacb, + 0x0000becb, + 0x0001becb, + 0x0002becb, + 0x0003becb, + 0x0000b2db, + 0x0001b2db, + 0x0002b2db, + 0x0003b2db, + 0x0000b6db, + 0x0001b6db, + 0x0002b6db, + 0x0003b6db, + 0x0000badb, + 0x0001badb, + 0x0002badb, + 0x0003badb, + 0x0000bedb, + 0x0001bedb, + 0x0002bedb, + 0x0003bedb, + 0x0000b2eb, + 0x0001b2eb, + 0x0002b2eb, + 0x0003b2eb, + 0x0000b6eb, + 0x0001b6eb, + 0x0002b6eb, + 0x0003b6eb, + 0x0000baeb, + 0x0001baeb, + 0x0002baeb, + 0x0003baeb, + 0x0000beeb, + 0x0001beeb, + 0x0002beeb, + 0x0003beeb, + 0x0000b2fb, + 0x0001b2fb, + 0x0002b2fb, + 0x0003b2fb, + 0x0000b6fb, + 0x0001b6fb, + 0x0002b6fb, + 0x0003b6fb, + 0x0000bafb, + 0x0001bafb, + 0x0002bafb, + 0x0003bafb, + 0x0000befb, + 0x0001befb, + 0x0002befb, + 0x0003befb, + 0x002cb2cb, + 0x006cb2cb, + 0x00acb2cb, + 0x00ecb2cb, + 0x002db2cb, + 0x006db2cb, + 0x00adb2cb, + 0x00edb2cb, + 0x002eb2cb, + 0x006eb2cb, + 0x00aeb2cb, + 0x00eeb2cb, + 0x002fb2cb, + 0x006fb2cb, + 0x00afb2cb, + 0x00efb2cb, + 0x002cb6cb, + 0x006cb6cb, + 0x00acb6cb, + 0x00ecb6cb, + 0x002db6cb, + 0x006db6cb, + 0x00adb6cb, + 0x00edb6cb, + 0x002eb6cb, + 0x006eb6cb, + 0x00aeb6cb, + 0x00eeb6cb, + 0x002fb6cb, + 0x006fb6cb, + 0x00afb6cb, + 0x00efb6cb, + 0x002cbacb, + 0x006cbacb, + 0x00acbacb, + 0x00ecbacb, + 0x002dbacb, + 0x006dbacb, + 0x00adbacb, + 0x00edbacb, + 0x002ebacb, + 0x006ebacb, + 0x00aebacb, + 0x00eebacb, + 0x002fbacb, + 0x006fbacb, + 0x00afbacb, + 0x00efbacb, + 0x002cbecb, + 0x006cbecb, + 0x00acbecb, + 0x00ecbecb, + 0x002dbecb, + 0x006dbecb, + 0x00adbecb, + 0x00edbecb, + 0x002ebecb, + 0x006ebecb, + 0x00aebecb, + 0x00eebecb, + 0x002fbecb, + 0x006fbecb, + 0x00afbecb, + 0x00efbecb, + 0x002cb2db, + 0x006cb2db, + 0x00acb2db, + 0x00ecb2db, + 0x002db2db, + 0x006db2db, + 0x00adb2db, + 0x00edb2db, + 0x002eb2db, + 0x006eb2db, + 0x00aeb2db, + 0x00eeb2db, + 0x002fb2db, + 0x006fb2db, + 0x00afb2db, + 0x00efb2db, + 0x002cb6db, + 0x006cb6db, + 0x00acb6db, + 0x00ecb6db, + 0x002db6db, + 0x006db6db, + 0x00adb6db, + 0x00edb6db, + 0x002eb6db, + 0x006eb6db, + 0x00aeb6db, + 0x00eeb6db, + 0x002fb6db, + 0x006fb6db, + 0x00afb6db, + 0x00efb6db, + 0x002cbadb, + 0x006cbadb, + 0x00acbadb, + 0x00ecbadb, + 0x002dbadb, + 0x006dbadb, + 0x00adbadb, + 0x00edbadb, + 0x002ebadb, + 0x006ebadb, + 0x00aebadb, + 0x00eebadb, + 0x002fbadb, + 0x006fbadb, + 0x00afbadb, + 0x00efbadb, + 0x002cbedb, + 0x006cbedb, + 0x00acbedb, + 0x00ecbedb, + 0x002dbedb, + 0x006dbedb, + 0x00adbedb, + 0x00edbedb, + 0x002ebedb, + 0x006ebedb, + 0x00aebedb, + 0x00eebedb, + 0x002fbedb, + 0x006fbedb, + 0x00afbedb, + 0x00efbedb, + 0x002cb2eb, + 0x006cb2eb, + 0x00acb2eb, + 0x00ecb2eb, + 0x002db2eb, + 0x006db2eb, + 0x00adb2eb, + 0x00edb2eb, + 0x002eb2eb, + 0x006eb2eb, + 0x00aeb2eb, + 0x00eeb2eb, + 0x002fb2eb, + 0x006fb2eb, + 0x00afb2eb, + 0x00efb2eb, + 0x002cb6eb, + 0x006cb6eb, + 0x00acb6eb, + 0x00ecb6eb, + 0x002db6eb, + 0x006db6eb, + 0x00adb6eb, + 0x00edb6eb, + 0x002eb6eb, + 0x006eb6eb, + 0x00aeb6eb, + 0x00eeb6eb, + 0x002fb6eb, + 0x006fb6eb, + 0x00afb6eb, + 0x00efb6eb, + 0x002cbaeb, + 0x006cbaeb, + 0x00acbaeb, + 0x00ecbaeb, + 0x002dbaeb, + 0x006dbaeb, + 0x00adbaeb, + 0x00edbaeb, + 0x002ebaeb, + 0x006ebaeb, + 0x00aebaeb, + 0x00eebaeb, + 0x002fbaeb, + 0x006fbaeb, + 0x00afbaeb, + 0x00efbaeb, + 0x002cbeeb, + 0x006cbeeb, + 0x00acbeeb, + 0x00ecbeeb, + 0x002dbeeb, + 0x006dbeeb, + 0x00adbeeb, + 0x00edbeeb, + 0x002ebeeb, + 0x006ebeeb, + 0x00aebeeb, + 0x00eebeeb, + 0x002fbeeb, + 0x006fbeeb, + 0x00afbeeb, + 0x00efbeeb, + 0x002cb2fb, + 0x006cb2fb, + 0x00acb2fb, + 0x00ecb2fb, + 0x002db2fb, + 0x006db2fb, + 0x00adb2fb, + 0x00edb2fb, + 0x002eb2fb, + 0x006eb2fb, + 0x00aeb2fb, + 0x00eeb2fb, + 0x002fb2fb, + 0x006fb2fb, + 0x00afb2fb, + 0x00efb2fb, + 0x002cb6fb, + 0x006cb6fb, + 0x00acb6fb, + 0x00ecb6fb, + 0x002db6fb, + 0x006db6fb, + 0x00adb6fb, + 0x00edb6fb, + 0x002eb6fb, + 0x006eb6fb, + 0x00aeb6fb, + 0x00eeb6fb, + 0x002fb6fb, + 0x006fb6fb, + 0x00afb6fb, + 0x00efb6fb, + 0x002cbafb, + 0x006cbafb, + 0x00acbafb, + 0x00ecbafb, + 0x002dbafb, + 0x006dbafb, + 0x00adbafb, + 0x00edbafb, + 0x002ebafb, + 0x006ebafb, + 0x00aebafb, + 0x00eebafb, + 0x002fbafb, + 0x006fbafb, + 0x00afbafb, + 0x00efbafb, + 0x002cbefb, + 0x006cbefb, + 0x00acbefb, + 0x00ecbefb, + 0x002dbefb, + 0x006dbefb, + 0x00adbefb, + 0x00edbefb, + 0x002ebefb, + 0x006ebefb, + 0x00aebefb, + 0x00eebefb, + 0x002fbefb, + 0x006fbefb, + 0x00afbefb, + 0x00efbefb, + 0x0b2cb2cb, + 0x1b2cb2cb, + 0x2b2cb2cb, + 0x3b2cb2cb, + 0x0b6cb2cb, + 0x1b6cb2cb, + 0x2b6cb2cb, + 0x3b6cb2cb, + 0x0bacb2cb, + 0x1bacb2cb, + 0x2bacb2cb, + 0x3bacb2cb, + 0x0becb2cb, + 0x1becb2cb, + 0x2becb2cb, + 0x3becb2cb, + 0x0b2db2cb, + 0x1b2db2cb, + 0x2b2db2cb, + 0x3b2db2cb, + 0x0b6db2cb, + 0x1b6db2cb, + 0x2b6db2cb, + 0x3b6db2cb, + 0x0badb2cb, + 0x1badb2cb, + 0x2badb2cb, + 0x3badb2cb, + 0x0bedb2cb, + 0x1bedb2cb, + 0x2bedb2cb, + 0x3bedb2cb, + 0x0b2eb2cb, + 0x1b2eb2cb, + 0x2b2eb2cb, + 0x3b2eb2cb, + 0x0b6eb2cb, + 0x1b6eb2cb, + 0x2b6eb2cb, + 0x3b6eb2cb, + 0x0baeb2cb, + 0x1baeb2cb, + 0x2baeb2cb, + 0x3baeb2cb, + 0x0beeb2cb, + 0x1beeb2cb, + 0x2beeb2cb, + 0x3beeb2cb, + 0x0b2fb2cb, + 0x1b2fb2cb, + 0x2b2fb2cb, + 0x3b2fb2cb, + 0x0b6fb2cb, + 0x1b6fb2cb, + 0x2b6fb2cb, + 0x3b6fb2cb, + 0x0bafb2cb, + 0x1bafb2cb, + 0x2bafb2cb, + 0x3bafb2cb, + 0x0befb2cb, + 0x1befb2cb, + 0x2befb2cb, + 0x3befb2cb, + 0x0b2cb6cb, + 0x1b2cb6cb, + 0x2b2cb6cb, + 0x3b2cb6cb, + 0x0b6cb6cb, + 0x1b6cb6cb, + 0x2b6cb6cb, + 0x3b6cb6cb, + 0x0bacb6cb, + 0x1bacb6cb, + 0x2bacb6cb, + 0x3bacb6cb, + 0x0becb6cb, + 0x1becb6cb, + 0x2becb6cb, + 0x3becb6cb, + 0x0b2db6cb, + 0x1b2db6cb, + 0x2b2db6cb, + 0x3b2db6cb, + 0x0b6db6cb, + 0x1b6db6cb, + 0x2b6db6cb, + 0x3b6db6cb, + 0x0badb6cb, + 0x1badb6cb, + 0x2badb6cb, + 0x3badb6cb, + 0x0bedb6cb, + 0x1bedb6cb, + 0x2bedb6cb, + 0x3bedb6cb, + 0x0b2eb6cb, + 0x1b2eb6cb, + 0x2b2eb6cb, + 0x3b2eb6cb, + 0x0b6eb6cb, + 0x1b6eb6cb, + 0x2b6eb6cb, + 0x3b6eb6cb, + 0x0baeb6cb, + 0x1baeb6cb, + 0x2baeb6cb, + 0x3baeb6cb, + 0x0beeb6cb, + 0x1beeb6cb, + 0x2beeb6cb, + 0x3beeb6cb, + 0x0b2fb6cb, + 0x1b2fb6cb, + 0x2b2fb6cb, + 0x3b2fb6cb, + 0x0b6fb6cb, + 0x1b6fb6cb, + 0x2b6fb6cb, + 0x3b6fb6cb, + 0x0bafb6cb, + 0x1bafb6cb, + 0x2bafb6cb, + 0x3bafb6cb, + 0x0befb6cb, + 0x1befb6cb, + 0x2befb6cb, + 0x3befb6cb, + 0x0b2cbacb, + 0x1b2cbacb, + 0x2b2cbacb, + 0x3b2cbacb, + 0x0b6cbacb, + 0x1b6cbacb, + 0x2b6cbacb, + 0x3b6cbacb, + 0x0bacbacb, + 0x1bacbacb, + 0x2bacbacb, + 0x3bacbacb, + 0x0becbacb, + 0x1becbacb, + 0x2becbacb, + 0x3becbacb, + 0x0b2dbacb, + 0x1b2dbacb, + 0x2b2dbacb, + 0x3b2dbacb, + 0x0b6dbacb, + 0x1b6dbacb, + 0x2b6dbacb, + 0x3b6dbacb, + 0x0badbacb, + 0x1badbacb, + 0x2badbacb, + 0x3badbacb, + 0x0bedbacb, + 0x1bedbacb, + 0x2bedbacb, + 0x3bedbacb, + 0x0b2ebacb, + 0x1b2ebacb, + 0x2b2ebacb, + 0x3b2ebacb, + 0x0b6ebacb, + 0x1b6ebacb, + 0x2b6ebacb, + 0x3b6ebacb, + 0x0baebacb, + 0x1baebacb, + 0x2baebacb, + 0x3baebacb, + 0x0beebacb, + 0x1beebacb, + 0x2beebacb, + 0x3beebacb, + 0x0b2fbacb, + 0x1b2fbacb, + 0x2b2fbacb, + 0x3b2fbacb, + 0x0b6fbacb, + 0x1b6fbacb, + 0x2b6fbacb, + 0x3b6fbacb, + 0x0bafbacb, + 0x1bafbacb, + 0x2bafbacb, + 0x3bafbacb, + 0x0befbacb, + 0x1befbacb, + 0x2befbacb, + 0x3befbacb, + 0x0b2cbecb, + 0x1b2cbecb, + 0x2b2cbecb, + 0x3b2cbecb, + 0x0b6cbecb, + 0x1b6cbecb, + 0x2b6cbecb, + 0x3b6cbecb, + 0x0bacbecb, + 0x1bacbecb, + 0x2bacbecb, + 0x3bacbecb, + 0x0becbecb, + 0x1becbecb, + 0x2becbecb, + 0x3becbecb, + 0x0b2dbecb, + 0x1b2dbecb, + 0x2b2dbecb, + 0x3b2dbecb, + 0x0b6dbecb, + 0x1b6dbecb, + 0x2b6dbecb, + 0x3b6dbecb, + 0x0badbecb, + 0x1badbecb, + 0x2badbecb, + 0x3badbecb, + 0x0bedbecb, + 0x1bedbecb, + 0x2bedbecb, + 0x3bedbecb, + 0x0b2ebecb, + 0x1b2ebecb, + 0x2b2ebecb, + 0x3b2ebecb, + 0x0b6ebecb, + 0x1b6ebecb, + 0x2b6ebecb, + 0x3b6ebecb, + 0x0baebecb, + 0x1baebecb, + 0x2baebecb, + 0x3baebecb, + 0x0beebecb, + 0x1beebecb, + 0x2beebecb, + 0x3beebecb, + 0x0b2fbecb, + 0x1b2fbecb, + 0x2b2fbecb, + 0x3b2fbecb, + 0x0b6fbecb, + 0x1b6fbecb, + 0x2b6fbecb, + 0x3b6fbecb, + 0x0bafbecb, + 0x1bafbecb, + 0x2bafbecb, + 0x3bafbecb, + 0x0befbecb, + 0x1befbecb, + 0x2befbecb, + 0x3befbecb, + 0x0b2cb2db, + 0x1b2cb2db, + 0x2b2cb2db, + 0x3b2cb2db, + 0x0b6cb2db, + 0x1b6cb2db, + 0x2b6cb2db, + 0x3b6cb2db, + 0x0bacb2db, + 0x1bacb2db, + 0x2bacb2db, + 0x3bacb2db, + 0x0becb2db, + 0x1becb2db, + 0x2becb2db, + 0x3becb2db, + 0x0b2db2db, + 0x1b2db2db, + 0x2b2db2db, + 0x3b2db2db, + 0x0b6db2db, + 0x1b6db2db, + 0x2b6db2db, + 0x3b6db2db, + 0x0badb2db, + 0x1badb2db, + 0x2badb2db, + 0x3badb2db, + 0x0bedb2db, + 0x1bedb2db, + 0x2bedb2db, + 0x3bedb2db, + 0x0b2eb2db, + 0x1b2eb2db, + 0x2b2eb2db, + 0x3b2eb2db, + 0x0b6eb2db, + 0x1b6eb2db, + 0x2b6eb2db, + 0x3b6eb2db, + 0x0baeb2db, + 0x1baeb2db, + 0x2baeb2db, + 0x3baeb2db, + 0x0beeb2db, + 0x1beeb2db, + 0x2beeb2db, + 0x3beeb2db, + 0x0b2fb2db, + 0x1b2fb2db, + 0x2b2fb2db, + 0x3b2fb2db, + 0x0b6fb2db, + 0x1b6fb2db, + 0x2b6fb2db, + 0x3b6fb2db, + 0x0bafb2db, + 0x1bafb2db, + 0x2bafb2db, + 0x3bafb2db, + 0x0befb2db, + 0x1befb2db, + 0x2befb2db, + 0x3befb2db, + 0x0b2cb6db, + 0x1b2cb6db, + 0x2b2cb6db, + 0x3b2cb6db, + 0x0b6cb6db, + 0x1b6cb6db, + 0x2b6cb6db, + 0x3b6cb6db, + 0x0bacb6db, + 0x1bacb6db, + 0x2bacb6db, + 0x3bacb6db, + 0x0becb6db, + 0x1becb6db, + 0x2becb6db, + 0x3becb6db, + 0x0b2db6db, + 0x1b2db6db, + 0x2b2db6db, + 0x3b2db6db, + 0x0b6db6db, + 0x1b6db6db, + 0x2b6db6db, + 0x3b6db6db, + 0x0badb6db, + 0x1badb6db, + 0x2badb6db, + 0x3badb6db, + 0x0bedb6db, + 0x1bedb6db, + 0x2bedb6db, + 0x3bedb6db, + 0x0b2eb6db, + 0x1b2eb6db, + 0x2b2eb6db, + 0x3b2eb6db, + 0x0b6eb6db, + 0x1b6eb6db, + 0x2b6eb6db, + 0x3b6eb6db, + 0x0baeb6db, + 0x1baeb6db, + 0x2baeb6db, + 0x3baeb6db, +} + +var kNonZeroRepsDepth = [numCommandSymbols]uint32{ + 6, + 6, + 6, + 6, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 12, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 18, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 24, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, + 30, +} + +var kStaticCommandCodeBits = [numCommandSymbols]uint16{ + 0, + 256, + 128, + 384, + 64, + 320, + 192, + 448, + 32, + 288, + 160, + 416, + 96, + 352, + 224, + 480, + 16, + 272, + 144, + 400, + 80, + 336, + 208, + 464, + 48, + 304, + 176, + 432, + 112, + 368, + 240, + 496, + 8, + 264, + 136, + 392, + 72, + 328, + 200, + 456, + 40, + 296, + 168, + 424, + 104, + 360, + 232, + 488, + 24, + 280, + 152, + 408, + 88, + 344, + 216, + 472, + 56, + 312, + 184, + 440, + 120, + 376, + 248, + 504, + 4, + 260, + 132, + 388, + 68, + 324, + 196, + 452, + 36, + 292, + 164, + 420, + 100, + 356, + 228, + 484, + 20, + 276, + 148, + 404, + 84, + 340, + 212, + 468, + 52, + 308, + 180, + 436, + 116, + 372, + 244, + 500, + 12, + 268, + 140, + 396, + 76, + 332, + 204, + 460, + 44, + 300, + 172, + 428, + 108, + 364, + 236, + 492, + 28, + 284, + 156, + 412, + 92, + 348, + 220, + 476, + 60, + 316, + 188, + 444, + 124, + 380, + 252, + 508, + 2, + 258, + 130, + 386, + 66, + 322, + 194, + 450, + 34, + 290, + 162, + 418, + 98, + 354, + 226, + 482, + 18, + 274, + 146, + 402, + 82, + 338, + 210, + 466, + 50, + 306, + 178, + 434, + 114, + 370, + 242, + 498, + 10, + 266, + 138, + 394, + 74, + 330, + 202, + 458, + 42, + 298, + 170, + 426, + 106, + 362, + 234, + 490, + 26, + 282, + 154, + 410, + 90, + 346, + 218, + 474, + 58, + 314, + 186, + 442, + 122, + 378, + 250, + 506, + 6, + 262, + 134, + 390, + 70, + 326, + 198, + 454, + 38, + 294, + 166, + 422, + 102, + 358, + 230, + 486, + 22, + 278, + 150, + 406, + 86, + 342, + 214, + 470, + 54, + 310, + 182, + 438, + 118, + 374, + 246, + 502, + 14, + 270, + 142, + 398, + 78, + 334, + 206, + 462, + 46, + 302, + 174, + 430, + 110, + 366, + 238, + 494, + 30, + 286, + 158, + 414, + 94, + 350, + 222, + 478, + 62, + 318, + 190, + 446, + 126, + 382, + 254, + 510, + 1, + 257, + 129, + 385, + 65, + 321, + 193, + 449, + 33, + 289, + 161, + 417, + 97, + 353, + 225, + 481, + 17, + 273, + 145, + 401, + 81, + 337, + 209, + 465, + 49, + 305, + 177, + 433, + 113, + 369, + 241, + 497, + 9, + 265, + 137, + 393, + 73, + 329, + 201, + 457, + 41, + 297, + 169, + 425, + 105, + 361, + 233, + 489, + 25, + 281, + 153, + 409, + 89, + 345, + 217, + 473, + 57, + 313, + 185, + 441, + 121, + 377, + 249, + 505, + 5, + 261, + 133, + 389, + 69, + 325, + 197, + 453, + 37, + 293, + 165, + 421, + 101, + 357, + 229, + 485, + 21, + 277, + 149, + 405, + 85, + 341, + 213, + 469, + 53, + 309, + 181, + 437, + 117, + 373, + 245, + 501, + 13, + 269, + 141, + 397, + 77, + 333, + 205, + 461, + 45, + 301, + 173, + 429, + 109, + 365, + 237, + 493, + 29, + 285, + 157, + 413, + 93, + 349, + 221, + 477, + 61, + 317, + 189, + 445, + 125, + 381, + 253, + 509, + 3, + 259, + 131, + 387, + 67, + 323, + 195, + 451, + 35, + 291, + 163, + 419, + 99, + 355, + 227, + 483, + 19, + 275, + 147, + 403, + 83, + 339, + 211, + 467, + 51, + 307, + 179, + 435, + 115, + 371, + 243, + 499, + 11, + 267, + 139, + 395, + 75, + 331, + 203, + 459, + 43, + 299, + 171, + 427, + 107, + 363, + 235, + 491, + 27, + 283, + 155, + 411, + 91, + 347, + 219, + 475, + 59, + 315, + 187, + 443, + 123, + 379, + 251, + 507, + 7, + 1031, + 519, + 1543, + 263, + 1287, + 775, + 1799, + 135, + 1159, + 647, + 1671, + 391, + 1415, + 903, + 1927, + 71, + 1095, + 583, + 1607, + 327, + 1351, + 839, + 1863, + 199, + 1223, + 711, + 1735, + 455, + 1479, + 967, + 1991, + 39, + 1063, + 551, + 1575, + 295, + 1319, + 807, + 1831, + 167, + 1191, + 679, + 1703, + 423, + 1447, + 935, + 1959, + 103, + 1127, + 615, + 1639, + 359, + 1383, + 871, + 1895, + 231, + 1255, + 743, + 1767, + 487, + 1511, + 999, + 2023, + 23, + 1047, + 535, + 1559, + 279, + 1303, + 791, + 1815, + 151, + 1175, + 663, + 1687, + 407, + 1431, + 919, + 1943, + 87, + 1111, + 599, + 1623, + 343, + 1367, + 855, + 1879, + 215, + 1239, + 727, + 1751, + 471, + 1495, + 983, + 2007, + 55, + 1079, + 567, + 1591, + 311, + 1335, + 823, + 1847, + 183, + 1207, + 695, + 1719, + 439, + 1463, + 951, + 1975, + 119, + 1143, + 631, + 1655, + 375, + 1399, + 887, + 1911, + 247, + 1271, + 759, + 1783, + 503, + 1527, + 1015, + 2039, + 15, + 1039, + 527, + 1551, + 271, + 1295, + 783, + 1807, + 143, + 1167, + 655, + 1679, + 399, + 1423, + 911, + 1935, + 79, + 1103, + 591, + 1615, + 335, + 1359, + 847, + 1871, + 207, + 1231, + 719, + 1743, + 463, + 1487, + 975, + 1999, + 47, + 1071, + 559, + 1583, + 303, + 1327, + 815, + 1839, + 175, + 1199, + 687, + 1711, + 431, + 1455, + 943, + 1967, + 111, + 1135, + 623, + 1647, + 367, + 1391, + 879, + 1903, + 239, + 1263, + 751, + 1775, + 495, + 1519, + 1007, + 2031, + 31, + 1055, + 543, + 1567, + 287, + 1311, + 799, + 1823, + 159, + 1183, + 671, + 1695, + 415, + 1439, + 927, + 1951, + 95, + 1119, + 607, + 1631, + 351, + 1375, + 863, + 1887, + 223, + 1247, + 735, + 1759, + 479, + 1503, + 991, + 2015, + 63, + 1087, + 575, + 1599, + 319, + 1343, + 831, + 1855, + 191, + 1215, + 703, + 1727, + 447, + 1471, + 959, + 1983, + 127, + 1151, + 639, + 1663, + 383, + 1407, + 895, + 1919, + 255, + 1279, + 767, + 1791, + 511, + 1535, + 1023, + 2047, +} + +func storeStaticCommandHuffmanTree(storage_ix *uint, storage []byte) { + writeBits(56, 0x92624416307003, storage_ix, storage) + writeBits(3, 0x00000000, storage_ix, storage) +} + +var kStaticDistanceCodeBits = [64]uint16{ + 0, + 32, + 16, + 48, + 8, + 40, + 24, + 56, + 4, + 36, + 20, + 52, + 12, + 44, + 28, + 60, + 2, + 34, + 18, + 50, + 10, + 42, + 26, + 58, + 6, + 38, + 22, + 54, + 14, + 46, + 30, + 62, + 1, + 33, + 17, + 49, + 9, + 41, + 25, + 57, + 5, + 37, + 21, + 53, + 13, + 45, + 29, + 61, + 3, + 35, + 19, + 51, + 11, + 43, + 27, + 59, + 7, + 39, + 23, + 55, + 15, + 47, + 31, + 63, +} + +func storeStaticDistanceHuffmanTree(storage_ix *uint, storage []byte) { + writeBits(28, 0x0369DC03, storage_ix, storage) +} diff --git a/vendor/github.com/andybalholm/brotli/fast_log.go b/vendor/github.com/andybalholm/brotli/fast_log.go new file mode 100644 index 0000000000..9d6607f7e2 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/fast_log.go @@ -0,0 +1,290 @@ +package brotli + +import ( + "math" + "math/bits" +) + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Utilities for fast computation of logarithms. */ + +func log2FloorNonZero(n uint) uint32 { + return uint32(bits.Len(n)) - 1 +} + +/* A lookup table for small values of log2(int) to be used in entropy + computation. + + ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */ +var kLog2Table = []float32{ + 0.0000000000000000, + 0.0000000000000000, + 1.0000000000000000, + 1.5849625007211563, + 2.0000000000000000, + 2.3219280948873622, + 2.5849625007211561, + 2.8073549220576042, + 3.0000000000000000, + 3.1699250014423126, + 3.3219280948873626, + 3.4594316186372978, + 3.5849625007211565, + 3.7004397181410922, + 3.8073549220576037, + 3.9068905956085187, + 4.0000000000000000, + 4.0874628412503400, + 4.1699250014423122, + 4.2479275134435852, + 4.3219280948873626, + 4.3923174227787607, + 4.4594316186372973, + 4.5235619560570131, + 4.5849625007211570, + 4.6438561897747244, + 4.7004397181410926, + 4.7548875021634691, + 4.8073549220576037, + 4.8579809951275728, + 4.9068905956085187, + 4.9541963103868758, + 5.0000000000000000, + 5.0443941193584534, + 5.0874628412503400, + 5.1292830169449664, + 5.1699250014423122, + 5.2094533656289501, + 5.2479275134435852, + 5.2854022188622487, + 5.3219280948873626, + 5.3575520046180838, + 5.3923174227787607, + 5.4262647547020979, + 5.4594316186372973, + 5.4918530963296748, + 5.5235619560570131, + 5.5545888516776376, + 5.5849625007211570, + 5.6147098441152083, + 5.6438561897747244, + 5.6724253419714961, + 5.7004397181410926, + 5.7279204545631996, + 5.7548875021634691, + 5.7813597135246599, + 5.8073549220576046, + 5.8328900141647422, + 5.8579809951275719, + 5.8826430493618416, + 5.9068905956085187, + 5.9307373375628867, + 5.9541963103868758, + 5.9772799234999168, + 6.0000000000000000, + 6.0223678130284544, + 6.0443941193584534, + 6.0660891904577721, + 6.0874628412503400, + 6.1085244567781700, + 6.1292830169449672, + 6.1497471195046822, + 6.1699250014423122, + 6.1898245588800176, + 6.2094533656289510, + 6.2288186904958804, + 6.2479275134435861, + 6.2667865406949019, + 6.2854022188622487, + 6.3037807481771031, + 6.3219280948873617, + 6.3398500028846252, + 6.3575520046180847, + 6.3750394313469254, + 6.3923174227787598, + 6.4093909361377026, + 6.4262647547020979, + 6.4429434958487288, + 6.4594316186372982, + 6.4757334309663976, + 6.4918530963296748, + 6.5077946401986964, + 6.5235619560570131, + 6.5391588111080319, + 6.5545888516776376, + 6.5698556083309478, + 6.5849625007211561, + 6.5999128421871278, + 6.6147098441152092, + 6.6293566200796095, + 6.6438561897747253, + 6.6582114827517955, + 6.6724253419714952, + 6.6865005271832185, + 6.7004397181410917, + 6.7142455176661224, + 6.7279204545631988, + 6.7414669864011465, + 6.7548875021634691, + 6.7681843247769260, + 6.7813597135246599, + 6.7944158663501062, + 6.8073549220576037, + 6.8201789624151887, + 6.8328900141647422, + 6.8454900509443757, + 6.8579809951275719, + 6.8703647195834048, + 6.8826430493618416, + 6.8948177633079437, + 6.9068905956085187, + 6.9188632372745955, + 6.9307373375628867, + 6.9425145053392399, + 6.9541963103868758, + 6.9657842846620879, + 6.9772799234999168, + 6.9886846867721664, + 7.0000000000000000, + 7.0112272554232540, + 7.0223678130284544, + 7.0334230015374501, + 7.0443941193584534, + 7.0552824355011898, + 7.0660891904577721, + 7.0768155970508317, + 7.0874628412503400, + 7.0980320829605272, + 7.1085244567781700, + 7.1189410727235076, + 7.1292830169449664, + 7.1395513523987937, + 7.1497471195046822, + 7.1598713367783891, + 7.1699250014423130, + 7.1799090900149345, + 7.1898245588800176, + 7.1996723448363644, + 7.2094533656289492, + 7.2191685204621621, + 7.2288186904958804, + 7.2384047393250794, + 7.2479275134435861, + 7.2573878426926521, + 7.2667865406949019, + 7.2761244052742384, + 7.2854022188622487, + 7.2946207488916270, + 7.3037807481771031, + 7.3128829552843557, + 7.3219280948873617, + 7.3309168781146177, + 7.3398500028846243, + 7.3487281542310781, + 7.3575520046180847, + 7.3663222142458151, + 7.3750394313469254, + 7.3837042924740528, + 7.3923174227787607, + 7.4008794362821844, + 7.4093909361377026, + 7.4178525148858991, + 7.4262647547020979, + 7.4346282276367255, + 7.4429434958487288, + 7.4512111118323299, + 7.4594316186372973, + 7.4676055500829976, + 7.4757334309663976, + 7.4838157772642564, + 7.4918530963296748, + 7.4998458870832057, + 7.5077946401986964, + 7.5156998382840436, + 7.5235619560570131, + 7.5313814605163119, + 7.5391588111080319, + 7.5468944598876373, + 7.5545888516776376, + 7.5622424242210728, + 7.5698556083309478, + 7.5774288280357487, + 7.5849625007211561, + 7.5924570372680806, + 7.5999128421871278, + 7.6073303137496113, + 7.6147098441152075, + 7.6220518194563764, + 7.6293566200796095, + 7.6366246205436488, + 7.6438561897747244, + 7.6510516911789290, + 7.6582114827517955, + 7.6653359171851765, + 7.6724253419714952, + 7.6794800995054464, + 7.6865005271832185, + 7.6934869574993252, + 7.7004397181410926, + 7.7073591320808825, + 7.7142455176661224, + 7.7210991887071856, + 7.7279204545631996, + 7.7347096202258392, + 7.7414669864011465, + 7.7481928495894596, + 7.7548875021634691, + 7.7615512324444795, + 7.7681843247769260, + 7.7747870596011737, + 7.7813597135246608, + 7.7879025593914317, + 7.7944158663501062, + 7.8008998999203047, + 7.8073549220576037, + 7.8137811912170374, + 7.8201789624151887, + 7.8265484872909159, + 7.8328900141647422, + 7.8392037880969445, + 7.8454900509443757, + 7.8517490414160571, + 7.8579809951275719, + 7.8641861446542798, + 7.8703647195834048, + 7.8765169465650002, + 7.8826430493618425, + 7.8887432488982601, + 7.8948177633079446, + 7.9008668079807496, + 7.9068905956085187, + 7.9128893362299619, + 7.9188632372745955, + 7.9248125036057813, + 7.9307373375628867, + 7.9366379390025719, + 7.9425145053392399, + 7.9483672315846778, + 7.9541963103868758, + 7.9600019320680806, + 7.9657842846620870, + 7.9715435539507720, + 7.9772799234999168, + 7.9829935746943104, + 7.9886846867721664, + 7.9943534368588578, +} + +/* Faster logarithm for small integers, with the property of log2(0) == 0. */ +func fastLog2(v uint) float64 { + if v < uint(len(kLog2Table)) { + return float64(kLog2Table[v]) + } + + return math.Log2(float64(v)) +} diff --git a/vendor/github.com/andybalholm/brotli/find_match_length.go b/vendor/github.com/andybalholm/brotli/find_match_length.go new file mode 100644 index 0000000000..09d2ae6726 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/find_match_length.go @@ -0,0 +1,45 @@ +package brotli + +import ( + "encoding/binary" + "math/bits" + "runtime" +) + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Function to find maximal matching prefixes of strings. */ +func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { + var matched uint = 0 + _, _ = s1[limit-1], s2[limit-1] // bounds check + switch runtime.GOARCH { + case "amd64": + // Compare 8 bytes at at time. + for matched+8 <= limit { + w1 := binary.LittleEndian.Uint64(s1[matched:]) + w2 := binary.LittleEndian.Uint64(s2[matched:]) + if w1 != w2 { + return matched + uint(bits.TrailingZeros64(w1^w2)>>3) + } + matched += 8 + } + case "386": + // Compare 4 bytes at at time. + for matched+4 <= limit { + w1 := binary.LittleEndian.Uint32(s1[matched:]) + w2 := binary.LittleEndian.Uint32(s2[matched:]) + if w1 != w2 { + return matched + uint(bits.TrailingZeros32(w1^w2)>>3) + } + matched += 4 + } + } + for matched < limit && s1[matched] == s2[matched] { + matched++ + } + return matched +} diff --git a/vendor/github.com/andybalholm/brotli/h10.go b/vendor/github.com/andybalholm/brotli/h10.go new file mode 100644 index 0000000000..5662fbbbb5 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/h10.go @@ -0,0 +1,287 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2016 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func (*h10) HashTypeLength() uint { + return 4 +} + +func (*h10) StoreLookahead() uint { + return 128 +} + +func hashBytesH10(data []byte) uint32 { + var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return h >> (32 - 17) +} + +/* A (forgetful) hash table where each hash bucket contains a binary tree of + sequences whose first 4 bytes share the same hash code. + Each sequence is 128 long and is identified by its starting + position in the input data. The binary tree is sorted by the lexicographic + order of the sequences, and it is also a max-heap with respect to the + starting positions. */ +type h10 struct { + hasherCommon + window_mask_ uint + buckets_ [1 << 17]uint32 + invalid_pos_ uint32 + forest []uint32 +} + +func (h *h10) Initialize(params *encoderParams) { + h.window_mask_ = (1 << params.lgwin) - 1 + h.invalid_pos_ = uint32(0 - h.window_mask_) + var num_nodes uint = uint(1) << params.lgwin + h.forest = make([]uint32, 2*num_nodes) +} + +func (h *h10) Prepare(one_shot bool, input_size uint, data []byte) { + var invalid_pos uint32 = h.invalid_pos_ + var i uint32 + for i = 0; i < 1<<17; i++ { + h.buckets_[i] = invalid_pos + } +} + +func leftChildIndexH10(self *h10, pos uint) uint { + return 2 * (pos & self.window_mask_) +} + +func rightChildIndexH10(self *h10, pos uint) uint { + return 2*(pos&self.window_mask_) + 1 +} + +/* Stores the hash of the next 4 bytes and in a single tree-traversal, the + hash bucket's binary tree is searched for matches and is re-rooted at the + current position. + + If less than 128 data is available, the hash bucket of the + current position is searched for matches, but the state of the hash table + is not changed, since we can not know the final sorting order of the + current (incomplete) sequence. + + This function must be called with increasing cur_ix positions. */ +func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch { + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var max_comp_len uint = brotli_min_size_t(max_length, 128) + var should_reroot_tree bool = (max_length >= 128) + var key uint32 = hashBytesH10(data[cur_ix_masked:]) + var forest []uint32 = self.forest + var prev_ix uint = uint(self.buckets_[key]) + var node_left uint = leftChildIndexH10(self, cur_ix) + var node_right uint = rightChildIndexH10(self, cur_ix) + var best_len_left uint = 0 + var best_len_right uint = 0 + var depth_remaining uint + /* The forest index of the rightmost node of the left subtree of the new + root, updated as we traverse and re-root the tree of the hash bucket. */ + + /* The forest index of the leftmost node of the right subtree of the new + root, updated as we traverse and re-root the tree of the hash bucket. */ + + /* The match length of the rightmost node of the left subtree of the new + root, updated as we traverse and re-root the tree of the hash bucket. */ + + /* The match length of the leftmost node of the right subtree of the new + root, updated as we traverse and re-root the tree of the hash bucket. */ + if should_reroot_tree { + self.buckets_[key] = uint32(cur_ix) + } + + for depth_remaining = 64; ; depth_remaining-- { + var backward uint = cur_ix - prev_ix + var prev_ix_masked uint = prev_ix & ring_buffer_mask + if backward == 0 || backward > max_backward || depth_remaining == 0 { + if should_reroot_tree { + forest[node_left] = self.invalid_pos_ + forest[node_right] = self.invalid_pos_ + } + + break + } + { + var cur_len uint = brotli_min_size_t(best_len_left, best_len_right) + var len uint + assert(cur_len <= 128) + len = cur_len + findMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len) + if matches != nil && len > *best_len { + *best_len = uint(len) + initBackwardMatch(&matches[0], backward, uint(len)) + matches = matches[1:] + } + + if len >= max_comp_len { + if should_reroot_tree { + forest[node_left] = forest[leftChildIndexH10(self, prev_ix)] + forest[node_right] = forest[rightChildIndexH10(self, prev_ix)] + } + + break + } + + if data[cur_ix_masked+len] > data[prev_ix_masked+len] { + best_len_left = uint(len) + if should_reroot_tree { + forest[node_left] = uint32(prev_ix) + } + + node_left = rightChildIndexH10(self, prev_ix) + prev_ix = uint(forest[node_left]) + } else { + best_len_right = uint(len) + if should_reroot_tree { + forest[node_right] = uint32(prev_ix) + } + + node_right = leftChildIndexH10(self, prev_ix) + prev_ix = uint(forest[node_right]) + } + } + } + + return matches +} + +/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the + length of max_length and stores the position cur_ix in the hash table. + + Sets *num_matches to the number of matches found, and stores the found + matches in matches[0] to matches[*num_matches - 1]. The matches will be + sorted by strictly increasing length and (non-strictly) increasing + distance. */ +func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint { + var orig_matches []backwardMatch = matches + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var best_len uint = 1 + var short_match_max_backward uint + if params.quality != hqZopflificationQuality { + short_match_max_backward = 16 + } else { + short_match_max_backward = 64 + } + var stop uint = cur_ix - short_match_max_backward + var dict_matches [maxStaticDictionaryMatchLen + 1]uint32 + var i uint + if cur_ix < short_match_max_backward { + stop = 0 + } + for i = cur_ix - 1; i > stop && best_len <= 2; i-- { + var prev_ix uint = i + var backward uint = cur_ix - prev_ix + if backward > max_backward { + break + } + + prev_ix &= ring_buffer_mask + if data[cur_ix_masked] != data[prev_ix] || data[cur_ix_masked+1] != data[prev_ix+1] { + continue + } + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len > best_len { + best_len = uint(len) + initBackwardMatch(&matches[0], backward, uint(len)) + matches = matches[1:] + } + } + } + + if best_len < max_length { + matches = storeAndFindMatchesH10(handle, data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches) + } + + for i = 0; i <= maxStaticDictionaryMatchLen; i++ { + dict_matches[i] = kInvalidMatch + } + { + var minlen uint = brotli_max_size_t(4, best_len+1) + if findAllStaticDictionaryMatches(dictionary, data[cur_ix_masked:], minlen, max_length, dict_matches[0:]) { + var maxlen uint = brotli_min_size_t(maxStaticDictionaryMatchLen, max_length) + var l uint + for l = minlen; l <= maxlen; l++ { + var dict_id uint32 = dict_matches[l] + if dict_id < kInvalidMatch { + var distance uint = max_backward + gap + uint(dict_id>>5) + 1 + if distance <= params.dist.max_distance { + initDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31)) + matches = matches[1:] + } + } + } + } + } + + return uint(-cap(matches) + cap(orig_matches)) +} + +/* Stores the hash of the next 4 bytes and re-roots the binary tree at the + current sequence, without returning any matches. + REQUIRES: ix + 128 <= end-of-current-block */ +func (h *h10) Store(data []byte, mask uint, ix uint) { + var max_backward uint = h.window_mask_ - windowGap + 1 + /* Maximum distance is window size - 16, see section 9.1. of the spec. */ + storeAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil) +} + +func (h *h10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { + var i uint = ix_start + var j uint = ix_start + if ix_start+63 <= ix_end { + i = ix_end - 63 + } + + if ix_start+512 <= i { + for ; j < i; j += 8 { + h.Store(data, mask, j) + } + } + + for ; i < ix_end; i++ { + h.Store(data, mask, i) + } +} + +func (h *h10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) { + if num_bytes >= h.HashTypeLength()-1 && position >= 128 { + var i_start uint = position - 128 + 1 + var i_end uint = brotli_min_size_t(position, i_start+num_bytes) + /* Store the last `128 - 1` positions in the hasher. + These could not be calculated before, since they require knowledge + of both the previous and the current block. */ + + var i uint + for i = i_start; i < i_end; i++ { + /* Maximum distance is window size - 16, see section 9.1. of the spec. + Furthermore, we have to make sure that we don't look further back + from the start of the next block than the window size, otherwise we + could access already overwritten areas of the ring-buffer. */ + var max_backward uint = h.window_mask_ - brotli_max_size_t(windowGap-1, position-i) + + /* We know that i + 128 <= position + num_bytes, i.e. the + end of the current block and that we have at least + 128 tail in the ring-buffer. */ + storeAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil) + } + } +} + +/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */ +const maxNumMatchesH10 = 128 + +func (*h10) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + panic("unimplemented") +} + +func (*h10) PrepareDistanceCache(distance_cache []int) { + panic("unimplemented") +} diff --git a/vendor/github.com/andybalholm/brotli/h5.go b/vendor/github.com/andybalholm/brotli/h5.go new file mode 100644 index 0000000000..f391b73fdd --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/h5.go @@ -0,0 +1,214 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* A (forgetful) hash table to the data seen by the compressor, to + help create backward references to previous data. + + This is a hash map of fixed size (bucket_size_) to a ring buffer of + fixed size (block_size_). The ring buffer contains the last block_size_ + index positions of the given hash key in the compressed data. */ +func (*h5) HashTypeLength() uint { + return 4 +} + +func (*h5) StoreLookahead() uint { + return 4 +} + +/* HashBytes is the function that chooses the bucket to place the address in. */ +func hashBytesH5(data []byte, shift int) uint32 { + var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return uint32(h >> uint(shift)) +} + +type h5 struct { + hasherCommon + bucket_size_ uint + block_size_ uint + hash_shift_ int + block_mask_ uint32 + num []uint16 + buckets []uint32 +} + +func (h *h5) Initialize(params *encoderParams) { + h.hash_shift_ = 32 - h.params.bucket_bits + h.bucket_size_ = uint(1) << uint(h.params.bucket_bits) + h.block_size_ = uint(1) << uint(h.params.block_bits) + h.block_mask_ = uint32(h.block_size_ - 1) + h.num = make([]uint16, h.bucket_size_) + h.buckets = make([]uint32, h.block_size_*h.bucket_size_) +} + +func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) { + var num []uint16 = h.num + var partial_prepare_threshold uint = h.bucket_size_ >> 6 + /* Partial preparation is 100 times slower (per socket). */ + if one_shot && input_size <= partial_prepare_threshold { + var i uint + for i = 0; i < input_size; i++ { + var key uint32 = hashBytesH5(data[i:], h.hash_shift_) + num[key] = 0 + } + } else { + for i := 0; i < int(h.bucket_size_); i++ { + num[i] = 0 + } + } +} + +/* Look at 4 bytes at &data[ix & mask]. + Compute a hash from these, and store the value of ix at that position. */ +func (h *h5) Store(data []byte, mask uint, ix uint) { + var num []uint16 = h.num + var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_) + var minor_ix uint = uint(num[key]) & uint(h.block_mask_) + var offset uint = minor_ix + uint(key<= h.HashTypeLength()-1 && position >= 3 { + /* Prepare the hashes for three last bytes of the last write. + These could not be calculated before, since they require knowledge + of both the previous and the current block. */ + h.Store(ringbuffer, ringbuffer_mask, position-3) + h.Store(ringbuffer, ringbuffer_mask, position-2) + h.Store(ringbuffer, ringbuffer_mask, position-1) + } +} + +func (h *h5) PrepareDistanceCache(distance_cache []int) { + prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check) +} + +/* Find a longest backward match of &data[cur_ix] up to the length of + max_length and stores the position cur_ix in the hash table. + + REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache + values; if this method is invoked repeatedly with the same distance + cache values, it is enough to invoke PrepareDistanceCacheH5 once. + + Does not look for matches longer than max_length. + Does not look for matches further away than max_backward. + Writes the best match into |out|. + |out|->score is updated only if a better match is found. */ +func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + var num []uint16 = h.num + var buckets []uint32 = h.buckets + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var min_score uint = out.score + var best_score uint = out.score + var best_len uint = out.len + var i uint + var bucket []uint32 + /* Don't accept a short copy from far away. */ + out.len = 0 + + out.len_code_delta = 0 + + /* Try last distance first. */ + for i = 0; i < uint(h.params.num_last_distances_to_check); i++ { + var backward uint = uint(distance_cache[i]) + var prev_ix uint = uint(cur_ix - backward) + if prev_ix >= cur_ix { + continue + } + + if backward > max_backward { + continue + } + + prev_ix &= ring_buffer_mask + + if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { + continue + } + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 3 || (len == 2 && i < 2) { + /* Comparing for >= 2 does not change the semantics, but just saves for + a few unnecessary binary logarithms in backward reference score, + since we are not interested in such short matches. */ + var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) + if best_score < score { + if i != 0 { + score -= backwardReferencePenaltyUsingLastDistance(i) + } + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = best_score + } + } + } + } + } + { + var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_) + bucket = buckets[key< h.block_size_ { + down = uint(num[key]) - h.block_size_ + } else { + down = 0 + } + for i = uint(num[key]); i > down; { + var prev_ix uint + i-- + prev_ix = uint(bucket[uint32(i)&h.block_mask_]) + var backward uint = cur_ix - prev_ix + if backward > max_backward { + break + } + + prev_ix &= ring_buffer_mask + if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { + continue + } + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 4 { + /* Comparing for >= 3 does not change the semantics, but just saves + for a few unnecessary binary logarithms in backward reference + score, since we are not interested in such short matches. */ + var score uint = backwardReferenceScore(uint(len), backward) + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = best_score + } + } + } + } + + bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix) + num[key]++ + } + + if min_score == out.score { + searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false) + } +} diff --git a/vendor/github.com/andybalholm/brotli/h6.go b/vendor/github.com/andybalholm/brotli/h6.go new file mode 100644 index 0000000000..80bb224aa8 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/h6.go @@ -0,0 +1,216 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* A (forgetful) hash table to the data seen by the compressor, to + help create backward references to previous data. + + This is a hash map of fixed size (bucket_size_) to a ring buffer of + fixed size (block_size_). The ring buffer contains the last block_size_ + index positions of the given hash key in the compressed data. */ +func (*h6) HashTypeLength() uint { + return 8 +} + +func (*h6) StoreLookahead() uint { + return 8 +} + +/* HashBytes is the function that chooses the bucket to place the address in. */ +func hashBytesH6(data []byte, mask uint64, shift int) uint32 { + var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return uint32(h >> uint(shift)) +} + +type h6 struct { + hasherCommon + bucket_size_ uint + block_size_ uint + hash_shift_ int + hash_mask_ uint64 + block_mask_ uint32 + num []uint16 + buckets []uint32 +} + +func (h *h6) Initialize(params *encoderParams) { + h.hash_shift_ = 64 - h.params.bucket_bits + h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len) + h.bucket_size_ = uint(1) << uint(h.params.bucket_bits) + h.block_size_ = uint(1) << uint(h.params.block_bits) + h.block_mask_ = uint32(h.block_size_ - 1) + h.num = make([]uint16, h.bucket_size_) + h.buckets = make([]uint32, h.block_size_*h.bucket_size_) +} + +func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) { + var num []uint16 = h.num + var partial_prepare_threshold uint = h.bucket_size_ >> 6 + /* Partial preparation is 100 times slower (per socket). */ + if one_shot && input_size <= partial_prepare_threshold { + var i uint + for i = 0; i < input_size; i++ { + var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_) + num[key] = 0 + } + } else { + for i := 0; i < int(h.bucket_size_); i++ { + num[i] = 0 + } + } +} + +/* Look at 4 bytes at &data[ix & mask]. + Compute a hash from these, and store the value of ix at that position. */ +func (h *h6) Store(data []byte, mask uint, ix uint) { + var num []uint16 = h.num + var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_) + var minor_ix uint = uint(num[key]) & uint(h.block_mask_) + var offset uint = minor_ix + uint(key<= h.HashTypeLength()-1 && position >= 3 { + /* Prepare the hashes for three last bytes of the last write. + These could not be calculated before, since they require knowledge + of both the previous and the current block. */ + h.Store(ringbuffer, ringbuffer_mask, position-3) + h.Store(ringbuffer, ringbuffer_mask, position-2) + h.Store(ringbuffer, ringbuffer_mask, position-1) + } +} + +func (h *h6) PrepareDistanceCache(distance_cache []int) { + prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check) +} + +/* Find a longest backward match of &data[cur_ix] up to the length of + max_length and stores the position cur_ix in the hash table. + + REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache + values; if this method is invoked repeatedly with the same distance + cache values, it is enough to invoke PrepareDistanceCacheH6 once. + + Does not look for matches longer than max_length. + Does not look for matches further away than max_backward. + Writes the best match into |out|. + |out|->score is updated only if a better match is found. */ +func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + var num []uint16 = h.num + var buckets []uint32 = h.buckets + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var min_score uint = out.score + var best_score uint = out.score + var best_len uint = out.len + var i uint + var bucket []uint32 + /* Don't accept a short copy from far away. */ + out.len = 0 + + out.len_code_delta = 0 + + /* Try last distance first. */ + for i = 0; i < uint(h.params.num_last_distances_to_check); i++ { + var backward uint = uint(distance_cache[i]) + var prev_ix uint = uint(cur_ix - backward) + if prev_ix >= cur_ix { + continue + } + + if backward > max_backward { + continue + } + + prev_ix &= ring_buffer_mask + + if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { + continue + } + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 3 || (len == 2 && i < 2) { + /* Comparing for >= 2 does not change the semantics, but just saves for + a few unnecessary binary logarithms in backward reference score, + since we are not interested in such short matches. */ + var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) + if best_score < score { + if i != 0 { + score -= backwardReferencePenaltyUsingLastDistance(i) + } + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = best_score + } + } + } + } + } + { + var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_) + bucket = buckets[key< h.block_size_ { + down = uint(num[key]) - h.block_size_ + } else { + down = 0 + } + for i = uint(num[key]); i > down; { + var prev_ix uint + i-- + prev_ix = uint(bucket[uint32(i)&h.block_mask_]) + var backward uint = cur_ix - prev_ix + if backward > max_backward { + break + } + + prev_ix &= ring_buffer_mask + if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { + continue + } + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 4 { + /* Comparing for >= 3 does not change the semantics, but just saves + for a few unnecessary binary logarithms in backward reference + score, since we are not interested in such short matches. */ + var score uint = backwardReferenceScore(uint(len), backward) + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = best_score + } + } + } + } + + bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix) + num[key]++ + } + + if min_score == out.score { + searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false) + } +} diff --git a/vendor/github.com/andybalholm/brotli/hash.go b/vendor/github.com/andybalholm/brotli/hash.go new file mode 100644 index 0000000000..00f812e87e --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/hash.go @@ -0,0 +1,342 @@ +package brotli + +import ( + "encoding/binary" + "fmt" +) + +type hasherCommon struct { + params hasherParams + is_prepared_ bool + dict_num_lookups uint + dict_num_matches uint +} + +func (h *hasherCommon) Common() *hasherCommon { + return h +} + +type hasherHandle interface { + Common() *hasherCommon + Initialize(params *encoderParams) + Prepare(one_shot bool, input_size uint, data []byte) + StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) + HashTypeLength() uint + StoreLookahead() uint + PrepareDistanceCache(distance_cache []int) + FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) + StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) + Store(data []byte, mask uint, ix uint) +} + +const kCutoffTransformsCount uint32 = 10 + +/* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */ +/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */ +const kCutoffTransforms uint64 = 0x071B520ADA2D3200 + +type hasherSearchResult struct { + len uint + distance uint + score uint + len_code_delta int +} + +/* kHashMul32 multiplier has these properties: + * The multiplier must be odd. Otherwise we may lose the highest bit. + * No long streaks of ones or zeros. + * There is no effort to ensure that it is a prime, the oddity is enough + for this use. + * The number has been tuned heuristically against compression benchmarks. */ +const kHashMul32 uint32 = 0x1E35A7BD + +const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD + +const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3 + +func hash14(data []byte) uint32 { + var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return h >> (32 - 14) +} + +func prepareDistanceCache(distance_cache []int, num_distances int) { + if num_distances > 4 { + var last_distance int = distance_cache[0] + distance_cache[4] = last_distance - 1 + distance_cache[5] = last_distance + 1 + distance_cache[6] = last_distance - 2 + distance_cache[7] = last_distance + 2 + distance_cache[8] = last_distance - 3 + distance_cache[9] = last_distance + 3 + if num_distances > 10 { + var next_last_distance int = distance_cache[1] + distance_cache[10] = next_last_distance - 1 + distance_cache[11] = next_last_distance + 1 + distance_cache[12] = next_last_distance - 2 + distance_cache[13] = next_last_distance + 2 + distance_cache[14] = next_last_distance - 3 + distance_cache[15] = next_last_distance + 3 + } + } +} + +const literalByteScore = 135 + +const distanceBitPenalty = 30 + +/* Score must be positive after applying maximal penalty. */ +const scoreBase = (distanceBitPenalty * 8 * 8) + +/* Usually, we always choose the longest backward reference. This function + allows for the exception of that rule. + + If we choose a backward reference that is further away, it will + usually be coded with more bits. We approximate this by assuming + log2(distance). If the distance can be expressed in terms of the + last four distances, we use some heuristic constants to estimate + the bits cost. For the first up to four literals we use the bit + cost of the literals from the literal cost model, after that we + use the average bit cost of the cost model. + + This function is used to sometimes discard a longer backward reference + when it is not much longer and the bit cost for encoding it is more + than the saved literals. + + backward_reference_offset MUST be positive. */ +func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint { + return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset)) +} + +func backwardReferenceScoreUsingLastDistance(copy_length uint) uint { + return literalByteScore*uint(copy_length) + scoreBase + 15 +} + +func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint { + return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE) +} + +func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool { + var len uint + var word_idx uint + var offset uint + var matchlen uint + var backward uint + var score uint + len = item & 0x1F + word_idx = item >> 5 + offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx + if len > max_length { + return false + } + + matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len)) + if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 { + return false + } + { + var cut uint = len - matchlen + var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F) + backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len]) + } + + if backward > max_distance { + return false + } + + score = backwardReferenceScore(matchlen, backward) + if score < out.score { + return false + } + + out.len = matchlen + out.len_code_delta = int(len) - int(matchlen) + out.distance = backward + out.score = score + return true +} + +func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) { + var key uint + var i uint + var self *hasherCommon = handle.Common() + if self.dict_num_matches < self.dict_num_lookups>>7 { + return + } + + key = uint(hash14(data) << 1) + for i = 0; ; (func() { i++; key++ })() { + var tmp uint + if shallow { + tmp = 1 + } else { + tmp = 2 + } + if i >= tmp { + break + } + var item uint = uint(dictionary.hash_table[key]) + self.dict_num_lookups++ + if item != 0 { + var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out) + if item_matches { + self.dict_num_matches++ + } + } + } +} + +type backwardMatch struct { + distance uint32 + length_and_code uint32 +} + +func initBackwardMatch(self *backwardMatch, dist uint, len uint) { + self.distance = uint32(dist) + self.length_and_code = uint32(len << 5) +} + +func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) { + self.distance = uint32(dist) + var tmp uint + if len == len_code { + tmp = 0 + } else { + tmp = len_code + } + self.length_and_code = uint32(len<<5 | tmp) +} + +func backwardMatchLength(self *backwardMatch) uint { + return uint(self.length_and_code >> 5) +} + +func backwardMatchLengthCode(self *backwardMatch) uint { + var code uint = uint(self.length_and_code) & 31 + if code != 0 { + return code + } else { + return backwardMatchLength(self) + } +} + +func hasherReset(handle hasherHandle) { + if handle == nil { + return + } + handle.Common().is_prepared_ = false +} + +func newHasher(typ int) hasherHandle { + switch typ { + case 2: + return &hashLongestMatchQuickly{ + bucketBits: 16, + bucketSweep: 1, + hashLen: 5, + useDictionary: true, + } + case 3: + return &hashLongestMatchQuickly{ + bucketBits: 16, + bucketSweep: 2, + hashLen: 5, + useDictionary: false, + } + case 4: + return &hashLongestMatchQuickly{ + bucketBits: 17, + bucketSweep: 4, + hashLen: 5, + useDictionary: true, + } + case 5: + return new(h5) + case 6: + return new(h6) + case 10: + return new(h10) + case 35: + return &hashComposite{ + ha: newHasher(3), + hb: &hashRolling{jump: 4}, + } + case 40: + return &hashForgetfulChain{ + bucketBits: 15, + numBanks: 1, + bankBits: 16, + numLastDistancesToCheck: 4, + } + case 41: + return &hashForgetfulChain{ + bucketBits: 15, + numBanks: 1, + bankBits: 16, + numLastDistancesToCheck: 10, + } + case 42: + return &hashForgetfulChain{ + bucketBits: 15, + numBanks: 512, + bankBits: 9, + numLastDistancesToCheck: 16, + } + case 54: + return &hashLongestMatchQuickly{ + bucketBits: 20, + bucketSweep: 4, + hashLen: 7, + useDictionary: false, + } + case 55: + return &hashComposite{ + ha: newHasher(54), + hb: &hashRolling{jump: 4}, + } + case 65: + return &hashComposite{ + ha: newHasher(6), + hb: &hashRolling{jump: 1}, + } + } + + panic(fmt.Sprintf("unknown hasher type: %d", typ)) +} + +func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) { + var self hasherHandle = nil + var common *hasherCommon = nil + var one_shot bool = (position == 0 && is_last) + if *handle == nil { + chooseHasher(params, ¶ms.hasher) + self = newHasher(params.hasher.type_) + + *handle = self + common = self.Common() + common.params = params.hasher + self.Initialize(params) + } + + self = *handle + common = self.Common() + if !common.is_prepared_ { + self.Prepare(one_shot, input_size, data) + + if position == 0 { + common.dict_num_lookups = 0 + common.dict_num_matches = 0 + } + + common.is_prepared_ = true + } +} + +func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) { + var self hasherHandle + hasherSetup(handle, params, data, position, input_size, is_last) + self = *handle + self.StitchToPreviousBlock(input_size, position, data, mask) +} diff --git a/vendor/github.com/andybalholm/brotli/hash_composite.go b/vendor/github.com/andybalholm/brotli/hash_composite.go new file mode 100644 index 0000000000..a65fe2e6a9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/hash_composite.go @@ -0,0 +1,93 @@ +package brotli + +/* Copyright 2018 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func (h *hashComposite) HashTypeLength() uint { + var a uint = h.ha.HashTypeLength() + var b uint = h.hb.HashTypeLength() + if a > b { + return a + } else { + return b + } +} + +func (h *hashComposite) StoreLookahead() uint { + var a uint = h.ha.StoreLookahead() + var b uint = h.hb.StoreLookahead() + if a > b { + return a + } else { + return b + } +} + +/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A + and HASHER_B. */ +type hashComposite struct { + hasherCommon + ha hasherHandle + hb hasherHandle + params *encoderParams +} + +func (h *hashComposite) Initialize(params *encoderParams) { + h.params = params +} + +/* TODO: Initialize of the hashers is defered to Prepare (and params + remembered here) because we don't get the one_shot and input_size params + here that are needed to know the memory size of them. Instead provide + those params to all hashers InitializehashComposite */ +func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) { + if h.ha == nil { + var common_a *hasherCommon + var common_b *hasherCommon + + common_a = h.ha.Common() + common_a.params = h.params.hasher + common_a.is_prepared_ = false + common_a.dict_num_lookups = 0 + common_a.dict_num_matches = 0 + h.ha.Initialize(h.params) + + common_b = h.hb.Common() + common_b.params = h.params.hasher + common_b.is_prepared_ = false + common_b.dict_num_lookups = 0 + common_b.dict_num_matches = 0 + h.hb.Initialize(h.params) + } + + h.ha.Prepare(one_shot, input_size, data) + h.hb.Prepare(one_shot, input_size, data) +} + +func (h *hashComposite) Store(data []byte, mask uint, ix uint) { + h.ha.Store(data, mask, ix) + h.hb.Store(data, mask, ix) +} + +func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { + h.ha.StoreRange(data, mask, ix_start, ix_end) + h.hb.StoreRange(data, mask, ix_start, ix_end) +} + +func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) { + h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask) + h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask) +} + +func (h *hashComposite) PrepareDistanceCache(distance_cache []int) { + h.ha.PrepareDistanceCache(distance_cache) + h.hb.PrepareDistanceCache(distance_cache) +} + +func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out) + h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out) +} diff --git a/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go b/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go new file mode 100644 index 0000000000..306e46d3db --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/hash_forgetful_chain.go @@ -0,0 +1,252 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2016 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func (*hashForgetfulChain) HashTypeLength() uint { + return 4 +} + +func (*hashForgetfulChain) StoreLookahead() uint { + return 4 +} + +/* HashBytes is the function that chooses the bucket to place the address in.*/ +func (h *hashForgetfulChain) HashBytes(data []byte) uint { + var hash uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return uint(hash >> (32 - h.bucketBits)) +} + +type slot struct { + delta uint16 + next uint16 +} + +/* A (forgetful) hash table to the data seen by the compressor, to + help create backward references to previous data. + + Hashes are stored in chains which are bucketed to groups. Group of chains + share a storage "bank". When more than "bank size" chain nodes are added, + oldest nodes are replaced; this way several chains may share a tail. */ +type hashForgetfulChain struct { + hasherCommon + + bucketBits uint + numBanks uint + bankBits uint + numLastDistancesToCheck int + + addr []uint32 + head []uint16 + tiny_hash [65536]byte + banks [][]slot + free_slot_idx []uint16 + max_hops uint +} + +func (h *hashForgetfulChain) Initialize(params *encoderParams) { + var q uint + if params.quality > 6 { + q = 7 + } else { + q = 8 + } + h.max_hops = q << uint(params.quality-4) + + bankSize := 1 << h.bankBits + bucketSize := 1 << h.bucketBits + + h.addr = make([]uint32, bucketSize) + h.head = make([]uint16, bucketSize) + h.banks = make([][]slot, h.numBanks) + for i := range h.banks { + h.banks[i] = make([]slot, bankSize) + } + h.free_slot_idx = make([]uint16, h.numBanks) +} + +func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte) { + var partial_prepare_threshold uint = (1 << h.bucketBits) >> 6 + /* Partial preparation is 100 times slower (per socket). */ + if one_shot && input_size <= partial_prepare_threshold { + var i uint + for i = 0; i < input_size; i++ { + var bucket uint = h.HashBytes(data[i:]) + + /* See InitEmpty comment. */ + h.addr[bucket] = 0xCCCCCCCC + + h.head[bucket] = 0xCCCC + } + } else { + /* Fill |addr| array with 0xCCCCCCCC value. Because of wrapping, position + processed by hasher never reaches 3GB + 64M; this makes all new chains + to be terminated after the first node. */ + for i := range h.addr { + h.addr[i] = 0xCCCCCCCC + } + + for i := range h.head { + h.head[i] = 0 + } + } + + h.tiny_hash = [65536]byte{} + for i := range h.free_slot_idx { + h.free_slot_idx[i] = 0 + } +} + +/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend + node to corresponding chain; also update tiny_hash for current position. */ +func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) { + var key uint = h.HashBytes(data[ix&mask:]) + var bank uint = key & (h.numBanks - 1) + idx := uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1) + h.free_slot_idx[bank]++ + var delta uint = ix - uint(h.addr[key]) + h.tiny_hash[uint16(ix)] = byte(key) + if delta > 0xFFFF { + delta = 0xFFFF + } + h.banks[bank][idx].delta = uint16(delta) + h.banks[bank][idx].next = h.head[key] + h.addr[key] = uint32(ix) + h.head[key] = uint16(idx) +} + +func (h *hashForgetfulChain) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { + var i uint + for i = ix_start; i < ix_end; i++ { + h.Store(data, mask, i) + } +} + +func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) { + if num_bytes >= h.HashTypeLength()-1 && position >= 3 { + /* Prepare the hashes for three last bytes of the last write. + These could not be calculated before, since they require knowledge + of both the previous and the current block. */ + h.Store(ringbuffer, ring_buffer_mask, position-3) + h.Store(ringbuffer, ring_buffer_mask, position-2) + h.Store(ringbuffer, ring_buffer_mask, position-1) + } +} + +func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) { + prepareDistanceCache(distance_cache, h.numLastDistancesToCheck) +} + +/* Find a longest backward match of &data[cur_ix] up to the length of + max_length and stores the position cur_ix in the hash table. + + REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache + values; if this method is invoked repeatedly with the same distance + cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once. + + Does not look for matches longer than max_length. + Does not look for matches further away than max_backward. + Writes the best match into |out|. + |out|->score is updated only if a better match is found. */ +func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var min_score uint = out.score + var best_score uint = out.score + var best_len uint = out.len + var key uint = h.HashBytes(data[cur_ix_masked:]) + var tiny_hash byte = byte(key) + /* Don't accept a short copy from far away. */ + out.len = 0 + + out.len_code_delta = 0 + + /* Try last distance first. */ + for i := 0; i < h.numLastDistancesToCheck; i++ { + var backward uint = uint(distance_cache[i]) + var prev_ix uint = (cur_ix - backward) + + /* For distance code 0 we want to consider 2-byte matches. */ + if i > 0 && h.tiny_hash[uint16(prev_ix)] != tiny_hash { + continue + } + if prev_ix >= cur_ix || backward > max_backward { + continue + } + + prev_ix &= ring_buffer_mask + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 2 { + var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) + if best_score < score { + if i != 0 { + score -= backwardReferencePenaltyUsingLastDistance(uint(i)) + } + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = best_score + } + } + } + } + } + { + var bank uint = key & (h.numBanks - 1) + var backward uint = 0 + var hops uint = h.max_hops + var delta uint = cur_ix - uint(h.addr[key]) + var slot uint = uint(h.head[key]) + for { + tmp6 := hops + hops-- + if tmp6 == 0 { + break + } + var prev_ix uint + var last uint = slot + backward += delta + if backward > max_backward { + break + } + prev_ix = (cur_ix - backward) & ring_buffer_mask + slot = uint(h.banks[bank][last].next) + delta = uint(h.banks[bank][last].delta) + if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { + continue + } + { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 4 { + /* Comparing for >= 3 does not change the semantics, but just saves + for a few unnecessary binary logarithms in backward reference + score, since we are not interested in such short matches. */ + var score uint = backwardReferenceScore(uint(len), backward) + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = best_score + } + } + } + } + + h.Store(data, ring_buffer_mask, cur_ix) + } + + if out.score == min_score { + searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false) + } +} diff --git a/vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go b/vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go new file mode 100644 index 0000000000..9375dc1553 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/hash_longest_match_quickly.go @@ -0,0 +1,214 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression + a little faster (0.5% - 1%) and it compresses 0.15% better on small text + and HTML inputs. */ + +func (*hashLongestMatchQuickly) HashTypeLength() uint { + return 8 +} + +func (*hashLongestMatchQuickly) StoreLookahead() uint { + return 8 +} + +/* HashBytes is the function that chooses the bucket to place + the address in. The HashLongestMatch and hashLongestMatchQuickly + classes have separate, different implementations of hashing. */ +func (h *hashLongestMatchQuickly) HashBytes(data []byte) uint32 { + var hash uint64 = ((binary.LittleEndian.Uint64(data) << (64 - 8*h.hashLen)) * kHashMul64) + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return uint32(hash >> (64 - h.bucketBits)) +} + +/* A (forgetful) hash table to the data seen by the compressor, to + help create backward references to previous data. + + This is a hash map of fixed size (1 << 16). Starting from the + given index, 1 buckets are used to store values of a key. */ +type hashLongestMatchQuickly struct { + hasherCommon + + bucketBits uint + bucketSweep int + hashLen uint + useDictionary bool + + buckets []uint32 +} + +func (h *hashLongestMatchQuickly) Initialize(params *encoderParams) { + h.buckets = make([]uint32, 1<> 7 + /* Partial preparation is 100 times slower (per socket). */ + if one_shot && input_size <= partial_prepare_threshold { + var i uint + for i = 0; i < input_size; i++ { + var key uint32 = h.HashBytes(data[i:]) + for j := 0; j < h.bucketSweep; j++ { + h.buckets[key+uint32(j)] = 0 + } + } + } else { + /* It is not strictly necessary to fill this buffer here, but + not filling will make the results of the compression stochastic + (but correct). This is because random data would cause the + system to find accidentally good backward references here and there. */ + for i := range h.buckets { + h.buckets[i] = 0 + } + } +} + +/* Look at 5 bytes at &data[ix & mask]. + Compute a hash from these, and store the value somewhere within + [ix .. ix+3]. */ +func (h *hashLongestMatchQuickly) Store(data []byte, mask uint, ix uint) { + var key uint32 = h.HashBytes(data[ix&mask:]) + var off uint32 = uint32(ix>>3) % uint32(h.bucketSweep) + /* Wiggle the value with the bucket sweep range. */ + h.buckets[key+off] = uint32(ix) +} + +func (h *hashLongestMatchQuickly) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { + var i uint + for i = ix_start; i < ix_end; i++ { + h.Store(data, mask, i) + } +} + +func (h *hashLongestMatchQuickly) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) { + if num_bytes >= h.HashTypeLength()-1 && position >= 3 { + /* Prepare the hashes for three last bytes of the last write. + These could not be calculated before, since they require knowledge + of both the previous and the current block. */ + h.Store(ringbuffer, ringbuffer_mask, position-3) + h.Store(ringbuffer, ringbuffer_mask, position-2) + h.Store(ringbuffer, ringbuffer_mask, position-1) + } +} + +func (*hashLongestMatchQuickly) PrepareDistanceCache(distance_cache []int) { +} + +/* Find a longest backward match of &data[cur_ix & ring_buffer_mask] + up to the length of max_length and stores the position cur_ix in the + hash table. + + Does not look for matches longer than max_length. + Does not look for matches further away than max_backward. + Writes the best match into |out|. + |out|->score is updated only if a better match is found. */ +func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + var best_len_in uint = out.len + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var key uint32 = h.HashBytes(data[cur_ix_masked:]) + var compare_char int = int(data[cur_ix_masked+best_len_in]) + var min_score uint = out.score + var best_score uint = out.score + var best_len uint = best_len_in + var cached_backward uint = uint(distance_cache[0]) + var prev_ix uint = cur_ix - cached_backward + var bucket []uint32 + out.len_code_delta = 0 + if prev_ix < cur_ix { + prev_ix &= uint(uint32(ring_buffer_mask)) + if compare_char == int(data[prev_ix+best_len]) { + var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 4 { + var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) + if best_score < score { + best_score = score + best_len = uint(len) + out.len = uint(len) + out.distance = cached_backward + out.score = best_score + compare_char = int(data[cur_ix_masked+best_len]) + if h.bucketSweep == 1 { + h.buckets[key] = uint32(cur_ix) + return + } + } + } + } + } + + if h.bucketSweep == 1 { + var backward uint + var len uint + + /* Only one to look for, don't bother to prepare for a loop. */ + prev_ix = uint(h.buckets[key]) + + h.buckets[key] = uint32(cur_ix) + backward = cur_ix - prev_ix + prev_ix &= uint(uint32(ring_buffer_mask)) + if compare_char != int(data[prev_ix+best_len_in]) { + return + } + + if backward == 0 || backward > max_backward { + return + } + + len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 4 { + var score uint = backwardReferenceScore(uint(len), backward) + if best_score < score { + out.len = uint(len) + out.distance = backward + out.score = score + return + } + } + } else { + bucket = h.buckets[key:] + var i int + prev_ix = uint(bucket[0]) + bucket = bucket[1:] + for i = 0; i < h.bucketSweep; (func() { i++; tmp3 := bucket; bucket = bucket[1:]; prev_ix = uint(tmp3[0]) })() { + var backward uint = cur_ix - prev_ix + var len uint + prev_ix &= uint(uint32(ring_buffer_mask)) + if compare_char != int(data[prev_ix+best_len]) { + continue + } + + if backward == 0 || backward > max_backward { + continue + } + + len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) + if len >= 4 { + var score uint = backwardReferenceScore(uint(len), backward) + if best_score < score { + best_score = score + best_len = uint(len) + out.len = best_len + out.distance = backward + out.score = score + compare_char = int(data[cur_ix_masked+best_len]) + } + } + } + } + + if h.useDictionary && min_score == out.score { + searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, true) + } + + h.buckets[key+uint32((cur_ix>>3)%uint(h.bucketSweep))] = uint32(cur_ix) +} diff --git a/vendor/github.com/andybalholm/brotli/hash_rolling.go b/vendor/github.com/andybalholm/brotli/hash_rolling.go new file mode 100644 index 0000000000..6630fc07e4 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/hash_rolling.go @@ -0,0 +1,168 @@ +package brotli + +/* Copyright 2018 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* NOTE: this hasher does not search in the dictionary. It is used as + backup-hasher, the main hasher already searches in it. */ + +const kRollingHashMul32 uint32 = 69069 + +const kInvalidPosHashRolling uint32 = 0xffffffff + +/* This hasher uses a longer forward length, but returning a higher value here + will hurt compression by the main hasher when combined with a composite + hasher. The hasher tests for forward itself instead. */ +func (*hashRolling) HashTypeLength() uint { + return 4 +} + +func (*hashRolling) StoreLookahead() uint { + return 4 +} + +/* Computes a code from a single byte. A lookup table of 256 values could be + used, but simply adding 1 works about as good. */ +func (*hashRolling) HashByte(b byte) uint32 { + return uint32(b) + 1 +} + +func (h *hashRolling) HashRollingFunctionInitial(state uint32, add byte, factor uint32) uint32 { + return uint32(factor*state + h.HashByte(add)) +} + +func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, factor uint32, factor_remove uint32) uint32 { + return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem)) +} + +/* Rolling hash for long distance long string matches. Stores one position + per bucket, bucket key is computed over a long region. */ +type hashRolling struct { + hasherCommon + + jump int + + state uint32 + table []uint32 + next_ix uint + factor uint32 + factor_remove uint32 +} + +func (h *hashRolling) Initialize(params *encoderParams) { + h.state = 0 + h.next_ix = 0 + + h.factor = kRollingHashMul32 + + /* Compute the factor of the oldest byte to remove: factor**steps modulo + 0xffffffff (the multiplications rely on 32-bit overflow) */ + h.factor_remove = 1 + + for i := 0; i < 32; i += h.jump { + h.factor_remove *= h.factor + } + + h.table = make([]uint32, 16777216) + for i := 0; i < 16777216; i++ { + h.table[i] = kInvalidPosHashRolling + } +} + +func (h *hashRolling) Prepare(one_shot bool, input_size uint, data []byte) { + /* Too small size, cannot use this hasher. */ + if input_size < 32 { + return + } + h.state = 0 + for i := 0; i < 32; i += h.jump { + h.state = h.HashRollingFunctionInitial(h.state, data[i], h.factor) + } +} + +func (*hashRolling) Store(data []byte, mask uint, ix uint) { +} + +func (*hashRolling) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { +} + +func (h *hashRolling) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) { + var position_masked uint + /* In this case we must re-initialize the hasher from scratch from the + current position. */ + + var available uint = num_bytes + if position&uint(h.jump-1) != 0 { + var diff uint = uint(h.jump) - (position & uint(h.jump-1)) + if diff > available { + available = 0 + } else { + available = available - diff + } + position += diff + } + + position_masked = position & ring_buffer_mask + + /* wrapping around ringbuffer not handled. */ + if available > ring_buffer_mask-position_masked { + available = ring_buffer_mask - position_masked + } + + h.Prepare(false, available, ringbuffer[position&ring_buffer_mask:]) + h.next_ix = position +} + +func (*hashRolling) PrepareDistanceCache(distance_cache []int) { +} + +func (h *hashRolling) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { + var cur_ix_masked uint = cur_ix & ring_buffer_mask + var pos uint = h.next_ix + + if cur_ix&uint(h.jump-1) != 0 { + return + } + + /* Not enough lookahead */ + if max_length < 32 { + return + } + + for pos = h.next_ix; pos <= cur_ix; pos += uint(h.jump) { + var code uint32 = h.state & ((16777216 * 64) - 1) + var rem byte = data[pos&ring_buffer_mask] + var add byte = data[(pos+32)&ring_buffer_mask] + var found_ix uint = uint(kInvalidPosHashRolling) + + h.state = h.HashRollingFunction(h.state, add, rem, h.factor, h.factor_remove) + + if code < 16777216 { + found_ix = uint(h.table[code]) + h.table[code] = uint32(pos) + if pos == cur_ix && uint32(found_ix) != kInvalidPosHashRolling { + /* The cast to 32-bit makes backward distances up to 4GB work even + if cur_ix is above 4GB, despite using 32-bit values in the table. */ + var backward uint = uint(uint32(cur_ix - found_ix)) + if backward <= max_backward { + var found_ix_masked uint = found_ix & ring_buffer_mask + var len uint = findMatchLengthWithLimit(data[found_ix_masked:], data[cur_ix_masked:], max_length) + if len >= 4 && len > out.len { + var score uint = backwardReferenceScore(uint(len), backward) + if score > out.score { + out.len = uint(len) + out.distance = backward + out.score = score + out.len_code_delta = 0 + } + } + } + } + } + } + + h.next_ix = cur_ix + uint(h.jump) +} diff --git a/vendor/github.com/andybalholm/brotli/histogram.go b/vendor/github.com/andybalholm/brotli/histogram.go new file mode 100644 index 0000000000..0346622beb --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/histogram.go @@ -0,0 +1,226 @@ +package brotli + +import "math" + +/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */ +const numHistogramDistanceSymbols = 544 + +type histogramLiteral struct { + data_ [numLiteralSymbols]uint32 + total_count_ uint + bit_cost_ float64 +} + +func histogramClearLiteral(self *histogramLiteral) { + self.data_ = [numLiteralSymbols]uint32{} + self.total_count_ = 0 + self.bit_cost_ = math.MaxFloat64 +} + +func clearHistogramsLiteral(array []histogramLiteral, length uint) { + var i uint + for i = 0; i < length; i++ { + histogramClearLiteral(&array[i:][0]) + } +} + +func histogramAddLiteral(self *histogramLiteral, val uint) { + self.data_[val]++ + self.total_count_++ +} + +func histogramAddVectorLiteral(self *histogramLiteral, p []byte, n uint) { + self.total_count_ += n + n += 1 + for { + n-- + if n == 0 { + break + } + self.data_[p[0]]++ + p = p[1:] + } +} + +func histogramAddHistogramLiteral(self *histogramLiteral, v *histogramLiteral) { + var i uint + self.total_count_ += v.total_count_ + for i = 0; i < numLiteralSymbols; i++ { + self.data_[i] += v.data_[i] + } +} + +func histogramDataSizeLiteral() uint { + return numLiteralSymbols +} + +type histogramCommand struct { + data_ [numCommandSymbols]uint32 + total_count_ uint + bit_cost_ float64 +} + +func histogramClearCommand(self *histogramCommand) { + self.data_ = [numCommandSymbols]uint32{} + self.total_count_ = 0 + self.bit_cost_ = math.MaxFloat64 +} + +func clearHistogramsCommand(array []histogramCommand, length uint) { + var i uint + for i = 0; i < length; i++ { + histogramClearCommand(&array[i:][0]) + } +} + +func histogramAddCommand(self *histogramCommand, val uint) { + self.data_[val]++ + self.total_count_++ +} + +func histogramAddVectorCommand(self *histogramCommand, p []uint16, n uint) { + self.total_count_ += n + n += 1 + for { + n-- + if n == 0 { + break + } + self.data_[p[0]]++ + p = p[1:] + } +} + +func histogramAddHistogramCommand(self *histogramCommand, v *histogramCommand) { + var i uint + self.total_count_ += v.total_count_ + for i = 0; i < numCommandSymbols; i++ { + self.data_[i] += v.data_[i] + } +} + +func histogramDataSizeCommand() uint { + return numCommandSymbols +} + +type histogramDistance struct { + data_ [numDistanceSymbols]uint32 + total_count_ uint + bit_cost_ float64 +} + +func histogramClearDistance(self *histogramDistance) { + self.data_ = [numDistanceSymbols]uint32{} + self.total_count_ = 0 + self.bit_cost_ = math.MaxFloat64 +} + +func clearHistogramsDistance(array []histogramDistance, length uint) { + var i uint + for i = 0; i < length; i++ { + histogramClearDistance(&array[i:][0]) + } +} + +func histogramAddDistance(self *histogramDistance, val uint) { + self.data_[val]++ + self.total_count_++ +} + +func histogramAddVectorDistance(self *histogramDistance, p []uint16, n uint) { + self.total_count_ += n + n += 1 + for { + n-- + if n == 0 { + break + } + self.data_[p[0]]++ + p = p[1:] + } +} + +func histogramAddHistogramDistance(self *histogramDistance, v *histogramDistance) { + var i uint + self.total_count_ += v.total_count_ + for i = 0; i < numDistanceSymbols; i++ { + self.data_[i] += v.data_[i] + } +} + +func histogramDataSizeDistance() uint { + return numDistanceSymbols +} + +type blockSplitIterator struct { + split_ *blockSplit + idx_ uint + type_ uint + length_ uint +} + +func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) { + self.split_ = split + self.idx_ = 0 + self.type_ = 0 + if len(split.lengths) > 0 { + self.length_ = uint(split.lengths[0]) + } else { + self.length_ = 0 + } +} + +func blockSplitIteratorNext(self *blockSplitIterator) { + if self.length_ == 0 { + self.idx_++ + self.type_ = uint(self.split_.types[self.idx_]) + self.length_ = uint(self.split_.lengths[self.idx_]) + } + + self.length_-- +} + +func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { + var pos uint = start_pos + var literal_it blockSplitIterator + var insert_and_copy_it blockSplitIterator + var dist_it blockSplitIterator + + initBlockSplitIterator(&literal_it, literal_split) + initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split) + initBlockSplitIterator(&dist_it, dist_split) + for i := range cmds { + var cmd *command = &cmds[i] + var j uint + blockSplitIteratorNext(&insert_and_copy_it) + histogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], uint(cmd.cmd_prefix_)) + + /* TODO: unwrap iterator blocks. */ + for j = uint(cmd.insert_len_); j != 0; j-- { + var context uint + blockSplitIteratorNext(&literal_it) + context = literal_it.type_ + if context_modes != nil { + var lut contextLUT = getContextLUT(context_modes[context]) + context = (context << literalContextBits) + uint(getContext(prev_byte, prev_byte2, lut)) + } + + histogramAddLiteral(&literal_histograms[context], uint(ringbuffer[pos&mask])) + prev_byte2 = prev_byte + prev_byte = ringbuffer[pos&mask] + pos++ + } + + pos += uint(commandCopyLen(cmd)) + if commandCopyLen(cmd) != 0 { + prev_byte2 = ringbuffer[(pos-2)&mask] + prev_byte = ringbuffer[(pos-1)&mask] + if cmd.cmd_prefix_ >= 128 { + var context uint + blockSplitIteratorNext(&dist_it) + context = uint(uint32(dist_it.type_< bestQ && + (spec.Value == "*" || spec.Value == offer) { + bestQ = spec.Q + bestOffer = offer + } + } + } + if bestQ == 0 { + bestOffer = "" + } + return bestOffer +} + +// acceptSpec describes an Accept* header. +type acceptSpec struct { + Value string + Q float64 +} + +// parseAccept parses Accept* headers. +func parseAccept(header http.Header, key string) (specs []acceptSpec) { +loop: + for _, s := range header[key] { + for { + var spec acceptSpec + spec.Value, s = expectTokenSlash(s) + if spec.Value == "" { + continue loop + } + spec.Q = 1.0 + s = skipSpace(s) + if strings.HasPrefix(s, ";") { + s = skipSpace(s[1:]) + if !strings.HasPrefix(s, "q=") { + continue loop + } + spec.Q, s = expectQuality(s[2:]) + if spec.Q < 0.0 { + continue loop + } + } + specs = append(specs, spec) + s = skipSpace(s) + if !strings.HasPrefix(s, ",") { + continue loop + } + s = skipSpace(s[1:]) + } + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectTokenSlash(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + b := s[i] + if (octetTypes[b]&isToken == 0) && b != '/' { + break + } + } + return s[:i], s[i:] +} + +func expectQuality(s string) (q float64, rest string) { + switch { + case len(s) == 0: + return -1, "" + case s[0] == '0': + q = 0 + case s[0] == '1': + q = 1 + default: + return -1, "" + } + s = s[1:] + if !strings.HasPrefix(s, ".") { + return q, s + } + s = s[1:] + i := 0 + n := 0 + d := 1 + for ; i < len(s); i++ { + b := s[i] + if b < '0' || b > '9' { + break + } + n = n*10 + int(b) - '0' + d *= 10 + } + return q + float64(n)/float64(d), s[i:] +} + +// Octet types from RFC 2616. +var octetTypes [256]octetType + +type octetType byte + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.ContainsRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) + if strings.ContainsRune(" \t\r\n", rune(c)) { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} diff --git a/vendor/github.com/andybalholm/brotli/huffman.go b/vendor/github.com/andybalholm/brotli/huffman.go new file mode 100644 index 0000000000..182f3d2a55 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/huffman.go @@ -0,0 +1,653 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Utilities for building Huffman decoding tables. */ + +const huffmanMaxCodeLength = 15 + +/* Maximum possible Huffman table size for an alphabet size of (index * 32), + max code length 15 and root table bits 8. */ +var kMaxHuffmanTableSize = []uint16{ + 256, + 402, + 436, + 468, + 500, + 534, + 566, + 598, + 630, + 662, + 694, + 726, + 758, + 790, + 822, + 854, + 886, + 920, + 952, + 984, + 1016, + 1048, + 1080, + 1112, + 1144, + 1176, + 1208, + 1240, + 1272, + 1304, + 1336, + 1368, + 1400, + 1432, + 1464, + 1496, + 1528, +} + +/* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */ +const huffmanMaxSize26 = 396 + +/* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */ +const huffmanMaxSize258 = 632 + +/* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */ +const huffmanMaxSize272 = 646 + +const huffmanMaxCodeLengthCodeLength = 5 + +/* Do not create this struct directly - use the ConstructHuffmanCode + * constructor below! */ +type huffmanCode struct { + bits byte + value uint16 +} + +func constructHuffmanCode(bits byte, value uint16) huffmanCode { + var h huffmanCode + h.bits = bits + h.value = value + return h +} + +/* Builds Huffman lookup table assuming code lengths are in symbol order. */ + +/* Builds Huffman lookup table assuming code lengths are in symbol order. + Returns size of resulting table. */ + +/* Builds a simple Huffman table. The |num_symbols| parameter is to be + interpreted as follows: 0 means 1 symbol, 1 means 2 symbols, + 2 means 3 symbols, 3 means 4 symbols with lengths [2, 2, 2, 2], + 4 means 4 symbols with lengths [1, 2, 3, 3]. */ + +/* Contains a collection of Huffman trees with the same alphabet size. */ +/* max_symbol is needed due to simple codes since log2(alphabet_size) could be + greater than log2(max_symbol). */ +type huffmanTreeGroup struct { + htrees [][]huffmanCode + codes []huffmanCode + alphabet_size uint16 + max_symbol uint16 + num_htrees uint16 +} + +const reverseBitsMax = 8 + +const reverseBitsBase = 0 + +var kReverseBits = [1 << reverseBitsMax]byte{ + 0x00, + 0x80, + 0x40, + 0xC0, + 0x20, + 0xA0, + 0x60, + 0xE0, + 0x10, + 0x90, + 0x50, + 0xD0, + 0x30, + 0xB0, + 0x70, + 0xF0, + 0x08, + 0x88, + 0x48, + 0xC8, + 0x28, + 0xA8, + 0x68, + 0xE8, + 0x18, + 0x98, + 0x58, + 0xD8, + 0x38, + 0xB8, + 0x78, + 0xF8, + 0x04, + 0x84, + 0x44, + 0xC4, + 0x24, + 0xA4, + 0x64, + 0xE4, + 0x14, + 0x94, + 0x54, + 0xD4, + 0x34, + 0xB4, + 0x74, + 0xF4, + 0x0C, + 0x8C, + 0x4C, + 0xCC, + 0x2C, + 0xAC, + 0x6C, + 0xEC, + 0x1C, + 0x9C, + 0x5C, + 0xDC, + 0x3C, + 0xBC, + 0x7C, + 0xFC, + 0x02, + 0x82, + 0x42, + 0xC2, + 0x22, + 0xA2, + 0x62, + 0xE2, + 0x12, + 0x92, + 0x52, + 0xD2, + 0x32, + 0xB2, + 0x72, + 0xF2, + 0x0A, + 0x8A, + 0x4A, + 0xCA, + 0x2A, + 0xAA, + 0x6A, + 0xEA, + 0x1A, + 0x9A, + 0x5A, + 0xDA, + 0x3A, + 0xBA, + 0x7A, + 0xFA, + 0x06, + 0x86, + 0x46, + 0xC6, + 0x26, + 0xA6, + 0x66, + 0xE6, + 0x16, + 0x96, + 0x56, + 0xD6, + 0x36, + 0xB6, + 0x76, + 0xF6, + 0x0E, + 0x8E, + 0x4E, + 0xCE, + 0x2E, + 0xAE, + 0x6E, + 0xEE, + 0x1E, + 0x9E, + 0x5E, + 0xDE, + 0x3E, + 0xBE, + 0x7E, + 0xFE, + 0x01, + 0x81, + 0x41, + 0xC1, + 0x21, + 0xA1, + 0x61, + 0xE1, + 0x11, + 0x91, + 0x51, + 0xD1, + 0x31, + 0xB1, + 0x71, + 0xF1, + 0x09, + 0x89, + 0x49, + 0xC9, + 0x29, + 0xA9, + 0x69, + 0xE9, + 0x19, + 0x99, + 0x59, + 0xD9, + 0x39, + 0xB9, + 0x79, + 0xF9, + 0x05, + 0x85, + 0x45, + 0xC5, + 0x25, + 0xA5, + 0x65, + 0xE5, + 0x15, + 0x95, + 0x55, + 0xD5, + 0x35, + 0xB5, + 0x75, + 0xF5, + 0x0D, + 0x8D, + 0x4D, + 0xCD, + 0x2D, + 0xAD, + 0x6D, + 0xED, + 0x1D, + 0x9D, + 0x5D, + 0xDD, + 0x3D, + 0xBD, + 0x7D, + 0xFD, + 0x03, + 0x83, + 0x43, + 0xC3, + 0x23, + 0xA3, + 0x63, + 0xE3, + 0x13, + 0x93, + 0x53, + 0xD3, + 0x33, + 0xB3, + 0x73, + 0xF3, + 0x0B, + 0x8B, + 0x4B, + 0xCB, + 0x2B, + 0xAB, + 0x6B, + 0xEB, + 0x1B, + 0x9B, + 0x5B, + 0xDB, + 0x3B, + 0xBB, + 0x7B, + 0xFB, + 0x07, + 0x87, + 0x47, + 0xC7, + 0x27, + 0xA7, + 0x67, + 0xE7, + 0x17, + 0x97, + 0x57, + 0xD7, + 0x37, + 0xB7, + 0x77, + 0xF7, + 0x0F, + 0x8F, + 0x4F, + 0xCF, + 0x2F, + 0xAF, + 0x6F, + 0xEF, + 0x1F, + 0x9F, + 0x5F, + 0xDF, + 0x3F, + 0xBF, + 0x7F, + 0xFF, +} + +const reverseBitsLowest = (uint64(1) << (reverseBitsMax - 1 + reverseBitsBase)) + +/* Returns reverse(num >> BROTLI_REVERSE_BITS_BASE, BROTLI_REVERSE_BITS_MAX), + where reverse(value, len) is the bit-wise reversal of the len least + significant bits of value. */ +func reverseBits8(num uint64) uint64 { + return uint64(kReverseBits[num]) +} + +/* Stores code in table[0], table[step], table[2*step], ..., table[end] */ +/* Assumes that end is an integer multiple of step */ +func replicateValue(table []huffmanCode, step int, end int, code huffmanCode) { + for { + end -= step + table[end] = code + if end <= 0 { + break + } + } +} + +/* Returns the table width of the next 2nd level table. |count| is the histogram + of bit lengths for the remaining symbols, |len| is the code length of the + next processed symbol. */ +func nextTableBitSize(count []uint16, len int, root_bits int) int { + var left int = 1 << uint(len-root_bits) + for len < huffmanMaxCodeLength { + left -= int(count[len]) + if left <= 0 { + break + } + len++ + left <<= 1 + } + + return len - root_bits +} + +func buildCodeLengthsHuffmanTable(table []huffmanCode, code_lengths []byte, count []uint16) { + var code huffmanCode /* current table entry */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* step size to replicate values in current table */ /* size of current table */ /* symbols sorted by code length */ + var symbol int + var key uint64 + var key_step uint64 + var step int + var table_size int + var sorted [codeLengthCodes]int + var offset [huffmanMaxCodeLengthCodeLength + 1]int + var bits int + var bits_count int + /* offsets in sorted table for each length */ + assert(huffmanMaxCodeLengthCodeLength <= reverseBitsMax) + + /* Generate offsets into sorted symbol table by code length. */ + symbol = -1 + + bits = 1 + var i int + for i = 0; i < huffmanMaxCodeLengthCodeLength; i++ { + symbol += int(count[bits]) + offset[bits] = symbol + bits++ + } + + /* Symbols with code length 0 are placed after all other symbols. */ + offset[0] = codeLengthCodes - 1 + + /* Sort symbols by length, by symbol order within each length. */ + symbol = codeLengthCodes + + for { + var i int + for i = 0; i < 6; i++ { + symbol-- + sorted[offset[code_lengths[symbol]]] = symbol + offset[code_lengths[symbol]]-- + } + if symbol == 0 { + break + } + } + + table_size = 1 << huffmanMaxCodeLengthCodeLength + + /* Special case: all symbols but one have 0 code length. */ + if offset[0] == 0 { + code = constructHuffmanCode(0, uint16(sorted[0])) + for key = 0; key < uint64(table_size); key++ { + table[key] = code + } + + return + } + + /* Fill in table. */ + key = 0 + + key_step = reverseBitsLowest + symbol = 0 + bits = 1 + step = 2 + for { + for bits_count = int(count[bits]); bits_count != 0; bits_count-- { + code = constructHuffmanCode(byte(bits), uint16(sorted[symbol])) + symbol++ + replicateValue(table[reverseBits8(key):], step, table_size, code) + key += key_step + } + + step <<= 1 + key_step >>= 1 + bits++ + if bits > huffmanMaxCodeLengthCodeLength { + break + } + } +} + +func buildHuffmanTable(root_table []huffmanCode, root_bits int, symbol_lists symbolList, count []uint16) uint32 { + var code huffmanCode /* current table entry */ /* next available space in table */ /* current code length */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* 2nd level table prefix code */ /* 2nd level table prefix code addend */ /* step size to replicate values in current table */ /* key length of current table */ /* size of current table */ /* sum of root table size and 2nd level table sizes */ + var table []huffmanCode + var len int + var symbol int + var key uint64 + var key_step uint64 + var sub_key uint64 + var sub_key_step uint64 + var step int + var table_bits int + var table_size int + var total_size int + var max_length int = -1 + var bits int + var bits_count int + + assert(root_bits <= reverseBitsMax) + assert(huffmanMaxCodeLength-root_bits <= reverseBitsMax) + + for symbolListGet(symbol_lists, max_length) == 0xFFFF { + max_length-- + } + max_length += huffmanMaxCodeLength + 1 + + table = root_table + table_bits = root_bits + table_size = 1 << uint(table_bits) + total_size = table_size + + /* Fill in the root table. Reduce the table size to if possible, + and create the repetitions by memcpy. */ + if table_bits > max_length { + table_bits = max_length + table_size = 1 << uint(table_bits) + } + + key = 0 + key_step = reverseBitsLowest + bits = 1 + step = 2 + for { + symbol = bits - (huffmanMaxCodeLength + 1) + for bits_count = int(count[bits]); bits_count != 0; bits_count-- { + symbol = int(symbolListGet(symbol_lists, symbol)) + code = constructHuffmanCode(byte(bits), uint16(symbol)) + replicateValue(table[reverseBits8(key):], step, table_size, code) + key += key_step + } + + step <<= 1 + key_step >>= 1 + bits++ + if bits > table_bits { + break + } + } + + /* If root_bits != table_bits then replicate to fill the remaining slots. */ + for total_size != table_size { + copy(table[table_size:], table[:uint(table_size)]) + table_size <<= 1 + } + + /* Fill in 2nd level tables and add pointers to root table. */ + key_step = reverseBitsLowest >> uint(root_bits-1) + + sub_key = reverseBitsLowest << 1 + sub_key_step = reverseBitsLowest + len = root_bits + 1 + step = 2 + for ; len <= max_length; len++ { + symbol = len - (huffmanMaxCodeLength + 1) + for ; count[len] != 0; count[len]-- { + if sub_key == reverseBitsLowest<<1 { + table = table[table_size:] + table_bits = nextTableBitSize(count, int(len), root_bits) + table_size = 1 << uint(table_bits) + total_size += table_size + sub_key = reverseBits8(key) + key += key_step + root_table[sub_key] = constructHuffmanCode(byte(table_bits+root_bits), uint16(uint64(uint(-cap(table)+cap(root_table)))-sub_key)) + sub_key = 0 + } + + symbol = int(symbolListGet(symbol_lists, symbol)) + code = constructHuffmanCode(byte(len-root_bits), uint16(symbol)) + replicateValue(table[reverseBits8(sub_key):], step, table_size, code) + sub_key += sub_key_step + } + + step <<= 1 + sub_key_step >>= 1 + } + + return uint32(total_size) +} + +func buildSimpleHuffmanTable(table []huffmanCode, root_bits int, val []uint16, num_symbols uint32) uint32 { + var table_size uint32 = 1 + var goal_size uint32 = 1 << uint(root_bits) + switch num_symbols { + case 0: + table[0] = constructHuffmanCode(0, val[0]) + + case 1: + if val[1] > val[0] { + table[0] = constructHuffmanCode(1, val[0]) + table[1] = constructHuffmanCode(1, val[1]) + } else { + table[0] = constructHuffmanCode(1, val[1]) + table[1] = constructHuffmanCode(1, val[0]) + } + + table_size = 2 + + case 2: + table[0] = constructHuffmanCode(1, val[0]) + table[2] = constructHuffmanCode(1, val[0]) + if val[2] > val[1] { + table[1] = constructHuffmanCode(2, val[1]) + table[3] = constructHuffmanCode(2, val[2]) + } else { + table[1] = constructHuffmanCode(2, val[2]) + table[3] = constructHuffmanCode(2, val[1]) + } + + table_size = 4 + + case 3: + var i int + var k int + for i = 0; i < 3; i++ { + for k = i + 1; k < 4; k++ { + if val[k] < val[i] { + var t uint16 = val[k] + val[k] = val[i] + val[i] = t + } + } + } + + table[0] = constructHuffmanCode(2, val[0]) + table[2] = constructHuffmanCode(2, val[1]) + table[1] = constructHuffmanCode(2, val[2]) + table[3] = constructHuffmanCode(2, val[3]) + table_size = 4 + + case 4: + if val[3] < val[2] { + var t uint16 = val[3] + val[3] = val[2] + val[2] = t + } + + table[0] = constructHuffmanCode(1, val[0]) + table[1] = constructHuffmanCode(2, val[1]) + table[2] = constructHuffmanCode(1, val[0]) + table[3] = constructHuffmanCode(3, val[2]) + table[4] = constructHuffmanCode(1, val[0]) + table[5] = constructHuffmanCode(2, val[1]) + table[6] = constructHuffmanCode(1, val[0]) + table[7] = constructHuffmanCode(3, val[3]) + table_size = 8 + } + + for table_size != goal_size { + copy(table[table_size:], table[:uint(table_size)]) + table_size <<= 1 + } + + return goal_size +} diff --git a/vendor/github.com/andybalholm/brotli/literal_cost.go b/vendor/github.com/andybalholm/brotli/literal_cost.go new file mode 100644 index 0000000000..5a9ace94ee --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/literal_cost.go @@ -0,0 +1,182 @@ +package brotli + +func utf8Position(last uint, c uint, clamp uint) uint { + if c < 128 { + return 0 /* Next one is the 'Byte 1' again. */ + } else if c >= 192 { /* Next one is the 'Byte 2' of utf-8 encoding. */ + return brotli_min_size_t(1, clamp) + } else { + /* Let's decide over the last byte if this ends the sequence. */ + if last < 0xE0 { + return 0 /* Completed two or three byte coding. */ /* Next one is the 'Byte 3' of utf-8 encoding. */ + } else { + return brotli_min_size_t(2, clamp) + } + } +} + +func decideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint { + var counts = [3]uint{0} /* should be 2, but 1 compresses better. */ + var max_utf8 uint = 1 + var last_c uint = 0 + var i uint + for i = 0; i < len; i++ { + var c uint = uint(data[(pos+i)&mask]) + counts[utf8Position(last_c, c, 2)]++ + last_c = c + } + + if counts[2] < 500 { + max_utf8 = 1 + } + + if counts[1]+counts[2] < 25 { + max_utf8 = 0 + } + + return max_utf8 +} + +func estimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte, cost []float32) { + var max_utf8 uint = decideMultiByteStatsLevel(pos, uint(len), mask, data) + /* Bootstrap histograms. */ + var histogram = [3][256]uint{[256]uint{0}} + var window_half uint = 495 + var in_window uint = brotli_min_size_t(window_half, uint(len)) + var in_window_utf8 = [3]uint{0} + /* max_utf8 is 0 (normal ASCII single byte modeling), + 1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */ + + var i uint + { + var last_c uint = 0 + var utf8_pos uint = 0 + for i = 0; i < in_window; i++ { + var c uint = uint(data[(pos+i)&mask]) + histogram[utf8_pos][c]++ + in_window_utf8[utf8_pos]++ + utf8_pos = utf8Position(last_c, c, max_utf8) + last_c = c + } + } + + /* Compute bit costs with sliding window. */ + for i = 0; i < len; i++ { + if i >= window_half { + var c uint + var last_c uint + if i < window_half+1 { + c = 0 + } else { + c = uint(data[(pos+i-window_half-1)&mask]) + } + if i < window_half+2 { + last_c = 0 + } else { + last_c = uint(data[(pos+i-window_half-2)&mask]) + } + /* Remove a byte in the past. */ + + var utf8_pos2 uint = utf8Position(last_c, c, max_utf8) + histogram[utf8_pos2][data[(pos+i-window_half)&mask]]-- + in_window_utf8[utf8_pos2]-- + } + + if i+window_half < len { + var c uint = uint(data[(pos+i+window_half-1)&mask]) + var last_c uint = uint(data[(pos+i+window_half-2)&mask]) + /* Add a byte in the future. */ + + var utf8_pos2 uint = utf8Position(last_c, c, max_utf8) + histogram[utf8_pos2][data[(pos+i+window_half)&mask]]++ + in_window_utf8[utf8_pos2]++ + } + { + var c uint + var last_c uint + if i < 1 { + c = 0 + } else { + c = uint(data[(pos+i-1)&mask]) + } + if i < 2 { + last_c = 0 + } else { + last_c = uint(data[(pos+i-2)&mask]) + } + var utf8_pos uint = utf8Position(last_c, c, max_utf8) + var masked_pos uint = (pos + i) & mask + var histo uint = histogram[utf8_pos][data[masked_pos]] + var lit_cost float64 + if histo == 0 { + histo = 1 + } + + lit_cost = fastLog2(in_window_utf8[utf8_pos]) - fastLog2(histo) + lit_cost += 0.02905 + if lit_cost < 1.0 { + lit_cost *= 0.5 + lit_cost += 0.5 + } + + /* Make the first bytes more expensive -- seems to help, not sure why. + Perhaps because the entropy source is changing its properties + rapidly in the beginning of the file, perhaps because the beginning + of the data is a statistical "anomaly". */ + if i < 2000 { + lit_cost += 0.7 - (float64(2000-i) / 2000.0 * 0.35) + } + + cost[i] = float32(lit_cost) + } + } +} + +func estimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byte, cost []float32) { + if isMostlyUTF8(data, pos, mask, uint(len), kMinUTF8Ratio) { + estimateBitCostsForLiteralsUTF8(pos, uint(len), mask, data, cost) + return + } else { + var histogram = [256]uint{0} + var window_half uint = 2000 + var in_window uint = brotli_min_size_t(window_half, uint(len)) + var i uint + /* Bootstrap histogram. */ + for i = 0; i < in_window; i++ { + histogram[data[(pos+i)&mask]]++ + } + + /* Compute bit costs with sliding window. */ + for i = 0; i < len; i++ { + var histo uint + if i >= window_half { + /* Remove a byte in the past. */ + histogram[data[(pos+i-window_half)&mask]]-- + + in_window-- + } + + if i+window_half < len { + /* Add a byte in the future. */ + histogram[data[(pos+i+window_half)&mask]]++ + + in_window++ + } + + histo = histogram[data[(pos+i)&mask]] + if histo == 0 { + histo = 1 + } + { + var lit_cost float64 = fastLog2(in_window) - fastLog2(histo) + lit_cost += 0.029 + if lit_cost < 1.0 { + lit_cost *= 0.5 + lit_cost += 0.5 + } + + cost[i] = float32(lit_cost) + } + } + } +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/emitter.go b/vendor/github.com/andybalholm/brotli/matchfinder/emitter.go new file mode 100644 index 0000000000..37ed8e1334 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/emitter.go @@ -0,0 +1,45 @@ +package matchfinder + +// An absoluteMatch is like a Match, but it stores indexes into the byte +// stream instead of lengths. +type absoluteMatch struct { + // Start is the index of the first byte. + Start int + + // End is the index of the byte after the last byte + // (so that End - Start = Length). + End int + + // Match is the index of the previous data that matches + // (Start - Match = Distance). + Match int +} + +// A matchEmitter manages the output of matches for a MatchFinder. +type matchEmitter struct { + // Dst is the destination slice that Matches are added to. + Dst []Match + + // NextEmit is the index of the next byte to emit. + NextEmit int +} + +func (e *matchEmitter) emit(m absoluteMatch) { + e.Dst = append(e.Dst, Match{ + Unmatched: m.Start - e.NextEmit, + Length: m.End - m.Start, + Distance: m.Start - m.Match, + }) + e.NextEmit = m.End +} + +// trim shortens m if it extends past maxEnd. Then if the length is at least +// minLength, the match is emitted. +func (e *matchEmitter) trim(m absoluteMatch, maxEnd int, minLength int) { + if m.End > maxEnd { + m.End = maxEnd + } + if m.End-m.Start >= minLength { + e.emit(m) + } +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/m0.go b/vendor/github.com/andybalholm/brotli/matchfinder/m0.go new file mode 100644 index 0000000000..773b7c49f3 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/m0.go @@ -0,0 +1,169 @@ +package matchfinder + +import ( + "encoding/binary" +) + +// M0 is an implementation of the MatchFinder interface based +// on the algorithm used by snappy, but modified to be more like the algorithm +// used by compression level 0 of the brotli reference implementation. +// +// It has a maximum block size of 65536 bytes. +type M0 struct { + // Lazy turns on "lazy matching," for higher compression but less speed. + Lazy bool + + MaxDistance int + MaxLength int +} + +func (M0) Reset() {} + +const ( + m0HashLen = 5 + + m0TableBits = 14 + m0TableSize = 1 << m0TableBits + m0Shift = 32 - m0TableBits + // m0TableMask is redundant, but helps the compiler eliminate bounds + // checks. + m0TableMask = m0TableSize - 1 +) + +func (m M0) hash(data uint64) uint64 { + hash := (data << (64 - 8*m0HashLen)) * hashMul64 + return hash >> (64 - m0TableBits) +} + +// FindMatches looks for matches in src, appends them to dst, and returns dst. +// src must not be longer than 65536 bytes. +func (m M0) FindMatches(dst []Match, src []byte) []Match { + const inputMargin = 16 - 1 + const minNonLiteralBlockSize = 1 + 1 + inputMargin + + if len(src) < minNonLiteralBlockSize { + dst = append(dst, Match{ + Unmatched: len(src), + }) + return dst + } + if len(src) > 65536 { + panic("block too long") + } + + var table [m0TableSize]uint16 + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := len(src) - inputMargin + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := 0 + + // The encoded form must start with a literal, as there are no previous + // bytes to copy, so we start looking for hash matches at s == 1. + s := 1 + nextHash := m.hash(binary.LittleEndian.Uint64(src[s:])) + + for { + // Copied from the C++ snappy implementation: + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match + // is found, immediately go back to looking at every byte. This is a + // small loss (~5% performance, ~0.1% density) for compressible data + // due to more bookkeeping, but for non-compressible data (such as + // JPEG) it's a huge win since the compressor quickly "realizes" the + // data is incompressible and doesn't bother looking for matches + // everywhere. + // + // The "skip" variable keeps track of how many bytes there are since + // the last match; dividing it by 32 (ie. right-shifting by five) gives + // the number of bytes to move ahead for each iteration. + skip := 32 + + nextS := s + candidate := 0 + for { + s = nextS + bytesBetweenHashLookups := skip >> 5 + nextS = s + bytesBetweenHashLookups + skip += bytesBetweenHashLookups + if nextS > sLimit { + goto emitRemainder + } + candidate = int(table[nextHash&m0TableMask]) + table[nextHash&m0TableMask] = uint16(s) + nextHash = m.hash(binary.LittleEndian.Uint64(src[nextS:])) + if m.MaxDistance != 0 && s-candidate > m.MaxDistance { + continue + } + if binary.LittleEndian.Uint32(src[s:]) == binary.LittleEndian.Uint32(src[candidate:]) { + break + } + } + + // Invariant: we have a 4-byte match at s. + base := s + s = extendMatch(src, candidate+4, s+4) + + origBase := base + if m.Lazy && base+1 < sLimit { + newBase := base + 1 + h := m.hash(binary.LittleEndian.Uint64(src[newBase:])) + newCandidate := int(table[h&m0TableMask]) + table[h&m0TableMask] = uint16(newBase) + okDistance := true + if m.MaxDistance != 0 && newBase-newCandidate > m.MaxDistance { + okDistance = false + } + if okDistance && binary.LittleEndian.Uint32(src[newBase:]) == binary.LittleEndian.Uint32(src[newCandidate:]) { + newS := extendMatch(src, newCandidate+4, newBase+4) + if newS-newBase > s-base+1 { + s = newS + base = newBase + candidate = newCandidate + } + } + } + + if m.MaxLength != 0 && s-base > m.MaxLength { + s = base + m.MaxLength + } + dst = append(dst, Match{ + Unmatched: base - nextEmit, + Length: s - base, + Distance: base - candidate, + }) + nextEmit = s + if s >= sLimit { + goto emitRemainder + } + + if m.Lazy { + // If lazy matching is enabled, we update the hash table for + // every byte in the match. + for i := origBase + 2; i < s-1; i++ { + x := binary.LittleEndian.Uint64(src[i:]) + table[m.hash(x)&m0TableMask] = uint16(i) + } + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. + x := binary.LittleEndian.Uint64(src[s-1:]) + prevHash := m.hash(x >> 0) + table[prevHash&m0TableMask] = uint16(s - 1) + nextHash = m.hash(x >> 8) + } + +emitRemainder: + if nextEmit < len(src) { + dst = append(dst, Match{ + Unmatched: len(src) - nextEmit, + }) + } + return dst +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/m4.go b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go new file mode 100644 index 0000000000..5b2acba2e1 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/m4.go @@ -0,0 +1,297 @@ +package matchfinder + +import ( + "encoding/binary" + "math/bits" + "runtime" +) + +// M4 is an implementation of the MatchFinder +// interface that uses a hash table to find matches, +// optional match chains, +// and the advanced parsing technique from +// https://fastcompression.blogspot.com/2011/12/advanced-parsing-strategies.html. +type M4 struct { + // MaxDistance is the maximum distance (in bytes) to look back for + // a match. The default is 65535. + MaxDistance int + + // MinLength is the length of the shortest match to return. + // The default is 4. + MinLength int + + // HashLen is the number of bytes to use to calculate the hashes. + // The maximum is 8 and the default is 6. + HashLen int + + // TableBits is the number of bits in the hash table indexes. + // The default is 17 (128K entries). + TableBits int + + // ChainLength is how many entries to search on the "match chain" of older + // locations with the same hash as the current location. + ChainLength int + + // DistanceBitCost is used when comparing two matches to see + // which is better. The comparison is primarily based on the length + // of the matches, but it can also take the distance into account, + // in terms of the number of bits needed to represent the distance. + // One byte of length is given a score of 256, so 32 (256/8) would + // be a reasonable first guess for the value of one bit. + // (The default is 0, which bases the comparison solely on length.) + DistanceBitCost int + + table []uint32 + chain []uint16 + + history []byte +} + +func (q *M4) Reset() { + for i := range q.table { + q.table[i] = 0 + } + q.history = q.history[:0] + q.chain = q.chain[:0] +} + +func (q *M4) score(m absoluteMatch) int { + return (m.End-m.Start)*256 + bits.LeadingZeros32(uint32(m.Start-m.Match))*q.DistanceBitCost +} + +func (q *M4) FindMatches(dst []Match, src []byte) []Match { + if q.MaxDistance == 0 { + q.MaxDistance = 65535 + } + if q.MinLength == 0 { + q.MinLength = 4 + } + if q.HashLen == 0 { + q.HashLen = 6 + } + if q.TableBits == 0 { + q.TableBits = 17 + } + if len(q.table) < 1< q.MaxDistance*2 { + // Trim down the history buffer. + delta := len(q.history) - q.MaxDistance + copy(q.history, q.history[delta:]) + q.history = q.history[:q.MaxDistance] + if q.ChainLength > 0 { + q.chain = q.chain[:q.MaxDistance] + } + + for i, v := range q.table { + newV := int(v) - delta + if newV < 0 { + newV = 0 + } + q.table[i] = uint32(newV) + } + } + + // Append src to the history buffer. + e.NextEmit = len(q.history) + q.history = append(q.history, src...) + if q.ChainLength > 0 { + q.chain = append(q.chain, make([]uint16, len(src))...) + } + src = q.history + + // matches stores the matches that have been found but not emitted, + // in reverse order. (matches[0] is the most recent one.) + var matches [3]absoluteMatch + for i := e.NextEmit; i < len(src)-7; i++ { + if matches[0] != (absoluteMatch{}) && i >= matches[0].End { + // We have found some matches, and we're far enough along that we probably + // won't find overlapping matches, so we might as well emit them. + if matches[1] != (absoluteMatch{}) { + e.trim(matches[1], matches[0].Start, q.MinLength) + } + e.emit(matches[0]) + matches = [3]absoluteMatch{} + } + + // Calculate and store the hash. + h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) + candidate := int(q.table[h]) + q.table[h] = uint32(i) + if q.ChainLength > 0 && candidate != 0 { + delta := i - candidate + if delta < 1<<16 { + q.chain[i] = uint16(delta) + } + } + + if i < matches[0].End && i != matches[0].End+2-q.HashLen { + continue + } + if candidate == 0 || i-candidate > q.MaxDistance { + continue + } + + // Look for a match. + var currentMatch absoluteMatch + + if i-candidate != matches[0].Start-matches[0].Match { + if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + m := extendMatch2(src, i, candidate, e.NextEmit) + if m.End-m.Start > q.MinLength { + currentMatch = m + } + } + } + + for j := 0; j < q.ChainLength; j++ { + delta := q.chain[candidate] + if delta == 0 { + break + } + candidate -= int(delta) + if candidate <= 0 || i-candidate > q.MaxDistance { + break + } + if i-candidate != matches[0].Start-matches[0].Match { + if binary.LittleEndian.Uint32(src[candidate:]) == binary.LittleEndian.Uint32(src[i:]) { + m := extendMatch2(src, i, candidate, e.NextEmit) + if m.End-m.Start > q.MinLength && q.score(m) > q.score(currentMatch) { + currentMatch = m + } + } + } + } + + if currentMatch.End-currentMatch.Start < q.MinLength { + continue + } + + overlapPenalty := 0 + if matches[0] != (absoluteMatch{}) { + overlapPenalty = 275 + if currentMatch.Start <= matches[1].End { + // This match would completely replace the previous match, + // so there is no penalty for overlap. + overlapPenalty = 0 + } + } + + if q.score(currentMatch) <= q.score(matches[0])+overlapPenalty { + continue + } + + matches = [3]absoluteMatch{ + currentMatch, + matches[0], + matches[1], + } + + if matches[2] == (absoluteMatch{}) { + continue + } + + // We have three matches, so it's time to emit one and/or eliminate one. + switch { + case matches[0].Start < matches[2].End: + // The first and third matches overlap; discard the one in between. + matches = [3]absoluteMatch{ + matches[0], + matches[2], + absoluteMatch{}, + } + + case matches[0].Start < matches[2].End+q.MinLength: + // The first and third matches don't overlap, but there's no room for + // another match between them. Emit the first match and discard the second. + e.emit(matches[2]) + matches = [3]absoluteMatch{ + matches[0], + absoluteMatch{}, + absoluteMatch{}, + } + + default: + // Emit the first match, shortening it if necessary to avoid overlap with the second. + e.trim(matches[2], matches[1].Start, q.MinLength) + matches[2] = absoluteMatch{} + } + } + + // We've found all the matches now; emit the remaining ones. + if matches[1] != (absoluteMatch{}) { + e.trim(matches[1], matches[0].Start, q.MinLength) + } + if matches[0] != (absoluteMatch{}) { + e.emit(matches[0]) + } + + dst = e.Dst + if e.NextEmit < len(src) { + dst = append(dst, Match{ + Unmatched: len(src) - e.NextEmit, + }) + } + + return dst +} + +const hashMul64 = 0x1E35A7BD1E35A7BD + +// extendMatch returns the largest k such that k <= len(src) and that +// src[i:i+k-j] and src[j:k] have the same contents. +// +// It assumes that: +// +// 0 <= i && i < j && j <= len(src) +func extendMatch(src []byte, i, j int) int { + switch runtime.GOARCH { + case "amd64": + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + for j+8 < len(src) { + iBytes := binary.LittleEndian.Uint64(src[i:]) + jBytes := binary.LittleEndian.Uint64(src[j:]) + if iBytes != jBytes { + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. The BSF instruction finds the + // least significant 1 bit, the amd64 architecture is little-endian, and + // the shift by 3 converts a bit index to a byte index. + return j + bits.TrailingZeros64(iBytes^jBytes)>>3 + } + i, j = i+8, j+8 + } + case "386": + // On a 32-bit CPU, we do it 4 bytes at a time. + for j+4 < len(src) { + iBytes := binary.LittleEndian.Uint32(src[i:]) + jBytes := binary.LittleEndian.Uint32(src[j:]) + if iBytes != jBytes { + return j + bits.TrailingZeros32(iBytes^jBytes)>>3 + } + i, j = i+4, j+4 + } + } + for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { + } + return j +} + +// Given a 4-byte match at src[start] and src[candidate], extendMatch2 extends it +// upward as far as possible, and downward no farther than to min. +func extendMatch2(src []byte, start, candidate, min int) absoluteMatch { + end := extendMatch(src, candidate+4, start+4) + for start > min && candidate > 0 && src[start-1] == src[candidate-1] { + start-- + candidate-- + } + return absoluteMatch{ + Start: start, + End: end, + Match: candidate, + } +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go b/vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go new file mode 100644 index 0000000000..f6bcfdb39c --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/matchfinder.go @@ -0,0 +1,103 @@ +// The matchfinder package defines reusable components for data compression. +// +// Many compression libraries have two main parts: +// - Something that looks for repeated sequences of bytes +// - An encoder for the compressed data format (often an entropy coder) +// +// Although these are logically two separate steps, the implementations are +// usually closely tied together. You can't use flate's matcher with snappy's +// encoder, for example. This package defines interfaces and an intermediate +// representation to allow mixing and matching compression components. +package matchfinder + +import "io" + +// A Match is the basic unit of LZ77 compression. +type Match struct { + Unmatched int // the number of unmatched bytes since the previous match + Length int // the number of bytes in the matched string; it may be 0 at the end of the input + Distance int // how far back in the stream to copy from +} + +// A MatchFinder performs the LZ77 stage of compression, looking for matches. +type MatchFinder interface { + // FindMatches looks for matches in src, appends them to dst, and returns dst. + FindMatches(dst []Match, src []byte) []Match + + // Reset clears any internal state, preparing the MatchFinder to be used with + // a new stream. + Reset() +} + +// An Encoder encodes the data in its final format. +type Encoder interface { + // Encode appends the encoded format of src to dst, using the match + // information from matches. + Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte + + // Reset clears any internal state, preparing the Encoder to be used with + // a new stream. + Reset() +} + +// A Writer uses MatchFinder and Encoder to write compressed data to Dest. +type Writer struct { + Dest io.Writer + MatchFinder MatchFinder + Encoder Encoder + + // BlockSize is the number of bytes to compress at a time. If it is zero, + // each Write operation will be treated as one block. + BlockSize int + + err error + inBuf []byte + outBuf []byte + matches []Match +} + +func (w *Writer) Write(p []byte) (n int, err error) { + if w.err != nil { + return 0, w.err + } + + if w.BlockSize == 0 { + return w.writeBlock(p, false) + } + + w.inBuf = append(w.inBuf, p...) + var pos int + for pos = 0; pos+w.BlockSize <= len(w.inBuf) && w.err == nil; pos += w.BlockSize { + w.writeBlock(w.inBuf[pos:pos+w.BlockSize], false) + } + if pos > 0 { + n := copy(w.inBuf, w.inBuf[pos:]) + w.inBuf = w.inBuf[:n] + } + + return len(p), w.err +} + +func (w *Writer) writeBlock(p []byte, lastBlock bool) (n int, err error) { + w.outBuf = w.outBuf[:0] + w.matches = w.MatchFinder.FindMatches(w.matches[:0], p) + w.outBuf = w.Encoder.Encode(w.outBuf, p, w.matches, lastBlock) + _, w.err = w.Dest.Write(w.outBuf) + return len(p), w.err +} + +func (w *Writer) Close() error { + w.writeBlock(w.inBuf, true) + w.inBuf = w.inBuf[:0] + return w.err +} + +func (w *Writer) Reset(newDest io.Writer) { + w.MatchFinder.Reset() + w.Encoder.Reset() + w.err = nil + w.inBuf = w.inBuf[:0] + w.outBuf = w.outBuf[:0] + w.matches = w.matches[:0] + w.Dest = newDest +} diff --git a/vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go b/vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go new file mode 100644 index 0000000000..75ecc5908b --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/matchfinder/textencoder.go @@ -0,0 +1,53 @@ +package matchfinder + +import "fmt" + +// A TextEncoder is an Encoder that produces a human-readable representation of +// the LZ77 compression. Matches are replaced with symbols. +type TextEncoder struct{} + +func (t TextEncoder) Reset() {} + +func (t TextEncoder) Encode(dst []byte, src []byte, matches []Match, lastBlock bool) []byte { + pos := 0 + for _, m := range matches { + if m.Unmatched > 0 { + dst = append(dst, src[pos:pos+m.Unmatched]...) + pos += m.Unmatched + } + if m.Length > 0 { + dst = append(dst, []byte(fmt.Sprintf("<%d,%d>", m.Length, m.Distance))...) + pos += m.Length + } + } + if pos < len(src) { + dst = append(dst, src[pos:]...) + } + return dst +} + +// A NoMatchFinder implements MatchFinder, but doesn't find any matches. +// It can be used to implement the equivalent of the standard library flate package's +// HuffmanOnly setting. +type NoMatchFinder struct{} + +func (n NoMatchFinder) Reset() {} + +func (n NoMatchFinder) FindMatches(dst []Match, src []byte) []Match { + return append(dst, Match{ + Unmatched: len(src), + }) +} + +// AutoReset wraps a MatchFinder that can return references to data in previous +// blocks, and calls Reset before each block. It is useful for (e.g.) using a +// snappy Encoder with a MatchFinder designed for flate. (Snappy doesn't +// support references between blocks.) +type AutoReset struct { + MatchFinder +} + +func (a AutoReset) FindMatches(dst []Match, src []byte) []Match { + a.Reset() + return a.MatchFinder.FindMatches(dst, src) +} diff --git a/vendor/github.com/andybalholm/brotli/memory.go b/vendor/github.com/andybalholm/brotli/memory.go new file mode 100644 index 0000000000..a07c7050a0 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/memory.go @@ -0,0 +1,66 @@ +package brotli + +/* Copyright 2016 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* +Dynamically grows array capacity to at least the requested size +T: data type +A: array +C: capacity +R: requested size +*/ +func brotli_ensure_capacity_uint8_t(a *[]byte, c *uint, r uint) { + if *c < r { + var new_size uint = *c + if new_size == 0 { + new_size = r + } + + for new_size < r { + new_size *= 2 + } + + if cap(*a) < int(new_size) { + var new_array []byte = make([]byte, new_size) + if *c != 0 { + copy(new_array, (*a)[:*c]) + } + + *a = new_array + } else { + *a = (*a)[:new_size] + } + + *c = new_size + } +} + +func brotli_ensure_capacity_uint32_t(a *[]uint32, c *uint, r uint) { + var new_array []uint32 + if *c < r { + var new_size uint = *c + if new_size == 0 { + new_size = r + } + + for new_size < r { + new_size *= 2 + } + + if cap(*a) < int(new_size) { + new_array = make([]uint32, new_size) + if *c != 0 { + copy(new_array, (*a)[:*c]) + } + + *a = new_array + } else { + *a = (*a)[:new_size] + } + *c = new_size + } +} diff --git a/vendor/github.com/andybalholm/brotli/metablock.go b/vendor/github.com/andybalholm/brotli/metablock.go new file mode 100644 index 0000000000..3014df8cdf --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/metablock.go @@ -0,0 +1,574 @@ +package brotli + +import ( + "sync" +) + +/* Copyright 2014 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Algorithms for distributing the literals and commands of a metablock between + block types and contexts. */ + +type metaBlockSplit struct { + literal_split blockSplit + command_split blockSplit + distance_split blockSplit + literal_context_map []uint32 + literal_context_map_size uint + distance_context_map []uint32 + distance_context_map_size uint + literal_histograms []histogramLiteral + literal_histograms_size uint + command_histograms []histogramCommand + command_histograms_size uint + distance_histograms []histogramDistance + distance_histograms_size uint +} + +var metaBlockPool sync.Pool + +func getMetaBlockSplit() *metaBlockSplit { + mb, _ := metaBlockPool.Get().(*metaBlockSplit) + + if mb == nil { + mb = &metaBlockSplit{} + } else { + initBlockSplit(&mb.literal_split) + initBlockSplit(&mb.command_split) + initBlockSplit(&mb.distance_split) + mb.literal_context_map = mb.literal_context_map[:0] + mb.literal_context_map_size = 0 + mb.distance_context_map = mb.distance_context_map[:0] + mb.distance_context_map_size = 0 + mb.literal_histograms = mb.literal_histograms[:0] + mb.command_histograms = mb.command_histograms[:0] + mb.distance_histograms = mb.distance_histograms[:0] + } + return mb +} + +func freeMetaBlockSplit(mb *metaBlockSplit) { + metaBlockPool.Put(mb) +} + +func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) { + var dist_params *distanceParams = ¶ms.dist + var alphabet_size uint32 + var max_distance uint32 + + dist_params.distance_postfix_bits = npostfix + dist_params.num_direct_distance_codes = ndirect + + alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), maxDistanceBits)) + max_distance = ndirect + (1 << (maxDistanceBits + npostfix + 2)) - (1 << (npostfix + 2)) + + if params.large_window { + var bound = [maxNpostfix + 1]uint32{0, 4, 12, 28} + var postfix uint32 = 1 << npostfix + alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), largeMaxDistanceBits)) + + /* The maximum distance is set so that no distance symbol used can encode + a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all + its extra bits set. */ + if ndirect < bound[npostfix] { + max_distance = maxAllowedDistance - (bound[npostfix] - ndirect) + } else if ndirect >= bound[npostfix]+postfix { + max_distance = (3 << 29) - 4 + (ndirect - bound[npostfix]) + } else { + max_distance = maxAllowedDistance + } + } + + dist_params.alphabet_size = alphabet_size + dist_params.max_distance = uint(max_distance) +} + +func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) { + if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { + return + } + + for i := range cmds { + var cmd *command = &cmds[i] + if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { + prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) + } + } +} + +func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { + var equal_params bool = false + var dist_prefix uint16 + var dist_extra uint32 + var extra_bits float64 = 0.0 + var histo histogramDistance + histogramClearDistance(&histo) + + if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { + equal_params = true + } + + for i := range cmds { + cmd := &cmds[i] + if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { + if equal_params { + dist_prefix = cmd.dist_prefix_ + } else { + var distance uint32 = commandRestoreDistanceCode(cmd, orig_params) + if distance > uint32(new_params.max_distance) { + return false + } + + prefixEncodeCopyDistance(uint(distance), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &dist_prefix, &dist_extra) + } + + histogramAddDistance(&histo, uint(dist_prefix)&0x3FF) + extra_bits += float64(dist_prefix >> 10) + } + } + + *cost = populationCostDistance(&histo) + extra_bits + return true +} + +var buildMetaBlock_kMaxNumberOfHistograms uint = 256 + +func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) { + var distance_histograms []histogramDistance + var literal_histograms []histogramLiteral + var literal_context_modes []int = nil + var literal_histograms_size uint + var distance_histograms_size uint + var i uint + var literal_context_multiplier uint = 1 + var npostfix uint32 + var ndirect_msb uint32 = 0 + var check_orig bool = true + var best_dist_cost float64 = 1e99 + var orig_params encoderParams = *params + /* Histogram ids need to fit in one byte. */ + + var new_params encoderParams = *params + + for npostfix = 0; npostfix <= maxNpostfix; npostfix++ { + for ; ndirect_msb < 16; ndirect_msb++ { + var ndirect uint32 = ndirect_msb << npostfix + var skip bool + var dist_cost float64 + initDistanceParams(&new_params, npostfix, ndirect) + if npostfix == orig_params.dist.distance_postfix_bits && ndirect == orig_params.dist.num_direct_distance_codes { + check_orig = false + } + + skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost) + if skip || (dist_cost > best_dist_cost) { + break + } + + best_dist_cost = dist_cost + params.dist = new_params.dist + } + + if ndirect_msb > 0 { + ndirect_msb-- + } + ndirect_msb /= 2 + } + + if check_orig { + var dist_cost float64 + computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost) + if dist_cost < best_dist_cost { + /* NB: currently unused; uncomment when more param tuning is added. */ + /* best_dist_cost = dist_cost; */ + params.dist = orig_params.dist + } + } + + recomputeDistancePrefixes(cmds, &orig_params.dist, ¶ms.dist) + + splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) + + if !params.disable_literal_context_modeling { + literal_context_multiplier = 1 << literalContextBits + literal_context_modes = make([]int, (mb.literal_split.num_types)) + for i = 0; i < mb.literal_split.num_types; i++ { + literal_context_modes[i] = literal_context_mode + } + } + + literal_histograms_size = mb.literal_split.num_types * literal_context_multiplier + literal_histograms = make([]histogramLiteral, literal_histograms_size) + clearHistogramsLiteral(literal_histograms, literal_histograms_size) + + distance_histograms_size = mb.distance_split.num_types << distanceContextBits + distance_histograms = make([]histogramDistance, distance_histograms_size) + clearHistogramsDistance(distance_histograms, distance_histograms_size) + + mb.command_histograms_size = mb.command_split.num_types + if cap(mb.command_histograms) < int(mb.command_histograms_size) { + mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) + } else { + mb.command_histograms = mb.command_histograms[:mb.command_histograms_size] + } + clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) + + buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) + literal_context_modes = nil + + mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits + if cap(mb.literal_context_map) < int(mb.literal_context_map_size) { + mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + } else { + mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size] + } + + mb.literal_histograms_size = mb.literal_context_map_size + if cap(mb.literal_histograms) < int(mb.literal_histograms_size) { + mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size)) + } else { + mb.literal_histograms = mb.literal_histograms[:mb.literal_histograms_size] + } + + clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map) + literal_histograms = nil + + if params.disable_literal_context_modeling { + /* Distribute assignment to all contexts. */ + for i = mb.literal_split.num_types; i != 0; { + var j uint = 0 + i-- + for ; j < 1< 0 { + var entropy [maxStaticContexts]float64 + var combined_histo []histogramLiteral = make([]histogramLiteral, (2 * num_contexts)) + var combined_entropy [2 * maxStaticContexts]float64 + var diff = [2]float64{0.0} + /* Try merging the set of histograms for the current block type with the + respective set of histograms for the last and second last block types. + Decide over the split based on the total reduction of entropy across + all contexts. */ + + var i uint + for i = 0; i < num_contexts; i++ { + var curr_histo_ix uint = self.curr_histogram_ix_ + i + var j uint + entropy[i] = bitsEntropy(histograms[curr_histo_ix].data_[:], self.alphabet_size_) + for j = 0; j < 2; j++ { + var jx uint = j*num_contexts + i + var last_histogram_ix uint = self.last_histogram_ix_[j] + i + combined_histo[jx] = histograms[curr_histo_ix] + histogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix]) + combined_entropy[jx] = bitsEntropy(combined_histo[jx].data_[0:], self.alphabet_size_) + diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx] + } + } + + if split.num_types < self.max_block_types_ && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { + /* Create new block. */ + split.lengths[self.num_blocks_] = uint32(self.block_size_) + + split.types[self.num_blocks_] = byte(split.num_types) + self.last_histogram_ix_[1] = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = split.num_types * num_contexts + for i = 0; i < num_contexts; i++ { + last_entropy[num_contexts+i] = last_entropy[i] + last_entropy[i] = entropy[i] + } + + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_ += num_contexts + if self.curr_histogram_ix_ < *self.histograms_size_ { + clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_) + } + + self.block_size_ = 0 + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else if diff[1] < diff[0]-20.0 { + split.lengths[self.num_blocks_] = uint32(self.block_size_) + split.types[self.num_blocks_] = split.types[self.num_blocks_-2] + /* Combine this block with second last block. */ + + var tmp uint = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] + self.last_histogram_ix_[1] = tmp + for i = 0; i < num_contexts; i++ { + histograms[self.last_histogram_ix_[0]+i] = combined_histo[num_contexts+i] + last_entropy[num_contexts+i] = last_entropy[i] + last_entropy[i] = combined_entropy[num_contexts+i] + histogramClearLiteral(&histograms[self.curr_histogram_ix_+i]) + } + + self.num_blocks_++ + self.block_size_ = 0 + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else { + /* Combine this block with last block. */ + split.lengths[self.num_blocks_-1] += uint32(self.block_size_) + + for i = 0; i < num_contexts; i++ { + histograms[self.last_histogram_ix_[0]+i] = combined_histo[i] + last_entropy[i] = combined_entropy[i] + if split.num_types == 1 { + last_entropy[num_contexts+i] = last_entropy[i] + } + + histogramClearLiteral(&histograms[self.curr_histogram_ix_+i]) + } + + self.block_size_ = 0 + self.merge_last_count_++ + if self.merge_last_count_ > 1 { + self.target_block_size_ += self.min_block_size_ + } + } + + combined_histo = nil + } + + if is_final { + *self.histograms_size_ = split.num_types * num_contexts + split.num_blocks = self.num_blocks_ + } +} + +/* Adds the next symbol to the current block type and context. When the + current block reaches the target size, decides on merging the block. */ +func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, context uint) { + histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol) + self.block_size_++ + if self.block_size_ == self.target_block_size_ { + contextBlockSplitterFinishBlock(self, false) /* is_final = */ + } +} + +func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) { + var i uint + mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits + if cap(mb.literal_context_map) < int(mb.literal_context_map_size) { + mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) + } else { + mb.literal_context_map = mb.literal_context_map[:mb.literal_context_map_size] + } + + for i = 0; i < mb.literal_split.num_types; i++ { + var offset uint32 = uint32(i * num_contexts) + var j uint + for j = 0; j < 1<= 128 { + blockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF) + } + } + } + + if num_contexts == 1 { + blockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */ + } else { + contextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */ + } + + blockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */ + blockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */ + + if num_contexts > 1 { + mapStaticContexts(num_contexts, static_context_map, mb) + } +} + +func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { + if num_contexts == 1 { + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb) + } else { + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb) + } +} + +func optimizeHistograms(num_distance_codes uint32, mb *metaBlockSplit) { + var good_for_rle [numCommandSymbols]byte + var i uint + for i = 0; i < mb.literal_histograms_size; i++ { + optimizeHuffmanCountsForRLE(256, mb.literal_histograms[i].data_[:], good_for_rle[:]) + } + + for i = 0; i < mb.command_histograms_size; i++ { + optimizeHuffmanCountsForRLE(numCommandSymbols, mb.command_histograms[i].data_[:], good_for_rle[:]) + } + + for i = 0; i < mb.distance_histograms_size; i++ { + optimizeHuffmanCountsForRLE(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:]) + } +} diff --git a/vendor/github.com/andybalholm/brotli/metablock_command.go b/vendor/github.com/andybalholm/brotli/metablock_command.go new file mode 100644 index 0000000000..14c7b77135 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/metablock_command.go @@ -0,0 +1,165 @@ +package brotli + +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Greedy block splitter for one block category (literal, command or distance). + */ +type blockSplitterCommand struct { + alphabet_size_ uint + min_block_size_ uint + split_threshold_ float64 + num_blocks_ uint + split_ *blockSplit + histograms_ []histogramCommand + histograms_size_ *uint + target_block_size_ uint + block_size_ uint + curr_histogram_ix_ uint + last_histogram_ix_ [2]uint + last_entropy_ [2]float64 + merge_last_count_ uint +} + +func initBlockSplitterCommand(self *blockSplitterCommand, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramCommand, histograms_size *uint) { + var max_num_blocks uint = num_symbols/min_block_size + 1 + var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1) + /* We have to allocate one more histogram than the maximum number of block + types for the current histogram when the meta-block is too big. */ + self.alphabet_size_ = alphabet_size + + self.min_block_size_ = min_block_size + self.split_threshold_ = split_threshold + self.num_blocks_ = 0 + self.split_ = split + self.histograms_size_ = histograms_size + self.target_block_size_ = min_block_size + self.block_size_ = 0 + self.curr_histogram_ix_ = 0 + self.merge_last_count_ = 0 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) + self.split_.num_blocks = max_num_blocks + *histograms_size = max_num_types + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramCommand, (*histograms_size)) + } else { + *histograms = (*histograms)[:*histograms_size] + } + self.histograms_ = *histograms + + /* Clear only current histogram. */ + histogramClearCommand(&self.histograms_[0]) + + self.last_histogram_ix_[1] = 0 + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] +} + +/* Does either of three things: + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. */ +func blockSplitterFinishBlockCommand(self *blockSplitterCommand, is_final bool) { + var split *blockSplit = self.split_ + var last_entropy []float64 = self.last_entropy_[:] + var histograms []histogramCommand = self.histograms_ + self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_) + if self.num_blocks_ == 0 { + /* Create first block. */ + split.lengths[0] = uint32(self.block_size_) + + split.types[0] = 0 + last_entropy[0] = bitsEntropy(histograms[0].data_[:], self.alphabet_size_) + last_entropy[1] = last_entropy[0] + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_++ + if self.curr_histogram_ix_ < *self.histograms_size_ { + histogramClearCommand(&histograms[self.curr_histogram_ix_]) + } + self.block_size_ = 0 + } else if self.block_size_ > 0 { + var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_) + var combined_histo [2]histogramCommand + var combined_entropy [2]float64 + var diff [2]float64 + var j uint + for j = 0; j < 2; j++ { + var last_histogram_ix uint = self.last_histogram_ix_[j] + combined_histo[j] = histograms[self.curr_histogram_ix_] + histogramAddHistogramCommand(&combined_histo[j], &histograms[last_histogram_ix]) + combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_) + diff[j] = combined_entropy[j] - entropy - last_entropy[j] + } + + if split.num_types < maxNumberOfBlockTypes && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { + /* Create new block. */ + split.lengths[self.num_blocks_] = uint32(self.block_size_) + + split.types[self.num_blocks_] = byte(split.num_types) + self.last_histogram_ix_[1] = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = uint(byte(split.num_types)) + last_entropy[1] = last_entropy[0] + last_entropy[0] = entropy + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_++ + if self.curr_histogram_ix_ < *self.histograms_size_ { + histogramClearCommand(&histograms[self.curr_histogram_ix_]) + } + self.block_size_ = 0 + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else if diff[1] < diff[0]-20.0 { + split.lengths[self.num_blocks_] = uint32(self.block_size_) + split.types[self.num_blocks_] = split.types[self.num_blocks_-2] + /* Combine this block with second last block. */ + + var tmp uint = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] + self.last_histogram_ix_[1] = tmp + histograms[self.last_histogram_ix_[0]] = combined_histo[1] + last_entropy[1] = last_entropy[0] + last_entropy[0] = combined_entropy[1] + self.num_blocks_++ + self.block_size_ = 0 + histogramClearCommand(&histograms[self.curr_histogram_ix_]) + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else { + /* Combine this block with last block. */ + split.lengths[self.num_blocks_-1] += uint32(self.block_size_) + + histograms[self.last_histogram_ix_[0]] = combined_histo[0] + last_entropy[0] = combined_entropy[0] + if split.num_types == 1 { + last_entropy[1] = last_entropy[0] + } + + self.block_size_ = 0 + histogramClearCommand(&histograms[self.curr_histogram_ix_]) + self.merge_last_count_++ + if self.merge_last_count_ > 1 { + self.target_block_size_ += self.min_block_size_ + } + } + } + + if is_final { + *self.histograms_size_ = split.num_types + split.num_blocks = self.num_blocks_ + } +} + +/* Adds the next symbol to the current histogram. When the current histogram + reaches the target size, decides on merging the block. */ +func blockSplitterAddSymbolCommand(self *blockSplitterCommand, symbol uint) { + histogramAddCommand(&self.histograms_[self.curr_histogram_ix_], symbol) + self.block_size_++ + if self.block_size_ == self.target_block_size_ { + blockSplitterFinishBlockCommand(self, false) /* is_final = */ + } +} diff --git a/vendor/github.com/andybalholm/brotli/metablock_distance.go b/vendor/github.com/andybalholm/brotli/metablock_distance.go new file mode 100644 index 0000000000..5110a810e9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/metablock_distance.go @@ -0,0 +1,165 @@ +package brotli + +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Greedy block splitter for one block category (literal, command or distance). + */ +type blockSplitterDistance struct { + alphabet_size_ uint + min_block_size_ uint + split_threshold_ float64 + num_blocks_ uint + split_ *blockSplit + histograms_ []histogramDistance + histograms_size_ *uint + target_block_size_ uint + block_size_ uint + curr_histogram_ix_ uint + last_histogram_ix_ [2]uint + last_entropy_ [2]float64 + merge_last_count_ uint +} + +func initBlockSplitterDistance(self *blockSplitterDistance, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramDistance, histograms_size *uint) { + var max_num_blocks uint = num_symbols/min_block_size + 1 + var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1) + /* We have to allocate one more histogram than the maximum number of block + types for the current histogram when the meta-block is too big. */ + self.alphabet_size_ = alphabet_size + + self.min_block_size_ = min_block_size + self.split_threshold_ = split_threshold + self.num_blocks_ = 0 + self.split_ = split + self.histograms_size_ = histograms_size + self.target_block_size_ = min_block_size + self.block_size_ = 0 + self.curr_histogram_ix_ = 0 + self.merge_last_count_ = 0 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) + self.split_.num_blocks = max_num_blocks + *histograms_size = max_num_types + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramDistance, *histograms_size) + } else { + *histograms = (*histograms)[:*histograms_size] + } + self.histograms_ = *histograms + + /* Clear only current histogram. */ + histogramClearDistance(&self.histograms_[0]) + + self.last_histogram_ix_[1] = 0 + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] +} + +/* Does either of three things: + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. */ +func blockSplitterFinishBlockDistance(self *blockSplitterDistance, is_final bool) { + var split *blockSplit = self.split_ + var last_entropy []float64 = self.last_entropy_[:] + var histograms []histogramDistance = self.histograms_ + self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_) + if self.num_blocks_ == 0 { + /* Create first block. */ + split.lengths[0] = uint32(self.block_size_) + + split.types[0] = 0 + last_entropy[0] = bitsEntropy(histograms[0].data_[:], self.alphabet_size_) + last_entropy[1] = last_entropy[0] + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_++ + if self.curr_histogram_ix_ < *self.histograms_size_ { + histogramClearDistance(&histograms[self.curr_histogram_ix_]) + } + self.block_size_ = 0 + } else if self.block_size_ > 0 { + var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_) + var combined_histo [2]histogramDistance + var combined_entropy [2]float64 + var diff [2]float64 + var j uint + for j = 0; j < 2; j++ { + var last_histogram_ix uint = self.last_histogram_ix_[j] + combined_histo[j] = histograms[self.curr_histogram_ix_] + histogramAddHistogramDistance(&combined_histo[j], &histograms[last_histogram_ix]) + combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_) + diff[j] = combined_entropy[j] - entropy - last_entropy[j] + } + + if split.num_types < maxNumberOfBlockTypes && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { + /* Create new block. */ + split.lengths[self.num_blocks_] = uint32(self.block_size_) + + split.types[self.num_blocks_] = byte(split.num_types) + self.last_histogram_ix_[1] = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = uint(byte(split.num_types)) + last_entropy[1] = last_entropy[0] + last_entropy[0] = entropy + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_++ + if self.curr_histogram_ix_ < *self.histograms_size_ { + histogramClearDistance(&histograms[self.curr_histogram_ix_]) + } + self.block_size_ = 0 + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else if diff[1] < diff[0]-20.0 { + split.lengths[self.num_blocks_] = uint32(self.block_size_) + split.types[self.num_blocks_] = split.types[self.num_blocks_-2] + /* Combine this block with second last block. */ + + var tmp uint = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] + self.last_histogram_ix_[1] = tmp + histograms[self.last_histogram_ix_[0]] = combined_histo[1] + last_entropy[1] = last_entropy[0] + last_entropy[0] = combined_entropy[1] + self.num_blocks_++ + self.block_size_ = 0 + histogramClearDistance(&histograms[self.curr_histogram_ix_]) + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else { + /* Combine this block with last block. */ + split.lengths[self.num_blocks_-1] += uint32(self.block_size_) + + histograms[self.last_histogram_ix_[0]] = combined_histo[0] + last_entropy[0] = combined_entropy[0] + if split.num_types == 1 { + last_entropy[1] = last_entropy[0] + } + + self.block_size_ = 0 + histogramClearDistance(&histograms[self.curr_histogram_ix_]) + self.merge_last_count_++ + if self.merge_last_count_ > 1 { + self.target_block_size_ += self.min_block_size_ + } + } + } + + if is_final { + *self.histograms_size_ = split.num_types + split.num_blocks = self.num_blocks_ + } +} + +/* Adds the next symbol to the current histogram. When the current histogram + reaches the target size, decides on merging the block. */ +func blockSplitterAddSymbolDistance(self *blockSplitterDistance, symbol uint) { + histogramAddDistance(&self.histograms_[self.curr_histogram_ix_], symbol) + self.block_size_++ + if self.block_size_ == self.target_block_size_ { + blockSplitterFinishBlockDistance(self, false) /* is_final = */ + } +} diff --git a/vendor/github.com/andybalholm/brotli/metablock_literal.go b/vendor/github.com/andybalholm/brotli/metablock_literal.go new file mode 100644 index 0000000000..307f8da88f --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/metablock_literal.go @@ -0,0 +1,165 @@ +package brotli + +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Greedy block splitter for one block category (literal, command or distance). + */ +type blockSplitterLiteral struct { + alphabet_size_ uint + min_block_size_ uint + split_threshold_ float64 + num_blocks_ uint + split_ *blockSplit + histograms_ []histogramLiteral + histograms_size_ *uint + target_block_size_ uint + block_size_ uint + curr_histogram_ix_ uint + last_histogram_ix_ [2]uint + last_entropy_ [2]float64 + merge_last_count_ uint +} + +func initBlockSplitterLiteral(self *blockSplitterLiteral, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) { + var max_num_blocks uint = num_symbols/min_block_size + 1 + var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1) + /* We have to allocate one more histogram than the maximum number of block + types for the current histogram when the meta-block is too big. */ + self.alphabet_size_ = alphabet_size + + self.min_block_size_ = min_block_size + self.split_threshold_ = split_threshold + self.num_blocks_ = 0 + self.split_ = split + self.histograms_size_ = histograms_size + self.target_block_size_ = min_block_size + self.block_size_ = 0 + self.curr_histogram_ix_ = 0 + self.merge_last_count_ = 0 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) + self.split_.num_blocks = max_num_blocks + *histograms_size = max_num_types + if histograms == nil || cap(*histograms) < int(*histograms_size) { + *histograms = make([]histogramLiteral, *histograms_size) + } else { + *histograms = (*histograms)[:*histograms_size] + } + self.histograms_ = *histograms + + /* Clear only current histogram. */ + histogramClearLiteral(&self.histograms_[0]) + + self.last_histogram_ix_[1] = 0 + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] +} + +/* Does either of three things: + (1) emits the current block with a new block type; + (2) emits the current block with the type of the second last block; + (3) merges the current block with the last block. */ +func blockSplitterFinishBlockLiteral(self *blockSplitterLiteral, is_final bool) { + var split *blockSplit = self.split_ + var last_entropy []float64 = self.last_entropy_[:] + var histograms []histogramLiteral = self.histograms_ + self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_) + if self.num_blocks_ == 0 { + /* Create first block. */ + split.lengths[0] = uint32(self.block_size_) + + split.types[0] = 0 + last_entropy[0] = bitsEntropy(histograms[0].data_[:], self.alphabet_size_) + last_entropy[1] = last_entropy[0] + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_++ + if self.curr_histogram_ix_ < *self.histograms_size_ { + histogramClearLiteral(&histograms[self.curr_histogram_ix_]) + } + self.block_size_ = 0 + } else if self.block_size_ > 0 { + var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_) + var combined_histo [2]histogramLiteral + var combined_entropy [2]float64 + var diff [2]float64 + var j uint + for j = 0; j < 2; j++ { + var last_histogram_ix uint = self.last_histogram_ix_[j] + combined_histo[j] = histograms[self.curr_histogram_ix_] + histogramAddHistogramLiteral(&combined_histo[j], &histograms[last_histogram_ix]) + combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_) + diff[j] = combined_entropy[j] - entropy - last_entropy[j] + } + + if split.num_types < maxNumberOfBlockTypes && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { + /* Create new block. */ + split.lengths[self.num_blocks_] = uint32(self.block_size_) + + split.types[self.num_blocks_] = byte(split.num_types) + self.last_histogram_ix_[1] = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = uint(byte(split.num_types)) + last_entropy[1] = last_entropy[0] + last_entropy[0] = entropy + self.num_blocks_++ + split.num_types++ + self.curr_histogram_ix_++ + if self.curr_histogram_ix_ < *self.histograms_size_ { + histogramClearLiteral(&histograms[self.curr_histogram_ix_]) + } + self.block_size_ = 0 + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else if diff[1] < diff[0]-20.0 { + split.lengths[self.num_blocks_] = uint32(self.block_size_) + split.types[self.num_blocks_] = split.types[self.num_blocks_-2] + /* Combine this block with second last block. */ + + var tmp uint = self.last_histogram_ix_[0] + self.last_histogram_ix_[0] = self.last_histogram_ix_[1] + self.last_histogram_ix_[1] = tmp + histograms[self.last_histogram_ix_[0]] = combined_histo[1] + last_entropy[1] = last_entropy[0] + last_entropy[0] = combined_entropy[1] + self.num_blocks_++ + self.block_size_ = 0 + histogramClearLiteral(&histograms[self.curr_histogram_ix_]) + self.merge_last_count_ = 0 + self.target_block_size_ = self.min_block_size_ + } else { + /* Combine this block with last block. */ + split.lengths[self.num_blocks_-1] += uint32(self.block_size_) + + histograms[self.last_histogram_ix_[0]] = combined_histo[0] + last_entropy[0] = combined_entropy[0] + if split.num_types == 1 { + last_entropy[1] = last_entropy[0] + } + + self.block_size_ = 0 + histogramClearLiteral(&histograms[self.curr_histogram_ix_]) + self.merge_last_count_++ + if self.merge_last_count_ > 1 { + self.target_block_size_ += self.min_block_size_ + } + } + } + + if is_final { + *self.histograms_size_ = split.num_types + split.num_blocks = self.num_blocks_ + } +} + +/* Adds the next symbol to the current histogram. When the current histogram + reaches the target size, decides on merging the block. */ +func blockSplitterAddSymbolLiteral(self *blockSplitterLiteral, symbol uint) { + histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_], symbol) + self.block_size_++ + if self.block_size_ == self.target_block_size_ { + blockSplitterFinishBlockLiteral(self, false) /* is_final = */ + } +} diff --git a/vendor/github.com/andybalholm/brotli/params.go b/vendor/github.com/andybalholm/brotli/params.go new file mode 100644 index 0000000000..0a4c687521 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/params.go @@ -0,0 +1,37 @@ +package brotli + +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Parameters for the Brotli encoder with chosen quality levels. */ +type hasherParams struct { + type_ int + bucket_bits int + block_bits int + hash_len int + num_last_distances_to_check int +} + +type distanceParams struct { + distance_postfix_bits uint32 + num_direct_distance_codes uint32 + alphabet_size uint32 + max_distance uint +} + +/* Encoding parameters */ +type encoderParams struct { + mode int + quality int + lgwin uint + lgblock int + size_hint uint + disable_literal_context_modeling bool + large_window bool + hasher hasherParams + dist distanceParams + dictionary encoderDictionary +} diff --git a/vendor/github.com/andybalholm/brotli/platform.go b/vendor/github.com/andybalholm/brotli/platform.go new file mode 100644 index 0000000000..4ebfb1528b --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/platform.go @@ -0,0 +1,103 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +func brotli_min_double(a float64, b float64) float64 { + if a < b { + return a + } else { + return b + } +} + +func brotli_max_double(a float64, b float64) float64 { + if a > b { + return a + } else { + return b + } +} + +func brotli_min_float(a float32, b float32) float32 { + if a < b { + return a + } else { + return b + } +} + +func brotli_max_float(a float32, b float32) float32 { + if a > b { + return a + } else { + return b + } +} + +func brotli_min_int(a int, b int) int { + if a < b { + return a + } else { + return b + } +} + +func brotli_max_int(a int, b int) int { + if a > b { + return a + } else { + return b + } +} + +func brotli_min_size_t(a uint, b uint) uint { + if a < b { + return a + } else { + return b + } +} + +func brotli_max_size_t(a uint, b uint) uint { + if a > b { + return a + } else { + return b + } +} + +func brotli_min_uint32_t(a uint32, b uint32) uint32 { + if a < b { + return a + } else { + return b + } +} + +func brotli_max_uint32_t(a uint32, b uint32) uint32 { + if a > b { + return a + } else { + return b + } +} + +func brotli_min_uint8_t(a byte, b byte) byte { + if a < b { + return a + } else { + return b + } +} + +func brotli_max_uint8_t(a byte, b byte) byte { + if a > b { + return a + } else { + return b + } +} diff --git a/vendor/github.com/andybalholm/brotli/prefix.go b/vendor/github.com/andybalholm/brotli/prefix.go new file mode 100644 index 0000000000..484df0d61e --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/prefix.go @@ -0,0 +1,30 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Functions for encoding of integers into prefix codes the amount of extra + bits, and the actual values of the extra bits. */ + +/* Here distance_code is an intermediate code, i.e. one of the special codes or + the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */ +func prefixEncodeCopyDistance(distance_code uint, num_direct_codes uint, postfix_bits uint, code *uint16, extra_bits *uint32) { + if distance_code < numDistanceShortCodes+num_direct_codes { + *code = uint16(distance_code) + *extra_bits = 0 + return + } else { + var dist uint = (uint(1) << (postfix_bits + 2)) + (distance_code - numDistanceShortCodes - num_direct_codes) + var bucket uint = uint(log2FloorNonZero(dist) - 1) + var postfix_mask uint = (1 << postfix_bits) - 1 + var postfix uint = dist & postfix_mask + var prefix uint = (dist >> bucket) & 1 + var offset uint = (2 + prefix) << bucket + var nbits uint = bucket - postfix_bits + *code = uint16(nbits<<10 | (numDistanceShortCodes + num_direct_codes + ((2*(nbits-1) + prefix) << postfix_bits) + postfix)) + *extra_bits = uint32((dist - offset) >> postfix_bits) + } +} diff --git a/vendor/github.com/andybalholm/brotli/prefix_dec.go b/vendor/github.com/andybalholm/brotli/prefix_dec.go new file mode 100644 index 0000000000..183f0d53fe --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/prefix_dec.go @@ -0,0 +1,723 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +type cmdLutElement struct { + insert_len_extra_bits byte + copy_len_extra_bits byte + distance_code int8 + context byte + insert_len_offset uint16 + copy_len_offset uint16 +} + +var kCmdLut = [numCommandSymbols]cmdLutElement{ + cmdLutElement{0x00, 0x00, 0, 0x00, 0x0000, 0x0002}, + cmdLutElement{0x00, 0x00, 0, 0x01, 0x0000, 0x0003}, + cmdLutElement{0x00, 0x00, 0, 0x02, 0x0000, 0x0004}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0005}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0006}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0007}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0008}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0009}, + cmdLutElement{0x00, 0x00, 0, 0x00, 0x0001, 0x0002}, + cmdLutElement{0x00, 0x00, 0, 0x01, 0x0001, 0x0003}, + cmdLutElement{0x00, 0x00, 0, 0x02, 0x0001, 0x0004}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0005}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0006}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0007}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0008}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0009}, + cmdLutElement{0x00, 0x00, 0, 0x00, 0x0002, 0x0002}, + cmdLutElement{0x00, 0x00, 0, 0x01, 0x0002, 0x0003}, + cmdLutElement{0x00, 0x00, 0, 0x02, 0x0002, 0x0004}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0005}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0006}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0007}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0008}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0009}, + cmdLutElement{0x00, 0x00, 0, 0x00, 0x0003, 0x0002}, + cmdLutElement{0x00, 0x00, 0, 0x01, 0x0003, 0x0003}, + cmdLutElement{0x00, 0x00, 0, 0x02, 0x0003, 0x0004}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0005}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0006}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0007}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0008}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0009}, + cmdLutElement{0x00, 0x00, 0, 0x00, 0x0004, 0x0002}, + cmdLutElement{0x00, 0x00, 0, 0x01, 0x0004, 0x0003}, + cmdLutElement{0x00, 0x00, 0, 0x02, 0x0004, 0x0004}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0005}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0006}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0007}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0008}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0009}, + cmdLutElement{0x00, 0x00, 0, 0x00, 0x0005, 0x0002}, + cmdLutElement{0x00, 0x00, 0, 0x01, 0x0005, 0x0003}, + cmdLutElement{0x00, 0x00, 0, 0x02, 0x0005, 0x0004}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0005}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0006}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0007}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0008}, + cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0009}, + cmdLutElement{0x01, 0x00, 0, 0x00, 0x0006, 0x0002}, + cmdLutElement{0x01, 0x00, 0, 0x01, 0x0006, 0x0003}, + cmdLutElement{0x01, 0x00, 0, 0x02, 0x0006, 0x0004}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0005}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0006}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0007}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0008}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0009}, + cmdLutElement{0x01, 0x00, 0, 0x00, 0x0008, 0x0002}, + cmdLutElement{0x01, 0x00, 0, 0x01, 0x0008, 0x0003}, + cmdLutElement{0x01, 0x00, 0, 0x02, 0x0008, 0x0004}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0005}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0006}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0007}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0008}, + cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0009}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0000, 0x000a}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0000, 0x000c}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0000, 0x000e}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0000, 0x0012}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0000, 0x0016}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0000, 0x001e}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0000, 0x0026}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0000, 0x0036}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0001, 0x000a}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0001, 0x000c}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0001, 0x000e}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0001, 0x0012}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0001, 0x0016}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0001, 0x001e}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0001, 0x0026}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0001, 0x0036}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0002, 0x000a}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0002, 0x000c}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0002, 0x000e}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0002, 0x0012}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0002, 0x0016}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0002, 0x001e}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0002, 0x0026}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0002, 0x0036}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0003, 0x000a}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0003, 0x000c}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0003, 0x000e}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0003, 0x0012}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0003, 0x0016}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0003, 0x001e}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0003, 0x0026}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0003, 0x0036}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0004, 0x000a}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0004, 0x000c}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0004, 0x000e}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0004, 0x0012}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0004, 0x0016}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0004, 0x001e}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0004, 0x0026}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0004, 0x0036}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0005, 0x000a}, + cmdLutElement{0x00, 0x01, 0, 0x03, 0x0005, 0x000c}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0005, 0x000e}, + cmdLutElement{0x00, 0x02, 0, 0x03, 0x0005, 0x0012}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0005, 0x0016}, + cmdLutElement{0x00, 0x03, 0, 0x03, 0x0005, 0x001e}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0005, 0x0026}, + cmdLutElement{0x00, 0x04, 0, 0x03, 0x0005, 0x0036}, + cmdLutElement{0x01, 0x01, 0, 0x03, 0x0006, 0x000a}, + cmdLutElement{0x01, 0x01, 0, 0x03, 0x0006, 0x000c}, + cmdLutElement{0x01, 0x02, 0, 0x03, 0x0006, 0x000e}, + cmdLutElement{0x01, 0x02, 0, 0x03, 0x0006, 0x0012}, + cmdLutElement{0x01, 0x03, 0, 0x03, 0x0006, 0x0016}, + cmdLutElement{0x01, 0x03, 0, 0x03, 0x0006, 0x001e}, + cmdLutElement{0x01, 0x04, 0, 0x03, 0x0006, 0x0026}, + cmdLutElement{0x01, 0x04, 0, 0x03, 0x0006, 0x0036}, + cmdLutElement{0x01, 0x01, 0, 0x03, 0x0008, 0x000a}, + cmdLutElement{0x01, 0x01, 0, 0x03, 0x0008, 0x000c}, + cmdLutElement{0x01, 0x02, 0, 0x03, 0x0008, 0x000e}, + cmdLutElement{0x01, 0x02, 0, 0x03, 0x0008, 0x0012}, + cmdLutElement{0x01, 0x03, 0, 0x03, 0x0008, 0x0016}, + cmdLutElement{0x01, 0x03, 0, 0x03, 0x0008, 0x001e}, + cmdLutElement{0x01, 0x04, 0, 0x03, 0x0008, 0x0026}, + cmdLutElement{0x01, 0x04, 0, 0x03, 0x0008, 0x0036}, + cmdLutElement{0x00, 0x00, -1, 0x00, 0x0000, 0x0002}, + cmdLutElement{0x00, 0x00, -1, 0x01, 0x0000, 0x0003}, + cmdLutElement{0x00, 0x00, -1, 0x02, 0x0000, 0x0004}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0005}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0006}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0007}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0008}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0009}, + cmdLutElement{0x00, 0x00, -1, 0x00, 0x0001, 0x0002}, + cmdLutElement{0x00, 0x00, -1, 0x01, 0x0001, 0x0003}, + cmdLutElement{0x00, 0x00, -1, 0x02, 0x0001, 0x0004}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0005}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0006}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0007}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0008}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0009}, + cmdLutElement{0x00, 0x00, -1, 0x00, 0x0002, 0x0002}, + cmdLutElement{0x00, 0x00, -1, 0x01, 0x0002, 0x0003}, + cmdLutElement{0x00, 0x00, -1, 0x02, 0x0002, 0x0004}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0005}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0006}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0007}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0008}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0009}, + cmdLutElement{0x00, 0x00, -1, 0x00, 0x0003, 0x0002}, + cmdLutElement{0x00, 0x00, -1, 0x01, 0x0003, 0x0003}, + cmdLutElement{0x00, 0x00, -1, 0x02, 0x0003, 0x0004}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0005}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0006}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0007}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0008}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0009}, + cmdLutElement{0x00, 0x00, -1, 0x00, 0x0004, 0x0002}, + cmdLutElement{0x00, 0x00, -1, 0x01, 0x0004, 0x0003}, + cmdLutElement{0x00, 0x00, -1, 0x02, 0x0004, 0x0004}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0005}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0006}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0007}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0008}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0009}, + cmdLutElement{0x00, 0x00, -1, 0x00, 0x0005, 0x0002}, + cmdLutElement{0x00, 0x00, -1, 0x01, 0x0005, 0x0003}, + cmdLutElement{0x00, 0x00, -1, 0x02, 0x0005, 0x0004}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0005}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0006}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0007}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0008}, + cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0009}, + cmdLutElement{0x01, 0x00, -1, 0x00, 0x0006, 0x0002}, + cmdLutElement{0x01, 0x00, -1, 0x01, 0x0006, 0x0003}, + cmdLutElement{0x01, 0x00, -1, 0x02, 0x0006, 0x0004}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0005}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0006}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0007}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0008}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0009}, + cmdLutElement{0x01, 0x00, -1, 0x00, 0x0008, 0x0002}, + cmdLutElement{0x01, 0x00, -1, 0x01, 0x0008, 0x0003}, + cmdLutElement{0x01, 0x00, -1, 0x02, 0x0008, 0x0004}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0005}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0006}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0007}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0008}, + cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0009}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0000, 0x000a}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0000, 0x000c}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0000, 0x000e}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0000, 0x0012}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0000, 0x0016}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0000, 0x001e}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0000, 0x0026}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0000, 0x0036}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0001, 0x000a}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0001, 0x000c}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0001, 0x000e}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0001, 0x0012}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0001, 0x0016}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0001, 0x001e}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0001, 0x0026}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0001, 0x0036}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0002, 0x000a}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0002, 0x000c}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0002, 0x000e}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0002, 0x0012}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0002, 0x0016}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0002, 0x001e}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0002, 0x0026}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0002, 0x0036}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0003, 0x000a}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0003, 0x000c}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0003, 0x000e}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0003, 0x0012}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0003, 0x0016}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0003, 0x001e}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0003, 0x0026}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0003, 0x0036}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0004, 0x000a}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0004, 0x000c}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0004, 0x000e}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0004, 0x0012}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0004, 0x0016}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0004, 0x001e}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0004, 0x0026}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0004, 0x0036}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0005, 0x000a}, + cmdLutElement{0x00, 0x01, -1, 0x03, 0x0005, 0x000c}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0005, 0x000e}, + cmdLutElement{0x00, 0x02, -1, 0x03, 0x0005, 0x0012}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0005, 0x0016}, + cmdLutElement{0x00, 0x03, -1, 0x03, 0x0005, 0x001e}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0005, 0x0026}, + cmdLutElement{0x00, 0x04, -1, 0x03, 0x0005, 0x0036}, + cmdLutElement{0x01, 0x01, -1, 0x03, 0x0006, 0x000a}, + cmdLutElement{0x01, 0x01, -1, 0x03, 0x0006, 0x000c}, + cmdLutElement{0x01, 0x02, -1, 0x03, 0x0006, 0x000e}, + cmdLutElement{0x01, 0x02, -1, 0x03, 0x0006, 0x0012}, + cmdLutElement{0x01, 0x03, -1, 0x03, 0x0006, 0x0016}, + cmdLutElement{0x01, 0x03, -1, 0x03, 0x0006, 0x001e}, + cmdLutElement{0x01, 0x04, -1, 0x03, 0x0006, 0x0026}, + cmdLutElement{0x01, 0x04, -1, 0x03, 0x0006, 0x0036}, + cmdLutElement{0x01, 0x01, -1, 0x03, 0x0008, 0x000a}, + cmdLutElement{0x01, 0x01, -1, 0x03, 0x0008, 0x000c}, + cmdLutElement{0x01, 0x02, -1, 0x03, 0x0008, 0x000e}, + cmdLutElement{0x01, 0x02, -1, 0x03, 0x0008, 0x0012}, + cmdLutElement{0x01, 0x03, -1, 0x03, 0x0008, 0x0016}, + cmdLutElement{0x01, 0x03, -1, 0x03, 0x0008, 0x001e}, + cmdLutElement{0x01, 0x04, -1, 0x03, 0x0008, 0x0026}, + cmdLutElement{0x01, 0x04, -1, 0x03, 0x0008, 0x0036}, + cmdLutElement{0x02, 0x00, -1, 0x00, 0x000a, 0x0002}, + cmdLutElement{0x02, 0x00, -1, 0x01, 0x000a, 0x0003}, + cmdLutElement{0x02, 0x00, -1, 0x02, 0x000a, 0x0004}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0005}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0006}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0007}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0008}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0009}, + cmdLutElement{0x02, 0x00, -1, 0x00, 0x000e, 0x0002}, + cmdLutElement{0x02, 0x00, -1, 0x01, 0x000e, 0x0003}, + cmdLutElement{0x02, 0x00, -1, 0x02, 0x000e, 0x0004}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0005}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0006}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0007}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0008}, + cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0009}, + cmdLutElement{0x03, 0x00, -1, 0x00, 0x0012, 0x0002}, + cmdLutElement{0x03, 0x00, -1, 0x01, 0x0012, 0x0003}, + cmdLutElement{0x03, 0x00, -1, 0x02, 0x0012, 0x0004}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0005}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0006}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0007}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0008}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0009}, + cmdLutElement{0x03, 0x00, -1, 0x00, 0x001a, 0x0002}, + cmdLutElement{0x03, 0x00, -1, 0x01, 0x001a, 0x0003}, + cmdLutElement{0x03, 0x00, -1, 0x02, 0x001a, 0x0004}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0005}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0006}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0007}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0008}, + cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0009}, + cmdLutElement{0x04, 0x00, -1, 0x00, 0x0022, 0x0002}, + cmdLutElement{0x04, 0x00, -1, 0x01, 0x0022, 0x0003}, + cmdLutElement{0x04, 0x00, -1, 0x02, 0x0022, 0x0004}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0005}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0006}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0007}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0008}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0009}, + cmdLutElement{0x04, 0x00, -1, 0x00, 0x0032, 0x0002}, + cmdLutElement{0x04, 0x00, -1, 0x01, 0x0032, 0x0003}, + cmdLutElement{0x04, 0x00, -1, 0x02, 0x0032, 0x0004}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0005}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0006}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0007}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0008}, + cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0009}, + cmdLutElement{0x05, 0x00, -1, 0x00, 0x0042, 0x0002}, + cmdLutElement{0x05, 0x00, -1, 0x01, 0x0042, 0x0003}, + cmdLutElement{0x05, 0x00, -1, 0x02, 0x0042, 0x0004}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0005}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0006}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0007}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0008}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0009}, + cmdLutElement{0x05, 0x00, -1, 0x00, 0x0062, 0x0002}, + cmdLutElement{0x05, 0x00, -1, 0x01, 0x0062, 0x0003}, + cmdLutElement{0x05, 0x00, -1, 0x02, 0x0062, 0x0004}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0005}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0006}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0007}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0008}, + cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0009}, + cmdLutElement{0x02, 0x01, -1, 0x03, 0x000a, 0x000a}, + cmdLutElement{0x02, 0x01, -1, 0x03, 0x000a, 0x000c}, + cmdLutElement{0x02, 0x02, -1, 0x03, 0x000a, 0x000e}, + cmdLutElement{0x02, 0x02, -1, 0x03, 0x000a, 0x0012}, + cmdLutElement{0x02, 0x03, -1, 0x03, 0x000a, 0x0016}, + cmdLutElement{0x02, 0x03, -1, 0x03, 0x000a, 0x001e}, + cmdLutElement{0x02, 0x04, -1, 0x03, 0x000a, 0x0026}, + cmdLutElement{0x02, 0x04, -1, 0x03, 0x000a, 0x0036}, + cmdLutElement{0x02, 0x01, -1, 0x03, 0x000e, 0x000a}, + cmdLutElement{0x02, 0x01, -1, 0x03, 0x000e, 0x000c}, + cmdLutElement{0x02, 0x02, -1, 0x03, 0x000e, 0x000e}, + cmdLutElement{0x02, 0x02, -1, 0x03, 0x000e, 0x0012}, + cmdLutElement{0x02, 0x03, -1, 0x03, 0x000e, 0x0016}, + cmdLutElement{0x02, 0x03, -1, 0x03, 0x000e, 0x001e}, + cmdLutElement{0x02, 0x04, -1, 0x03, 0x000e, 0x0026}, + cmdLutElement{0x02, 0x04, -1, 0x03, 0x000e, 0x0036}, + cmdLutElement{0x03, 0x01, -1, 0x03, 0x0012, 0x000a}, + cmdLutElement{0x03, 0x01, -1, 0x03, 0x0012, 0x000c}, + cmdLutElement{0x03, 0x02, -1, 0x03, 0x0012, 0x000e}, + cmdLutElement{0x03, 0x02, -1, 0x03, 0x0012, 0x0012}, + cmdLutElement{0x03, 0x03, -1, 0x03, 0x0012, 0x0016}, + cmdLutElement{0x03, 0x03, -1, 0x03, 0x0012, 0x001e}, + cmdLutElement{0x03, 0x04, -1, 0x03, 0x0012, 0x0026}, + cmdLutElement{0x03, 0x04, -1, 0x03, 0x0012, 0x0036}, + cmdLutElement{0x03, 0x01, -1, 0x03, 0x001a, 0x000a}, + cmdLutElement{0x03, 0x01, -1, 0x03, 0x001a, 0x000c}, + cmdLutElement{0x03, 0x02, -1, 0x03, 0x001a, 0x000e}, + cmdLutElement{0x03, 0x02, -1, 0x03, 0x001a, 0x0012}, + cmdLutElement{0x03, 0x03, -1, 0x03, 0x001a, 0x0016}, + cmdLutElement{0x03, 0x03, -1, 0x03, 0x001a, 0x001e}, + cmdLutElement{0x03, 0x04, -1, 0x03, 0x001a, 0x0026}, + cmdLutElement{0x03, 0x04, -1, 0x03, 0x001a, 0x0036}, + cmdLutElement{0x04, 0x01, -1, 0x03, 0x0022, 0x000a}, + cmdLutElement{0x04, 0x01, -1, 0x03, 0x0022, 0x000c}, + cmdLutElement{0x04, 0x02, -1, 0x03, 0x0022, 0x000e}, + cmdLutElement{0x04, 0x02, -1, 0x03, 0x0022, 0x0012}, + cmdLutElement{0x04, 0x03, -1, 0x03, 0x0022, 0x0016}, + cmdLutElement{0x04, 0x03, -1, 0x03, 0x0022, 0x001e}, + cmdLutElement{0x04, 0x04, -1, 0x03, 0x0022, 0x0026}, + cmdLutElement{0x04, 0x04, -1, 0x03, 0x0022, 0x0036}, + cmdLutElement{0x04, 0x01, -1, 0x03, 0x0032, 0x000a}, + cmdLutElement{0x04, 0x01, -1, 0x03, 0x0032, 0x000c}, + cmdLutElement{0x04, 0x02, -1, 0x03, 0x0032, 0x000e}, + cmdLutElement{0x04, 0x02, -1, 0x03, 0x0032, 0x0012}, + cmdLutElement{0x04, 0x03, -1, 0x03, 0x0032, 0x0016}, + cmdLutElement{0x04, 0x03, -1, 0x03, 0x0032, 0x001e}, + cmdLutElement{0x04, 0x04, -1, 0x03, 0x0032, 0x0026}, + cmdLutElement{0x04, 0x04, -1, 0x03, 0x0032, 0x0036}, + cmdLutElement{0x05, 0x01, -1, 0x03, 0x0042, 0x000a}, + cmdLutElement{0x05, 0x01, -1, 0x03, 0x0042, 0x000c}, + cmdLutElement{0x05, 0x02, -1, 0x03, 0x0042, 0x000e}, + cmdLutElement{0x05, 0x02, -1, 0x03, 0x0042, 0x0012}, + cmdLutElement{0x05, 0x03, -1, 0x03, 0x0042, 0x0016}, + cmdLutElement{0x05, 0x03, -1, 0x03, 0x0042, 0x001e}, + cmdLutElement{0x05, 0x04, -1, 0x03, 0x0042, 0x0026}, + cmdLutElement{0x05, 0x04, -1, 0x03, 0x0042, 0x0036}, + cmdLutElement{0x05, 0x01, -1, 0x03, 0x0062, 0x000a}, + cmdLutElement{0x05, 0x01, -1, 0x03, 0x0062, 0x000c}, + cmdLutElement{0x05, 0x02, -1, 0x03, 0x0062, 0x000e}, + cmdLutElement{0x05, 0x02, -1, 0x03, 0x0062, 0x0012}, + cmdLutElement{0x05, 0x03, -1, 0x03, 0x0062, 0x0016}, + cmdLutElement{0x05, 0x03, -1, 0x03, 0x0062, 0x001e}, + cmdLutElement{0x05, 0x04, -1, 0x03, 0x0062, 0x0026}, + cmdLutElement{0x05, 0x04, -1, 0x03, 0x0062, 0x0036}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0000, 0x0046}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0000, 0x0066}, + cmdLutElement{0x00, 0x06, -1, 0x03, 0x0000, 0x0086}, + cmdLutElement{0x00, 0x07, -1, 0x03, 0x0000, 0x00c6}, + cmdLutElement{0x00, 0x08, -1, 0x03, 0x0000, 0x0146}, + cmdLutElement{0x00, 0x09, -1, 0x03, 0x0000, 0x0246}, + cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0000, 0x0446}, + cmdLutElement{0x00, 0x18, -1, 0x03, 0x0000, 0x0846}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0001, 0x0046}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0001, 0x0066}, + cmdLutElement{0x00, 0x06, -1, 0x03, 0x0001, 0x0086}, + cmdLutElement{0x00, 0x07, -1, 0x03, 0x0001, 0x00c6}, + cmdLutElement{0x00, 0x08, -1, 0x03, 0x0001, 0x0146}, + cmdLutElement{0x00, 0x09, -1, 0x03, 0x0001, 0x0246}, + cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0001, 0x0446}, + cmdLutElement{0x00, 0x18, -1, 0x03, 0x0001, 0x0846}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0002, 0x0046}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0002, 0x0066}, + cmdLutElement{0x00, 0x06, -1, 0x03, 0x0002, 0x0086}, + cmdLutElement{0x00, 0x07, -1, 0x03, 0x0002, 0x00c6}, + cmdLutElement{0x00, 0x08, -1, 0x03, 0x0002, 0x0146}, + cmdLutElement{0x00, 0x09, -1, 0x03, 0x0002, 0x0246}, + cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0002, 0x0446}, + cmdLutElement{0x00, 0x18, -1, 0x03, 0x0002, 0x0846}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0003, 0x0046}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0003, 0x0066}, + cmdLutElement{0x00, 0x06, -1, 0x03, 0x0003, 0x0086}, + cmdLutElement{0x00, 0x07, -1, 0x03, 0x0003, 0x00c6}, + cmdLutElement{0x00, 0x08, -1, 0x03, 0x0003, 0x0146}, + cmdLutElement{0x00, 0x09, -1, 0x03, 0x0003, 0x0246}, + cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0003, 0x0446}, + cmdLutElement{0x00, 0x18, -1, 0x03, 0x0003, 0x0846}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0004, 0x0046}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0004, 0x0066}, + cmdLutElement{0x00, 0x06, -1, 0x03, 0x0004, 0x0086}, + cmdLutElement{0x00, 0x07, -1, 0x03, 0x0004, 0x00c6}, + cmdLutElement{0x00, 0x08, -1, 0x03, 0x0004, 0x0146}, + cmdLutElement{0x00, 0x09, -1, 0x03, 0x0004, 0x0246}, + cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0004, 0x0446}, + cmdLutElement{0x00, 0x18, -1, 0x03, 0x0004, 0x0846}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0005, 0x0046}, + cmdLutElement{0x00, 0x05, -1, 0x03, 0x0005, 0x0066}, + cmdLutElement{0x00, 0x06, -1, 0x03, 0x0005, 0x0086}, + cmdLutElement{0x00, 0x07, -1, 0x03, 0x0005, 0x00c6}, + cmdLutElement{0x00, 0x08, -1, 0x03, 0x0005, 0x0146}, + cmdLutElement{0x00, 0x09, -1, 0x03, 0x0005, 0x0246}, + cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0005, 0x0446}, + cmdLutElement{0x00, 0x18, -1, 0x03, 0x0005, 0x0846}, + cmdLutElement{0x01, 0x05, -1, 0x03, 0x0006, 0x0046}, + cmdLutElement{0x01, 0x05, -1, 0x03, 0x0006, 0x0066}, + cmdLutElement{0x01, 0x06, -1, 0x03, 0x0006, 0x0086}, + cmdLutElement{0x01, 0x07, -1, 0x03, 0x0006, 0x00c6}, + cmdLutElement{0x01, 0x08, -1, 0x03, 0x0006, 0x0146}, + cmdLutElement{0x01, 0x09, -1, 0x03, 0x0006, 0x0246}, + cmdLutElement{0x01, 0x0a, -1, 0x03, 0x0006, 0x0446}, + cmdLutElement{0x01, 0x18, -1, 0x03, 0x0006, 0x0846}, + cmdLutElement{0x01, 0x05, -1, 0x03, 0x0008, 0x0046}, + cmdLutElement{0x01, 0x05, -1, 0x03, 0x0008, 0x0066}, + cmdLutElement{0x01, 0x06, -1, 0x03, 0x0008, 0x0086}, + cmdLutElement{0x01, 0x07, -1, 0x03, 0x0008, 0x00c6}, + cmdLutElement{0x01, 0x08, -1, 0x03, 0x0008, 0x0146}, + cmdLutElement{0x01, 0x09, -1, 0x03, 0x0008, 0x0246}, + cmdLutElement{0x01, 0x0a, -1, 0x03, 0x0008, 0x0446}, + cmdLutElement{0x01, 0x18, -1, 0x03, 0x0008, 0x0846}, + cmdLutElement{0x06, 0x00, -1, 0x00, 0x0082, 0x0002}, + cmdLutElement{0x06, 0x00, -1, 0x01, 0x0082, 0x0003}, + cmdLutElement{0x06, 0x00, -1, 0x02, 0x0082, 0x0004}, + cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0005}, + cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0006}, + cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0007}, + cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0008}, + cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0009}, + cmdLutElement{0x07, 0x00, -1, 0x00, 0x00c2, 0x0002}, + cmdLutElement{0x07, 0x00, -1, 0x01, 0x00c2, 0x0003}, + cmdLutElement{0x07, 0x00, -1, 0x02, 0x00c2, 0x0004}, + cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0005}, + cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0006}, + cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0007}, + cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0008}, + cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0009}, + cmdLutElement{0x08, 0x00, -1, 0x00, 0x0142, 0x0002}, + cmdLutElement{0x08, 0x00, -1, 0x01, 0x0142, 0x0003}, + cmdLutElement{0x08, 0x00, -1, 0x02, 0x0142, 0x0004}, + cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0005}, + cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0006}, + cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0007}, + cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0008}, + cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0009}, + cmdLutElement{0x09, 0x00, -1, 0x00, 0x0242, 0x0002}, + cmdLutElement{0x09, 0x00, -1, 0x01, 0x0242, 0x0003}, + cmdLutElement{0x09, 0x00, -1, 0x02, 0x0242, 0x0004}, + cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0005}, + cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0006}, + cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0007}, + cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0008}, + cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0009}, + cmdLutElement{0x0a, 0x00, -1, 0x00, 0x0442, 0x0002}, + cmdLutElement{0x0a, 0x00, -1, 0x01, 0x0442, 0x0003}, + cmdLutElement{0x0a, 0x00, -1, 0x02, 0x0442, 0x0004}, + cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0005}, + cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0006}, + cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0007}, + cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0008}, + cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0009}, + cmdLutElement{0x0c, 0x00, -1, 0x00, 0x0842, 0x0002}, + cmdLutElement{0x0c, 0x00, -1, 0x01, 0x0842, 0x0003}, + cmdLutElement{0x0c, 0x00, -1, 0x02, 0x0842, 0x0004}, + cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0005}, + cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0006}, + cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0007}, + cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0008}, + cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0009}, + cmdLutElement{0x0e, 0x00, -1, 0x00, 0x1842, 0x0002}, + cmdLutElement{0x0e, 0x00, -1, 0x01, 0x1842, 0x0003}, + cmdLutElement{0x0e, 0x00, -1, 0x02, 0x1842, 0x0004}, + cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0005}, + cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0006}, + cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0007}, + cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0008}, + cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0009}, + cmdLutElement{0x18, 0x00, -1, 0x00, 0x5842, 0x0002}, + cmdLutElement{0x18, 0x00, -1, 0x01, 0x5842, 0x0003}, + cmdLutElement{0x18, 0x00, -1, 0x02, 0x5842, 0x0004}, + cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0005}, + cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0006}, + cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0007}, + cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0008}, + cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0009}, + cmdLutElement{0x02, 0x05, -1, 0x03, 0x000a, 0x0046}, + cmdLutElement{0x02, 0x05, -1, 0x03, 0x000a, 0x0066}, + cmdLutElement{0x02, 0x06, -1, 0x03, 0x000a, 0x0086}, + cmdLutElement{0x02, 0x07, -1, 0x03, 0x000a, 0x00c6}, + cmdLutElement{0x02, 0x08, -1, 0x03, 0x000a, 0x0146}, + cmdLutElement{0x02, 0x09, -1, 0x03, 0x000a, 0x0246}, + cmdLutElement{0x02, 0x0a, -1, 0x03, 0x000a, 0x0446}, + cmdLutElement{0x02, 0x18, -1, 0x03, 0x000a, 0x0846}, + cmdLutElement{0x02, 0x05, -1, 0x03, 0x000e, 0x0046}, + cmdLutElement{0x02, 0x05, -1, 0x03, 0x000e, 0x0066}, + cmdLutElement{0x02, 0x06, -1, 0x03, 0x000e, 0x0086}, + cmdLutElement{0x02, 0x07, -1, 0x03, 0x000e, 0x00c6}, + cmdLutElement{0x02, 0x08, -1, 0x03, 0x000e, 0x0146}, + cmdLutElement{0x02, 0x09, -1, 0x03, 0x000e, 0x0246}, + cmdLutElement{0x02, 0x0a, -1, 0x03, 0x000e, 0x0446}, + cmdLutElement{0x02, 0x18, -1, 0x03, 0x000e, 0x0846}, + cmdLutElement{0x03, 0x05, -1, 0x03, 0x0012, 0x0046}, + cmdLutElement{0x03, 0x05, -1, 0x03, 0x0012, 0x0066}, + cmdLutElement{0x03, 0x06, -1, 0x03, 0x0012, 0x0086}, + cmdLutElement{0x03, 0x07, -1, 0x03, 0x0012, 0x00c6}, + cmdLutElement{0x03, 0x08, -1, 0x03, 0x0012, 0x0146}, + cmdLutElement{0x03, 0x09, -1, 0x03, 0x0012, 0x0246}, + cmdLutElement{0x03, 0x0a, -1, 0x03, 0x0012, 0x0446}, + cmdLutElement{0x03, 0x18, -1, 0x03, 0x0012, 0x0846}, + cmdLutElement{0x03, 0x05, -1, 0x03, 0x001a, 0x0046}, + cmdLutElement{0x03, 0x05, -1, 0x03, 0x001a, 0x0066}, + cmdLutElement{0x03, 0x06, -1, 0x03, 0x001a, 0x0086}, + cmdLutElement{0x03, 0x07, -1, 0x03, 0x001a, 0x00c6}, + cmdLutElement{0x03, 0x08, -1, 0x03, 0x001a, 0x0146}, + cmdLutElement{0x03, 0x09, -1, 0x03, 0x001a, 0x0246}, + cmdLutElement{0x03, 0x0a, -1, 0x03, 0x001a, 0x0446}, + cmdLutElement{0x03, 0x18, -1, 0x03, 0x001a, 0x0846}, + cmdLutElement{0x04, 0x05, -1, 0x03, 0x0022, 0x0046}, + cmdLutElement{0x04, 0x05, -1, 0x03, 0x0022, 0x0066}, + cmdLutElement{0x04, 0x06, -1, 0x03, 0x0022, 0x0086}, + cmdLutElement{0x04, 0x07, -1, 0x03, 0x0022, 0x00c6}, + cmdLutElement{0x04, 0x08, -1, 0x03, 0x0022, 0x0146}, + cmdLutElement{0x04, 0x09, -1, 0x03, 0x0022, 0x0246}, + cmdLutElement{0x04, 0x0a, -1, 0x03, 0x0022, 0x0446}, + cmdLutElement{0x04, 0x18, -1, 0x03, 0x0022, 0x0846}, + cmdLutElement{0x04, 0x05, -1, 0x03, 0x0032, 0x0046}, + cmdLutElement{0x04, 0x05, -1, 0x03, 0x0032, 0x0066}, + cmdLutElement{0x04, 0x06, -1, 0x03, 0x0032, 0x0086}, + cmdLutElement{0x04, 0x07, -1, 0x03, 0x0032, 0x00c6}, + cmdLutElement{0x04, 0x08, -1, 0x03, 0x0032, 0x0146}, + cmdLutElement{0x04, 0x09, -1, 0x03, 0x0032, 0x0246}, + cmdLutElement{0x04, 0x0a, -1, 0x03, 0x0032, 0x0446}, + cmdLutElement{0x04, 0x18, -1, 0x03, 0x0032, 0x0846}, + cmdLutElement{0x05, 0x05, -1, 0x03, 0x0042, 0x0046}, + cmdLutElement{0x05, 0x05, -1, 0x03, 0x0042, 0x0066}, + cmdLutElement{0x05, 0x06, -1, 0x03, 0x0042, 0x0086}, + cmdLutElement{0x05, 0x07, -1, 0x03, 0x0042, 0x00c6}, + cmdLutElement{0x05, 0x08, -1, 0x03, 0x0042, 0x0146}, + cmdLutElement{0x05, 0x09, -1, 0x03, 0x0042, 0x0246}, + cmdLutElement{0x05, 0x0a, -1, 0x03, 0x0042, 0x0446}, + cmdLutElement{0x05, 0x18, -1, 0x03, 0x0042, 0x0846}, + cmdLutElement{0x05, 0x05, -1, 0x03, 0x0062, 0x0046}, + cmdLutElement{0x05, 0x05, -1, 0x03, 0x0062, 0x0066}, + cmdLutElement{0x05, 0x06, -1, 0x03, 0x0062, 0x0086}, + cmdLutElement{0x05, 0x07, -1, 0x03, 0x0062, 0x00c6}, + cmdLutElement{0x05, 0x08, -1, 0x03, 0x0062, 0x0146}, + cmdLutElement{0x05, 0x09, -1, 0x03, 0x0062, 0x0246}, + cmdLutElement{0x05, 0x0a, -1, 0x03, 0x0062, 0x0446}, + cmdLutElement{0x05, 0x18, -1, 0x03, 0x0062, 0x0846}, + cmdLutElement{0x06, 0x01, -1, 0x03, 0x0082, 0x000a}, + cmdLutElement{0x06, 0x01, -1, 0x03, 0x0082, 0x000c}, + cmdLutElement{0x06, 0x02, -1, 0x03, 0x0082, 0x000e}, + cmdLutElement{0x06, 0x02, -1, 0x03, 0x0082, 0x0012}, + cmdLutElement{0x06, 0x03, -1, 0x03, 0x0082, 0x0016}, + cmdLutElement{0x06, 0x03, -1, 0x03, 0x0082, 0x001e}, + cmdLutElement{0x06, 0x04, -1, 0x03, 0x0082, 0x0026}, + cmdLutElement{0x06, 0x04, -1, 0x03, 0x0082, 0x0036}, + cmdLutElement{0x07, 0x01, -1, 0x03, 0x00c2, 0x000a}, + cmdLutElement{0x07, 0x01, -1, 0x03, 0x00c2, 0x000c}, + cmdLutElement{0x07, 0x02, -1, 0x03, 0x00c2, 0x000e}, + cmdLutElement{0x07, 0x02, -1, 0x03, 0x00c2, 0x0012}, + cmdLutElement{0x07, 0x03, -1, 0x03, 0x00c2, 0x0016}, + cmdLutElement{0x07, 0x03, -1, 0x03, 0x00c2, 0x001e}, + cmdLutElement{0x07, 0x04, -1, 0x03, 0x00c2, 0x0026}, + cmdLutElement{0x07, 0x04, -1, 0x03, 0x00c2, 0x0036}, + cmdLutElement{0x08, 0x01, -1, 0x03, 0x0142, 0x000a}, + cmdLutElement{0x08, 0x01, -1, 0x03, 0x0142, 0x000c}, + cmdLutElement{0x08, 0x02, -1, 0x03, 0x0142, 0x000e}, + cmdLutElement{0x08, 0x02, -1, 0x03, 0x0142, 0x0012}, + cmdLutElement{0x08, 0x03, -1, 0x03, 0x0142, 0x0016}, + cmdLutElement{0x08, 0x03, -1, 0x03, 0x0142, 0x001e}, + cmdLutElement{0x08, 0x04, -1, 0x03, 0x0142, 0x0026}, + cmdLutElement{0x08, 0x04, -1, 0x03, 0x0142, 0x0036}, + cmdLutElement{0x09, 0x01, -1, 0x03, 0x0242, 0x000a}, + cmdLutElement{0x09, 0x01, -1, 0x03, 0x0242, 0x000c}, + cmdLutElement{0x09, 0x02, -1, 0x03, 0x0242, 0x000e}, + cmdLutElement{0x09, 0x02, -1, 0x03, 0x0242, 0x0012}, + cmdLutElement{0x09, 0x03, -1, 0x03, 0x0242, 0x0016}, + cmdLutElement{0x09, 0x03, -1, 0x03, 0x0242, 0x001e}, + cmdLutElement{0x09, 0x04, -1, 0x03, 0x0242, 0x0026}, + cmdLutElement{0x09, 0x04, -1, 0x03, 0x0242, 0x0036}, + cmdLutElement{0x0a, 0x01, -1, 0x03, 0x0442, 0x000a}, + cmdLutElement{0x0a, 0x01, -1, 0x03, 0x0442, 0x000c}, + cmdLutElement{0x0a, 0x02, -1, 0x03, 0x0442, 0x000e}, + cmdLutElement{0x0a, 0x02, -1, 0x03, 0x0442, 0x0012}, + cmdLutElement{0x0a, 0x03, -1, 0x03, 0x0442, 0x0016}, + cmdLutElement{0x0a, 0x03, -1, 0x03, 0x0442, 0x001e}, + cmdLutElement{0x0a, 0x04, -1, 0x03, 0x0442, 0x0026}, + cmdLutElement{0x0a, 0x04, -1, 0x03, 0x0442, 0x0036}, + cmdLutElement{0x0c, 0x01, -1, 0x03, 0x0842, 0x000a}, + cmdLutElement{0x0c, 0x01, -1, 0x03, 0x0842, 0x000c}, + cmdLutElement{0x0c, 0x02, -1, 0x03, 0x0842, 0x000e}, + cmdLutElement{0x0c, 0x02, -1, 0x03, 0x0842, 0x0012}, + cmdLutElement{0x0c, 0x03, -1, 0x03, 0x0842, 0x0016}, + cmdLutElement{0x0c, 0x03, -1, 0x03, 0x0842, 0x001e}, + cmdLutElement{0x0c, 0x04, -1, 0x03, 0x0842, 0x0026}, + cmdLutElement{0x0c, 0x04, -1, 0x03, 0x0842, 0x0036}, + cmdLutElement{0x0e, 0x01, -1, 0x03, 0x1842, 0x000a}, + cmdLutElement{0x0e, 0x01, -1, 0x03, 0x1842, 0x000c}, + cmdLutElement{0x0e, 0x02, -1, 0x03, 0x1842, 0x000e}, + cmdLutElement{0x0e, 0x02, -1, 0x03, 0x1842, 0x0012}, + cmdLutElement{0x0e, 0x03, -1, 0x03, 0x1842, 0x0016}, + cmdLutElement{0x0e, 0x03, -1, 0x03, 0x1842, 0x001e}, + cmdLutElement{0x0e, 0x04, -1, 0x03, 0x1842, 0x0026}, + cmdLutElement{0x0e, 0x04, -1, 0x03, 0x1842, 0x0036}, + cmdLutElement{0x18, 0x01, -1, 0x03, 0x5842, 0x000a}, + cmdLutElement{0x18, 0x01, -1, 0x03, 0x5842, 0x000c}, + cmdLutElement{0x18, 0x02, -1, 0x03, 0x5842, 0x000e}, + cmdLutElement{0x18, 0x02, -1, 0x03, 0x5842, 0x0012}, + cmdLutElement{0x18, 0x03, -1, 0x03, 0x5842, 0x0016}, + cmdLutElement{0x18, 0x03, -1, 0x03, 0x5842, 0x001e}, + cmdLutElement{0x18, 0x04, -1, 0x03, 0x5842, 0x0026}, + cmdLutElement{0x18, 0x04, -1, 0x03, 0x5842, 0x0036}, + cmdLutElement{0x06, 0x05, -1, 0x03, 0x0082, 0x0046}, + cmdLutElement{0x06, 0x05, -1, 0x03, 0x0082, 0x0066}, + cmdLutElement{0x06, 0x06, -1, 0x03, 0x0082, 0x0086}, + cmdLutElement{0x06, 0x07, -1, 0x03, 0x0082, 0x00c6}, + cmdLutElement{0x06, 0x08, -1, 0x03, 0x0082, 0x0146}, + cmdLutElement{0x06, 0x09, -1, 0x03, 0x0082, 0x0246}, + cmdLutElement{0x06, 0x0a, -1, 0x03, 0x0082, 0x0446}, + cmdLutElement{0x06, 0x18, -1, 0x03, 0x0082, 0x0846}, + cmdLutElement{0x07, 0x05, -1, 0x03, 0x00c2, 0x0046}, + cmdLutElement{0x07, 0x05, -1, 0x03, 0x00c2, 0x0066}, + cmdLutElement{0x07, 0x06, -1, 0x03, 0x00c2, 0x0086}, + cmdLutElement{0x07, 0x07, -1, 0x03, 0x00c2, 0x00c6}, + cmdLutElement{0x07, 0x08, -1, 0x03, 0x00c2, 0x0146}, + cmdLutElement{0x07, 0x09, -1, 0x03, 0x00c2, 0x0246}, + cmdLutElement{0x07, 0x0a, -1, 0x03, 0x00c2, 0x0446}, + cmdLutElement{0x07, 0x18, -1, 0x03, 0x00c2, 0x0846}, + cmdLutElement{0x08, 0x05, -1, 0x03, 0x0142, 0x0046}, + cmdLutElement{0x08, 0x05, -1, 0x03, 0x0142, 0x0066}, + cmdLutElement{0x08, 0x06, -1, 0x03, 0x0142, 0x0086}, + cmdLutElement{0x08, 0x07, -1, 0x03, 0x0142, 0x00c6}, + cmdLutElement{0x08, 0x08, -1, 0x03, 0x0142, 0x0146}, + cmdLutElement{0x08, 0x09, -1, 0x03, 0x0142, 0x0246}, + cmdLutElement{0x08, 0x0a, -1, 0x03, 0x0142, 0x0446}, + cmdLutElement{0x08, 0x18, -1, 0x03, 0x0142, 0x0846}, + cmdLutElement{0x09, 0x05, -1, 0x03, 0x0242, 0x0046}, + cmdLutElement{0x09, 0x05, -1, 0x03, 0x0242, 0x0066}, + cmdLutElement{0x09, 0x06, -1, 0x03, 0x0242, 0x0086}, + cmdLutElement{0x09, 0x07, -1, 0x03, 0x0242, 0x00c6}, + cmdLutElement{0x09, 0x08, -1, 0x03, 0x0242, 0x0146}, + cmdLutElement{0x09, 0x09, -1, 0x03, 0x0242, 0x0246}, + cmdLutElement{0x09, 0x0a, -1, 0x03, 0x0242, 0x0446}, + cmdLutElement{0x09, 0x18, -1, 0x03, 0x0242, 0x0846}, + cmdLutElement{0x0a, 0x05, -1, 0x03, 0x0442, 0x0046}, + cmdLutElement{0x0a, 0x05, -1, 0x03, 0x0442, 0x0066}, + cmdLutElement{0x0a, 0x06, -1, 0x03, 0x0442, 0x0086}, + cmdLutElement{0x0a, 0x07, -1, 0x03, 0x0442, 0x00c6}, + cmdLutElement{0x0a, 0x08, -1, 0x03, 0x0442, 0x0146}, + cmdLutElement{0x0a, 0x09, -1, 0x03, 0x0442, 0x0246}, + cmdLutElement{0x0a, 0x0a, -1, 0x03, 0x0442, 0x0446}, + cmdLutElement{0x0a, 0x18, -1, 0x03, 0x0442, 0x0846}, + cmdLutElement{0x0c, 0x05, -1, 0x03, 0x0842, 0x0046}, + cmdLutElement{0x0c, 0x05, -1, 0x03, 0x0842, 0x0066}, + cmdLutElement{0x0c, 0x06, -1, 0x03, 0x0842, 0x0086}, + cmdLutElement{0x0c, 0x07, -1, 0x03, 0x0842, 0x00c6}, + cmdLutElement{0x0c, 0x08, -1, 0x03, 0x0842, 0x0146}, + cmdLutElement{0x0c, 0x09, -1, 0x03, 0x0842, 0x0246}, + cmdLutElement{0x0c, 0x0a, -1, 0x03, 0x0842, 0x0446}, + cmdLutElement{0x0c, 0x18, -1, 0x03, 0x0842, 0x0846}, + cmdLutElement{0x0e, 0x05, -1, 0x03, 0x1842, 0x0046}, + cmdLutElement{0x0e, 0x05, -1, 0x03, 0x1842, 0x0066}, + cmdLutElement{0x0e, 0x06, -1, 0x03, 0x1842, 0x0086}, + cmdLutElement{0x0e, 0x07, -1, 0x03, 0x1842, 0x00c6}, + cmdLutElement{0x0e, 0x08, -1, 0x03, 0x1842, 0x0146}, + cmdLutElement{0x0e, 0x09, -1, 0x03, 0x1842, 0x0246}, + cmdLutElement{0x0e, 0x0a, -1, 0x03, 0x1842, 0x0446}, + cmdLutElement{0x0e, 0x18, -1, 0x03, 0x1842, 0x0846}, + cmdLutElement{0x18, 0x05, -1, 0x03, 0x5842, 0x0046}, + cmdLutElement{0x18, 0x05, -1, 0x03, 0x5842, 0x0066}, + cmdLutElement{0x18, 0x06, -1, 0x03, 0x5842, 0x0086}, + cmdLutElement{0x18, 0x07, -1, 0x03, 0x5842, 0x00c6}, + cmdLutElement{0x18, 0x08, -1, 0x03, 0x5842, 0x0146}, + cmdLutElement{0x18, 0x09, -1, 0x03, 0x5842, 0x0246}, + cmdLutElement{0x18, 0x0a, -1, 0x03, 0x5842, 0x0446}, + cmdLutElement{0x18, 0x18, -1, 0x03, 0x5842, 0x0846}, +} diff --git a/vendor/github.com/andybalholm/brotli/quality.go b/vendor/github.com/andybalholm/brotli/quality.go new file mode 100644 index 0000000000..49709a3823 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/quality.go @@ -0,0 +1,196 @@ +package brotli + +const fastOnePassCompressionQuality = 0 + +const fastTwoPassCompressionQuality = 1 + +const zopflificationQuality = 10 + +const hqZopflificationQuality = 11 + +const maxQualityForStaticEntropyCodes = 2 + +const minQualityForBlockSplit = 4 + +const minQualityForNonzeroDistanceParams = 4 + +const minQualityForOptimizeHistograms = 4 + +const minQualityForExtensiveReferenceSearch = 5 + +const minQualityForContextModeling = 5 + +const minQualityForHqContextModeling = 7 + +const minQualityForHqBlockSplitting = 10 + +/* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting, + so we buffer at most this much literals and commands. */ +const maxNumDelayedSymbols = 0x2FFF + +/* Returns hash-table size for quality levels 0 and 1. */ +func maxHashTableSize(quality int) uint { + if quality == fastOnePassCompressionQuality { + return 1 << 15 + } else { + return 1 << 17 + } +} + +/* The maximum length for which the zopflification uses distinct distances. */ +const maxZopfliLenQuality10 = 150 + +const maxZopfliLenQuality11 = 325 + +/* Do not thoroughly search when a long copy is found. */ +const longCopyQuickStep = 16384 + +func maxZopfliLen(params *encoderParams) uint { + if params.quality <= 10 { + return maxZopfliLenQuality10 + } else { + return maxZopfliLenQuality11 + } +} + +/* Number of best candidates to evaluate to expand Zopfli chain. */ +func maxZopfliCandidates(params *encoderParams) uint { + if params.quality <= 10 { + return 1 + } else { + return 5 + } +} + +func sanitizeParams(params *encoderParams) { + params.quality = brotli_min_int(maxQuality, brotli_max_int(minQuality, params.quality)) + if params.quality <= maxQualityForStaticEntropyCodes { + params.large_window = false + } + + if params.lgwin < minWindowBits { + params.lgwin = minWindowBits + } else { + var max_lgwin int + if params.large_window { + max_lgwin = largeMaxWindowBits + } else { + max_lgwin = maxWindowBits + } + if params.lgwin > uint(max_lgwin) { + params.lgwin = uint(max_lgwin) + } + } +} + +/* Returns optimized lg_block value. */ +func computeLgBlock(params *encoderParams) int { + var lgblock int = params.lgblock + if params.quality == fastOnePassCompressionQuality || params.quality == fastTwoPassCompressionQuality { + lgblock = int(params.lgwin) + } else if params.quality < minQualityForBlockSplit { + lgblock = 14 + } else if lgblock == 0 { + lgblock = 16 + if params.quality >= 9 && params.lgwin > uint(lgblock) { + lgblock = brotli_min_int(18, int(params.lgwin)) + } + } else { + lgblock = brotli_min_int(maxInputBlockBits, brotli_max_int(minInputBlockBits, lgblock)) + } + + return lgblock +} + +/* Returns log2 of the size of main ring buffer area. + Allocate at least lgwin + 1 bits for the ring buffer so that the newly + added block fits there completely and we still get lgwin bits and at least + read_block_size_bits + 1 bits because the copy tail length needs to be + smaller than ring-buffer size. */ +func computeRbBits(params *encoderParams) int { + return 1 + brotli_max_int(int(params.lgwin), params.lgblock) +} + +func maxMetablockSize(params *encoderParams) uint { + var bits int = brotli_min_int(computeRbBits(params), maxInputBlockBits) + return uint(1) << uint(bits) +} + +/* When searching for backward references and have not seen matches for a long + time, we can skip some match lookups. Unsuccessful match lookups are very + expensive and this kind of a heuristic speeds up compression quite a lot. + At first 8 byte strides are taken and every second byte is put to hasher. + After 4x more literals stride by 16 bytes, every put 4-th byte to hasher. + Applied only to qualities 2 to 9. */ +func literalSpreeLengthForSparseSearch(params *encoderParams) uint { + if params.quality < 9 { + return 64 + } else { + return 512 + } +} + +func chooseHasher(params *encoderParams, hparams *hasherParams) { + if params.quality > 9 { + hparams.type_ = 10 + } else if params.quality == 4 && params.size_hint >= 1<<20 { + hparams.type_ = 54 + } else if params.quality < 5 { + hparams.type_ = params.quality + } else if params.lgwin <= 16 { + if params.quality < 7 { + hparams.type_ = 40 + } else if params.quality < 9 { + hparams.type_ = 41 + } else { + hparams.type_ = 42 + } + } else if params.size_hint >= 1<<20 && params.lgwin >= 19 { + hparams.type_ = 6 + hparams.block_bits = params.quality - 1 + hparams.bucket_bits = 15 + hparams.hash_len = 5 + if params.quality < 7 { + hparams.num_last_distances_to_check = 4 + } else if params.quality < 9 { + hparams.num_last_distances_to_check = 10 + } else { + hparams.num_last_distances_to_check = 16 + } + } else { + hparams.type_ = 5 + hparams.block_bits = params.quality - 1 + if params.quality < 7 { + hparams.bucket_bits = 14 + } else { + hparams.bucket_bits = 15 + } + if params.quality < 7 { + hparams.num_last_distances_to_check = 4 + } else if params.quality < 9 { + hparams.num_last_distances_to_check = 10 + } else { + hparams.num_last_distances_to_check = 16 + } + } + + if params.lgwin > 24 { + /* Different hashers for large window brotli: not for qualities <= 2, + these are too fast for large window. Not for qualities >= 10: their + hasher already works well with large window. So the changes are: + H3 --> H35: for quality 3. + H54 --> H55: for quality 4 with size hint > 1MB + H6 --> H65: for qualities 5, 6, 7, 8, 9. */ + if hparams.type_ == 3 { + hparams.type_ = 35 + } + + if hparams.type_ == 54 { + hparams.type_ = 55 + } + + if hparams.type_ == 6 { + hparams.type_ = 65 + } + } +} diff --git a/vendor/github.com/andybalholm/brotli/reader.go b/vendor/github.com/andybalholm/brotli/reader.go new file mode 100644 index 0000000000..9419c79c17 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/reader.go @@ -0,0 +1,108 @@ +package brotli + +import ( + "errors" + "io" +) + +type decodeError int + +func (err decodeError) Error() string { + return "brotli: " + string(decoderErrorString(int(err))) +} + +var errExcessiveInput = errors.New("brotli: excessive input") +var errInvalidState = errors.New("brotli: invalid state") + +// readBufSize is a "good" buffer size that avoids excessive round-trips +// between C and Go but doesn't waste too much memory on buffering. +// It is arbitrarily chosen to be equal to the constant used in io.Copy. +const readBufSize = 32 * 1024 + +// NewReader creates a new Reader reading the given reader. +func NewReader(src io.Reader) *Reader { + r := new(Reader) + r.Reset(src) + return r +} + +// Reset discards the Reader's state and makes it equivalent to the result of +// its original state from NewReader, but reading from src instead. +// This permits reusing a Reader rather than allocating a new one. +// Error is always nil +func (r *Reader) Reset(src io.Reader) error { + if r.error_code < 0 { + // There was an unrecoverable error, leaving the Reader's state + // undefined. Clear out everything but the buffer. + *r = Reader{buf: r.buf} + } + + decoderStateInit(r) + r.src = src + if r.buf == nil { + r.buf = make([]byte, readBufSize) + } + return nil +} + +func (r *Reader) Read(p []byte) (n int, err error) { + if !decoderHasMoreOutput(r) && len(r.in) == 0 { + m, readErr := r.src.Read(r.buf) + if m == 0 { + // If readErr is `nil`, we just proxy underlying stream behavior. + return 0, readErr + } + r.in = r.buf[:m] + } + + if len(p) == 0 { + return 0, nil + } + + for { + var written uint + in_len := uint(len(r.in)) + out_len := uint(len(p)) + in_remaining := in_len + out_remaining := out_len + result := decoderDecompressStream(r, &in_remaining, &r.in, &out_remaining, &p) + written = out_len - out_remaining + n = int(written) + + switch result { + case decoderResultSuccess: + if len(r.in) > 0 { + return n, errExcessiveInput + } + return n, nil + case decoderResultError: + return n, decodeError(decoderGetErrorCode(r)) + case decoderResultNeedsMoreOutput: + if n == 0 { + return 0, io.ErrShortBuffer + } + return n, nil + case decoderNeedsMoreInput: + } + + if len(r.in) != 0 { + return 0, errInvalidState + } + + // Calling r.src.Read may block. Don't block if we have data to return. + if n > 0 { + return n, nil + } + + // Top off the buffer. + encN, err := r.src.Read(r.buf) + if encN == 0 { + // Not enough data to complete decoding. + if err == io.EOF { + return 0, io.ErrUnexpectedEOF + } + return 0, err + } + r.in = r.buf[:encN] + } +} diff --git a/vendor/github.com/andybalholm/brotli/ringbuffer.go b/vendor/github.com/andybalholm/brotli/ringbuffer.go new file mode 100644 index 0000000000..1c8f86feec --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/ringbuffer.go @@ -0,0 +1,134 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of + data in a circular manner: writing a byte writes it to: + `position() % (1 << window_bits)'. + For convenience, the ringBuffer array contains another copy of the + first `1 << tail_bits' bytes: + buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits), + and another copy of the last two bytes: + buffer_[-1] == buffer_[(1 << window_bits) - 1] and + buffer_[-2] == buffer_[(1 << window_bits) - 2]. */ +type ringBuffer struct { + size_ uint32 + mask_ uint32 + tail_size_ uint32 + total_size_ uint32 + cur_size_ uint32 + pos_ uint32 + data_ []byte + buffer_ []byte +} + +func ringBufferInit(rb *ringBuffer) { + rb.pos_ = 0 +} + +func ringBufferSetup(params *encoderParams, rb *ringBuffer) { + var window_bits int = computeRbBits(params) + var tail_bits int = params.lgblock + *(*uint32)(&rb.size_) = 1 << uint(window_bits) + *(*uint32)(&rb.mask_) = (1 << uint(window_bits)) - 1 + *(*uint32)(&rb.tail_size_) = 1 << uint(tail_bits) + *(*uint32)(&rb.total_size_) = rb.size_ + rb.tail_size_ +} + +const kSlackForEightByteHashingEverywhere uint = 7 + +/* Allocates or re-allocates data_ to the given length + plus some slack + region before and after. Fills the slack regions with zeros. */ +func ringBufferInitBuffer(buflen uint32, rb *ringBuffer) { + var new_data []byte + var i uint + size := 2 + int(buflen) + int(kSlackForEightByteHashingEverywhere) + if cap(rb.data_) < size { + new_data = make([]byte, size) + } else { + new_data = rb.data_[:size] + } + if rb.data_ != nil { + copy(new_data, rb.data_[:2+rb.cur_size_+uint32(kSlackForEightByteHashingEverywhere)]) + } + + rb.data_ = new_data + rb.cur_size_ = buflen + rb.buffer_ = rb.data_[2:] + rb.data_[1] = 0 + rb.data_[0] = rb.data_[1] + for i = 0; i < kSlackForEightByteHashingEverywhere; i++ { + rb.buffer_[rb.cur_size_+uint32(i)] = 0 + } +} + +func ringBufferWriteTail(bytes []byte, n uint, rb *ringBuffer) { + var masked_pos uint = uint(rb.pos_ & rb.mask_) + if uint32(masked_pos) < rb.tail_size_ { + /* Just fill the tail buffer with the beginning data. */ + var p uint = uint(rb.size_ + uint32(masked_pos)) + copy(rb.buffer_[p:], bytes[:brotli_min_size_t(n, uint(rb.tail_size_-uint32(masked_pos)))]) + } +} + +/* Push bytes into the ring buffer. */ +func ringBufferWrite(bytes []byte, n uint, rb *ringBuffer) { + if rb.pos_ == 0 && uint32(n) < rb.tail_size_ { + /* Special case for the first write: to process the first block, we don't + need to allocate the whole ring-buffer and we don't need the tail + either. However, we do this memory usage optimization only if the + first write is less than the tail size, which is also the input block + size, otherwise it is likely that other blocks will follow and we + will need to reallocate to the full size anyway. */ + rb.pos_ = uint32(n) + + ringBufferInitBuffer(rb.pos_, rb) + copy(rb.buffer_, bytes[:n]) + return + } + + if rb.cur_size_ < rb.total_size_ { + /* Lazily allocate the full buffer. */ + ringBufferInitBuffer(rb.total_size_, rb) + + /* Initialize the last two bytes to zero, so that we don't have to worry + later when we copy the last two bytes to the first two positions. */ + rb.buffer_[rb.size_-2] = 0 + + rb.buffer_[rb.size_-1] = 0 + } + { + var masked_pos uint = uint(rb.pos_ & rb.mask_) + + /* The length of the writes is limited so that we do not need to worry + about a write */ + ringBufferWriteTail(bytes, n, rb) + + if uint32(masked_pos+n) <= rb.size_ { + /* A single write fits. */ + copy(rb.buffer_[masked_pos:], bytes[:n]) + } else { + /* Split into two writes. + Copy into the end of the buffer, including the tail buffer. */ + copy(rb.buffer_[masked_pos:], bytes[:brotli_min_size_t(n, uint(rb.total_size_-uint32(masked_pos)))]) + + /* Copy into the beginning of the buffer */ + copy(rb.buffer_, bytes[rb.size_-uint32(masked_pos):][:uint32(n)-(rb.size_-uint32(masked_pos))]) + } + } + { + var not_first_lap bool = rb.pos_&(1<<31) != 0 + var rb_pos_mask uint32 = (1 << 31) - 1 + rb.data_[0] = rb.buffer_[rb.size_-2] + rb.data_[1] = rb.buffer_[rb.size_-1] + rb.pos_ = (rb.pos_ & rb_pos_mask) + uint32(uint32(n)&rb_pos_mask) + if not_first_lap { + /* Wrap, but preserve not-a-first-lap feature. */ + rb.pos_ |= 1 << 31 + } + } +} diff --git a/vendor/github.com/andybalholm/brotli/state.go b/vendor/github.com/andybalholm/brotli/state.go new file mode 100644 index 0000000000..38d753ebe4 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/state.go @@ -0,0 +1,294 @@ +package brotli + +import "io" + +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Brotli state for partial streaming decoding. */ +const ( + stateUninited = iota + stateLargeWindowBits + stateInitialize + stateMetablockBegin + stateMetablockHeader + stateMetablockHeader2 + stateContextModes + stateCommandBegin + stateCommandInner + stateCommandPostDecodeLiterals + stateCommandPostWrapCopy + stateUncompressed + stateMetadata + stateCommandInnerWrite + stateMetablockDone + stateCommandPostWrite1 + stateCommandPostWrite2 + stateHuffmanCode0 + stateHuffmanCode1 + stateHuffmanCode2 + stateHuffmanCode3 + stateContextMap1 + stateContextMap2 + stateTreeGroup + stateDone +) + +const ( + stateMetablockHeaderNone = iota + stateMetablockHeaderEmpty + stateMetablockHeaderNibbles + stateMetablockHeaderSize + stateMetablockHeaderUncompressed + stateMetablockHeaderReserved + stateMetablockHeaderBytes + stateMetablockHeaderMetadata +) + +const ( + stateUncompressedNone = iota + stateUncompressedWrite +) + +const ( + stateTreeGroupNone = iota + stateTreeGroupLoop +) + +const ( + stateContextMapNone = iota + stateContextMapReadPrefix + stateContextMapHuffman + stateContextMapDecode + stateContextMapTransform +) + +const ( + stateHuffmanNone = iota + stateHuffmanSimpleSize + stateHuffmanSimpleRead + stateHuffmanSimpleBuild + stateHuffmanComplex + stateHuffmanLengthSymbols +) + +const ( + stateDecodeUint8None = iota + stateDecodeUint8Short + stateDecodeUint8Long +) + +const ( + stateReadBlockLengthNone = iota + stateReadBlockLengthSuffix +) + +type Reader struct { + src io.Reader + buf []byte // scratch space for reading from src + in []byte // current chunk to decode; usually aliases buf + + state int + loop_counter int + br bitReader + buffer struct { + u64 uint64 + u8 [8]byte + } + buffer_length uint32 + pos int + max_backward_distance int + max_distance int + ringbuffer_size int + ringbuffer_mask int + dist_rb_idx int + dist_rb [4]int + error_code int + sub_loop_counter uint32 + ringbuffer []byte + ringbuffer_end []byte + htree_command []huffmanCode + context_lookup []byte + context_map_slice []byte + dist_context_map_slice []byte + literal_hgroup huffmanTreeGroup + insert_copy_hgroup huffmanTreeGroup + distance_hgroup huffmanTreeGroup + block_type_trees []huffmanCode + block_len_trees []huffmanCode + trivial_literal_context int + distance_context int + meta_block_remaining_len int + block_length_index uint32 + block_length [3]uint32 + num_block_types [3]uint32 + block_type_rb [6]uint32 + distance_postfix_bits uint32 + num_direct_distance_codes uint32 + distance_postfix_mask int + num_dist_htrees uint32 + dist_context_map []byte + literal_htree []huffmanCode + dist_htree_index byte + repeat_code_len uint32 + prev_code_len uint32 + copy_length int + distance_code int + rb_roundtrips uint + partial_pos_out uint + symbol uint32 + repeat uint32 + space uint32 + table [32]huffmanCode + symbol_lists symbolList + symbols_lists_array [huffmanMaxCodeLength + 1 + numCommandSymbols]uint16 + next_symbol [32]int + code_length_code_lengths [codeLengthCodes]byte + code_length_histo [16]uint16 + htree_index int + next []huffmanCode + context_index uint32 + max_run_length_prefix uint32 + code uint32 + context_map_table [huffmanMaxSize272]huffmanCode + substate_metablock_header int + substate_tree_group int + substate_context_map int + substate_uncompressed int + substate_huffman int + substate_decode_uint8 int + substate_read_block_length int + is_last_metablock uint + is_uncompressed uint + is_metadata uint + should_wrap_ringbuffer uint + canny_ringbuffer_allocation uint + large_window bool + size_nibbles uint + window_bits uint32 + new_ringbuffer_size int + num_literal_htrees uint32 + context_map []byte + context_modes []byte + dictionary *dictionary + transforms *transforms + trivial_literal_contexts [8]uint32 +} + +func decoderStateInit(s *Reader) bool { + s.error_code = 0 /* BROTLI_DECODER_NO_ERROR */ + + initBitReader(&s.br) + s.state = stateUninited + s.large_window = false + s.substate_metablock_header = stateMetablockHeaderNone + s.substate_tree_group = stateTreeGroupNone + s.substate_context_map = stateContextMapNone + s.substate_uncompressed = stateUncompressedNone + s.substate_huffman = stateHuffmanNone + s.substate_decode_uint8 = stateDecodeUint8None + s.substate_read_block_length = stateReadBlockLengthNone + + s.buffer_length = 0 + s.loop_counter = 0 + s.pos = 0 + s.rb_roundtrips = 0 + s.partial_pos_out = 0 + + s.block_type_trees = nil + s.block_len_trees = nil + s.ringbuffer_size = 0 + s.new_ringbuffer_size = 0 + s.ringbuffer_mask = 0 + + s.context_map = nil + s.context_modes = nil + s.dist_context_map = nil + s.context_map_slice = nil + s.dist_context_map_slice = nil + + s.sub_loop_counter = 0 + + s.literal_hgroup.codes = nil + s.literal_hgroup.htrees = nil + s.insert_copy_hgroup.codes = nil + s.insert_copy_hgroup.htrees = nil + s.distance_hgroup.codes = nil + s.distance_hgroup.htrees = nil + + s.is_last_metablock = 0 + s.is_uncompressed = 0 + s.is_metadata = 0 + s.should_wrap_ringbuffer = 0 + s.canny_ringbuffer_allocation = 1 + + s.window_bits = 0 + s.max_distance = 0 + s.dist_rb[0] = 16 + s.dist_rb[1] = 15 + s.dist_rb[2] = 11 + s.dist_rb[3] = 4 + s.dist_rb_idx = 0 + s.block_type_trees = nil + s.block_len_trees = nil + + s.symbol_lists.storage = s.symbols_lists_array[:] + s.symbol_lists.offset = huffmanMaxCodeLength + 1 + + s.dictionary = getDictionary() + s.transforms = getTransforms() + + return true +} + +func decoderStateMetablockBegin(s *Reader) { + s.meta_block_remaining_len = 0 + s.block_length[0] = 1 << 24 + s.block_length[1] = 1 << 24 + s.block_length[2] = 1 << 24 + s.num_block_types[0] = 1 + s.num_block_types[1] = 1 + s.num_block_types[2] = 1 + s.block_type_rb[0] = 1 + s.block_type_rb[1] = 0 + s.block_type_rb[2] = 1 + s.block_type_rb[3] = 0 + s.block_type_rb[4] = 1 + s.block_type_rb[5] = 0 + s.context_map = nil + s.context_modes = nil + s.dist_context_map = nil + s.context_map_slice = nil + s.literal_htree = nil + s.dist_context_map_slice = nil + s.dist_htree_index = 0 + s.context_lookup = nil + s.literal_hgroup.codes = nil + s.literal_hgroup.htrees = nil + s.insert_copy_hgroup.codes = nil + s.insert_copy_hgroup.htrees = nil + s.distance_hgroup.codes = nil + s.distance_hgroup.htrees = nil +} + +func decoderStateCleanupAfterMetablock(s *Reader) { + s.context_modes = nil + s.context_map = nil + s.dist_context_map = nil + s.literal_hgroup.htrees = nil + s.insert_copy_hgroup.htrees = nil + s.distance_hgroup.htrees = nil +} + +func decoderHuffmanTreeGroupInit(s *Reader, group *huffmanTreeGroup, alphabet_size uint32, max_symbol uint32, ntrees uint32) bool { + var max_table_size uint = uint(kMaxHuffmanTableSize[(alphabet_size+31)>>5]) + group.alphabet_size = uint16(alphabet_size) + group.max_symbol = uint16(max_symbol) + group.num_htrees = uint16(ntrees) + group.htrees = make([][]huffmanCode, ntrees) + group.codes = make([]huffmanCode, (uint(ntrees) * max_table_size)) + return !(group.codes == nil) +} diff --git a/vendor/github.com/andybalholm/brotli/static_dict.go b/vendor/github.com/andybalholm/brotli/static_dict.go new file mode 100644 index 0000000000..bc05566d6f --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/static_dict.go @@ -0,0 +1,662 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Class to model the static dictionary. */ + +const maxStaticDictionaryMatchLen = 37 + +const kInvalidMatch uint32 = 0xFFFFFFF + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ +func hash(data []byte) uint32 { + var h uint32 = binary.LittleEndian.Uint32(data) * kDictHashMul32 + + /* The higher bits contain more mixture from the multiplication, + so we take our results from there. */ + return h >> uint(32-kDictNumBits) +} + +func addMatch(distance uint, len uint, len_code uint, matches []uint32) { + var match uint32 = uint32((distance << 5) + len_code) + matches[len] = brotli_min_uint32_t(matches[len], match) +} + +func dictMatchLength(dict *dictionary, data []byte, id uint, len uint, maxlen uint) uint { + var offset uint = uint(dict.offsets_by_length[len]) + len*id + return findMatchLengthWithLimit(dict.data[offset:], data, brotli_min_size_t(uint(len), maxlen)) +} + +func isMatch(d *dictionary, w dictWord, data []byte, max_length uint) bool { + if uint(w.len) > max_length { + return false + } else { + var offset uint = uint(d.offsets_by_length[w.len]) + uint(w.len)*uint(w.idx) + var dict []byte = d.data[offset:] + if w.transform == 0 { + /* Match against base dictionary word. */ + return findMatchLengthWithLimit(dict, data, uint(w.len)) == uint(w.len) + } else if w.transform == 10 { + /* Match against uppercase first transform. + Note that there are only ASCII uppercase words in the lookup table. */ + return dict[0] >= 'a' && dict[0] <= 'z' && (dict[0]^32) == data[0] && findMatchLengthWithLimit(dict[1:], data[1:], uint(w.len)-1) == uint(w.len-1) + } else { + /* Match against uppercase all transform. + Note that there are only ASCII uppercase words in the lookup table. */ + var i uint + for i = 0; i < uint(w.len); i++ { + if dict[i] >= 'a' && dict[i] <= 'z' { + if (dict[i] ^ 32) != data[i] { + return false + } + } else { + if dict[i] != data[i] { + return false + } + } + } + + return true + } + } +} + +func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_length uint, max_length uint, matches []uint32) bool { + var has_found_match bool = false + { + var offset uint = uint(dict.buckets[hash(data)]) + var end bool = offset == 0 + for !end { + w := dict.dict_words[offset] + offset++ + var l uint = uint(w.len) & 0x1F + var n uint = uint(1) << dict.words.size_bits_by_length[l] + var id uint = uint(w.idx) + end = !(w.len&0x80 == 0) + w.len = byte(l) + if w.transform == 0 { + var matchlen uint = dictMatchLength(dict.words, data, id, l, max_length) + var s []byte + var minlen uint + var maxlen uint + var len uint + + /* Transform "" + BROTLI_TRANSFORM_IDENTITY + "" */ + if matchlen == l { + addMatch(id, l, l, matches) + has_found_match = true + } + + /* Transforms "" + BROTLI_TRANSFORM_OMIT_LAST_1 + "" and + "" + BROTLI_TRANSFORM_OMIT_LAST_1 + "ing " */ + if matchlen >= l-1 { + addMatch(id+12*n, l-1, l, matches) + if l+2 < max_length && data[l-1] == 'i' && data[l] == 'n' && data[l+1] == 'g' && data[l+2] == ' ' { + addMatch(id+49*n, l+3, l, matches) + } + + has_found_match = true + } + + /* Transform "" + BROTLI_TRANSFORM_OMIT_LAST_# + "" (# = 2 .. 9) */ + minlen = min_length + + if l > 9 { + minlen = brotli_max_size_t(minlen, l-9) + } + maxlen = brotli_min_size_t(matchlen, l-2) + for len = minlen; len <= maxlen; len++ { + var cut uint = l - len + var transform_id uint = (cut << 2) + uint((dict.cutoffTransforms>>(cut*6))&0x3F) + addMatch(id+transform_id*n, uint(len), l, matches) + has_found_match = true + } + + if matchlen < l || l+6 >= max_length { + continue + } + + s = data[l:] + + /* Transforms "" + BROTLI_TRANSFORM_IDENTITY + */ + if s[0] == ' ' { + addMatch(id+n, l+1, l, matches) + if s[1] == 'a' { + if s[2] == ' ' { + addMatch(id+28*n, l+3, l, matches) + } else if s[2] == 's' { + if s[3] == ' ' { + addMatch(id+46*n, l+4, l, matches) + } + } else if s[2] == 't' { + if s[3] == ' ' { + addMatch(id+60*n, l+4, l, matches) + } + } else if s[2] == 'n' { + if s[3] == 'd' && s[4] == ' ' { + addMatch(id+10*n, l+5, l, matches) + } + } + } else if s[1] == 'b' { + if s[2] == 'y' && s[3] == ' ' { + addMatch(id+38*n, l+4, l, matches) + } + } else if s[1] == 'i' { + if s[2] == 'n' { + if s[3] == ' ' { + addMatch(id+16*n, l+4, l, matches) + } + } else if s[2] == 's' { + if s[3] == ' ' { + addMatch(id+47*n, l+4, l, matches) + } + } + } else if s[1] == 'f' { + if s[2] == 'o' { + if s[3] == 'r' && s[4] == ' ' { + addMatch(id+25*n, l+5, l, matches) + } + } else if s[2] == 'r' { + if s[3] == 'o' && s[4] == 'm' && s[5] == ' ' { + addMatch(id+37*n, l+6, l, matches) + } + } + } else if s[1] == 'o' { + if s[2] == 'f' { + if s[3] == ' ' { + addMatch(id+8*n, l+4, l, matches) + } + } else if s[2] == 'n' { + if s[3] == ' ' { + addMatch(id+45*n, l+4, l, matches) + } + } + } else if s[1] == 'n' { + if s[2] == 'o' && s[3] == 't' && s[4] == ' ' { + addMatch(id+80*n, l+5, l, matches) + } + } else if s[1] == 't' { + if s[2] == 'h' { + if s[3] == 'e' { + if s[4] == ' ' { + addMatch(id+5*n, l+5, l, matches) + } + } else if s[3] == 'a' { + if s[4] == 't' && s[5] == ' ' { + addMatch(id+29*n, l+6, l, matches) + } + } + } else if s[2] == 'o' { + if s[3] == ' ' { + addMatch(id+17*n, l+4, l, matches) + } + } + } else if s[1] == 'w' { + if s[2] == 'i' && s[3] == 't' && s[4] == 'h' && s[5] == ' ' { + addMatch(id+35*n, l+6, l, matches) + } + } + } else if s[0] == '"' { + addMatch(id+19*n, l+1, l, matches) + if s[1] == '>' { + addMatch(id+21*n, l+2, l, matches) + } + } else if s[0] == '.' { + addMatch(id+20*n, l+1, l, matches) + if s[1] == ' ' { + addMatch(id+31*n, l+2, l, matches) + if s[2] == 'T' && s[3] == 'h' { + if s[4] == 'e' { + if s[5] == ' ' { + addMatch(id+43*n, l+6, l, matches) + } + } else if s[4] == 'i' { + if s[5] == 's' && s[6] == ' ' { + addMatch(id+75*n, l+7, l, matches) + } + } + } + } + } else if s[0] == ',' { + addMatch(id+76*n, l+1, l, matches) + if s[1] == ' ' { + addMatch(id+14*n, l+2, l, matches) + } + } else if s[0] == '\n' { + addMatch(id+22*n, l+1, l, matches) + if s[1] == '\t' { + addMatch(id+50*n, l+2, l, matches) + } + } else if s[0] == ']' { + addMatch(id+24*n, l+1, l, matches) + } else if s[0] == '\'' { + addMatch(id+36*n, l+1, l, matches) + } else if s[0] == ':' { + addMatch(id+51*n, l+1, l, matches) + } else if s[0] == '(' { + addMatch(id+57*n, l+1, l, matches) + } else if s[0] == '=' { + if s[1] == '"' { + addMatch(id+70*n, l+2, l, matches) + } else if s[1] == '\'' { + addMatch(id+86*n, l+2, l, matches) + } + } else if s[0] == 'a' { + if s[1] == 'l' && s[2] == ' ' { + addMatch(id+84*n, l+3, l, matches) + } + } else if s[0] == 'e' { + if s[1] == 'd' { + if s[2] == ' ' { + addMatch(id+53*n, l+3, l, matches) + } + } else if s[1] == 'r' { + if s[2] == ' ' { + addMatch(id+82*n, l+3, l, matches) + } + } else if s[1] == 's' { + if s[2] == 't' && s[3] == ' ' { + addMatch(id+95*n, l+4, l, matches) + } + } + } else if s[0] == 'f' { + if s[1] == 'u' && s[2] == 'l' && s[3] == ' ' { + addMatch(id+90*n, l+4, l, matches) + } + } else if s[0] == 'i' { + if s[1] == 'v' { + if s[2] == 'e' && s[3] == ' ' { + addMatch(id+92*n, l+4, l, matches) + } + } else if s[1] == 'z' { + if s[2] == 'e' && s[3] == ' ' { + addMatch(id+100*n, l+4, l, matches) + } + } + } else if s[0] == 'l' { + if s[1] == 'e' { + if s[2] == 's' && s[3] == 's' && s[4] == ' ' { + addMatch(id+93*n, l+5, l, matches) + } + } else if s[1] == 'y' { + if s[2] == ' ' { + addMatch(id+61*n, l+3, l, matches) + } + } + } else if s[0] == 'o' { + if s[1] == 'u' && s[2] == 's' && s[3] == ' ' { + addMatch(id+106*n, l+4, l, matches) + } + } + } else { + var is_all_caps bool = (w.transform != transformUppercaseFirst) + /* Set is_all_caps=0 for BROTLI_TRANSFORM_UPPERCASE_FIRST and + is_all_caps=1 otherwise (BROTLI_TRANSFORM_UPPERCASE_ALL) + transform. */ + + var s []byte + if !isMatch(dict.words, w, data, max_length) { + continue + } + + /* Transform "" + kUppercase{First,All} + "" */ + var tmp int + if is_all_caps { + tmp = 44 + } else { + tmp = 9 + } + addMatch(id+uint(tmp)*n, l, l, matches) + + has_found_match = true + if l+1 >= max_length { + continue + } + + /* Transforms "" + kUppercase{First,All} + */ + s = data[l:] + + if s[0] == ' ' { + var tmp int + if is_all_caps { + tmp = 68 + } else { + tmp = 4 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + } else if s[0] == '"' { + var tmp int + if is_all_caps { + tmp = 87 + } else { + tmp = 66 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + if s[1] == '>' { + var tmp int + if is_all_caps { + tmp = 97 + } else { + tmp = 69 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } + } else if s[0] == '.' { + var tmp int + if is_all_caps { + tmp = 101 + } else { + tmp = 79 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + if s[1] == ' ' { + var tmp int + if is_all_caps { + tmp = 114 + } else { + tmp = 88 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } + } else if s[0] == ',' { + var tmp int + if is_all_caps { + tmp = 112 + } else { + tmp = 99 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + if s[1] == ' ' { + var tmp int + if is_all_caps { + tmp = 107 + } else { + tmp = 58 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } + } else if s[0] == '\'' { + var tmp int + if is_all_caps { + tmp = 94 + } else { + tmp = 74 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + } else if s[0] == '(' { + var tmp int + if is_all_caps { + tmp = 113 + } else { + tmp = 78 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + } else if s[0] == '=' { + if s[1] == '"' { + var tmp int + if is_all_caps { + tmp = 105 + } else { + tmp = 104 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } else if s[1] == '\'' { + var tmp int + if is_all_caps { + tmp = 116 + } else { + tmp = 108 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } + } + } + } + } + + /* Transforms with prefixes " " and "." */ + if max_length >= 5 && (data[0] == ' ' || data[0] == '.') { + var is_space bool = (data[0] == ' ') + var offset uint = uint(dict.buckets[hash(data[1:])]) + var end bool = offset == 0 + for !end { + w := dict.dict_words[offset] + offset++ + var l uint = uint(w.len) & 0x1F + var n uint = uint(1) << dict.words.size_bits_by_length[l] + var id uint = uint(w.idx) + end = !(w.len&0x80 == 0) + w.len = byte(l) + if w.transform == 0 { + var s []byte + if !isMatch(dict.words, w, data[1:], max_length-1) { + continue + } + + /* Transforms " " + BROTLI_TRANSFORM_IDENTITY + "" and + "." + BROTLI_TRANSFORM_IDENTITY + "" */ + var tmp int + if is_space { + tmp = 6 + } else { + tmp = 32 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + + has_found_match = true + if l+2 >= max_length { + continue + } + + /* Transforms " " + BROTLI_TRANSFORM_IDENTITY + and + "." + BROTLI_TRANSFORM_IDENTITY + + */ + s = data[l+1:] + + if s[0] == ' ' { + var tmp int + if is_space { + tmp = 2 + } else { + tmp = 77 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } else if s[0] == '(' { + var tmp int + if is_space { + tmp = 89 + } else { + tmp = 67 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } else if is_space { + if s[0] == ',' { + addMatch(id+103*n, l+2, l, matches) + if s[1] == ' ' { + addMatch(id+33*n, l+3, l, matches) + } + } else if s[0] == '.' { + addMatch(id+71*n, l+2, l, matches) + if s[1] == ' ' { + addMatch(id+52*n, l+3, l, matches) + } + } else if s[0] == '=' { + if s[1] == '"' { + addMatch(id+81*n, l+3, l, matches) + } else if s[1] == '\'' { + addMatch(id+98*n, l+3, l, matches) + } + } + } + } else if is_space { + var is_all_caps bool = (w.transform != transformUppercaseFirst) + /* Set is_all_caps=0 for BROTLI_TRANSFORM_UPPERCASE_FIRST and + is_all_caps=1 otherwise (BROTLI_TRANSFORM_UPPERCASE_ALL) + transform. */ + + var s []byte + if !isMatch(dict.words, w, data[1:], max_length-1) { + continue + } + + /* Transforms " " + kUppercase{First,All} + "" */ + var tmp int + if is_all_caps { + tmp = 85 + } else { + tmp = 30 + } + addMatch(id+uint(tmp)*n, l+1, l, matches) + + has_found_match = true + if l+2 >= max_length { + continue + } + + /* Transforms " " + kUppercase{First,All} + */ + s = data[l+1:] + + if s[0] == ' ' { + var tmp int + if is_all_caps { + tmp = 83 + } else { + tmp = 15 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + } else if s[0] == ',' { + if !is_all_caps { + addMatch(id+109*n, l+2, l, matches) + } + + if s[1] == ' ' { + var tmp int + if is_all_caps { + tmp = 111 + } else { + tmp = 65 + } + addMatch(id+uint(tmp)*n, l+3, l, matches) + } + } else if s[0] == '.' { + var tmp int + if is_all_caps { + tmp = 115 + } else { + tmp = 96 + } + addMatch(id+uint(tmp)*n, l+2, l, matches) + if s[1] == ' ' { + var tmp int + if is_all_caps { + tmp = 117 + } else { + tmp = 91 + } + addMatch(id+uint(tmp)*n, l+3, l, matches) + } + } else if s[0] == '=' { + if s[1] == '"' { + var tmp int + if is_all_caps { + tmp = 110 + } else { + tmp = 118 + } + addMatch(id+uint(tmp)*n, l+3, l, matches) + } else if s[1] == '\'' { + var tmp int + if is_all_caps { + tmp = 119 + } else { + tmp = 120 + } + addMatch(id+uint(tmp)*n, l+3, l, matches) + } + } + } + } + } + + if max_length >= 6 { + /* Transforms with prefixes "e ", "s ", ", " and "\xC2\xA0" */ + if (data[1] == ' ' && (data[0] == 'e' || data[0] == 's' || data[0] == ',')) || (data[0] == 0xC2 && data[1] == 0xA0) { + var offset uint = uint(dict.buckets[hash(data[2:])]) + var end bool = offset == 0 + for !end { + w := dict.dict_words[offset] + offset++ + var l uint = uint(w.len) & 0x1F + var n uint = uint(1) << dict.words.size_bits_by_length[l] + var id uint = uint(w.idx) + end = !(w.len&0x80 == 0) + w.len = byte(l) + if w.transform == 0 && isMatch(dict.words, w, data[2:], max_length-2) { + if data[0] == 0xC2 { + addMatch(id+102*n, l+2, l, matches) + has_found_match = true + } else if l+2 < max_length && data[l+2] == ' ' { + var t uint = 13 + if data[0] == 'e' { + t = 18 + } else if data[0] == 's' { + t = 7 + } + addMatch(id+t*n, l+3, l, matches) + has_found_match = true + } + } + } + } + } + + if max_length >= 9 { + /* Transforms with prefixes " the " and ".com/" */ + if (data[0] == ' ' && data[1] == 't' && data[2] == 'h' && data[3] == 'e' && data[4] == ' ') || (data[0] == '.' && data[1] == 'c' && data[2] == 'o' && data[3] == 'm' && data[4] == '/') { + var offset uint = uint(dict.buckets[hash(data[5:])]) + var end bool = offset == 0 + for !end { + w := dict.dict_words[offset] + offset++ + var l uint = uint(w.len) & 0x1F + var n uint = uint(1) << dict.words.size_bits_by_length[l] + var id uint = uint(w.idx) + end = !(w.len&0x80 == 0) + w.len = byte(l) + if w.transform == 0 && isMatch(dict.words, w, data[5:], max_length-5) { + var tmp int + if data[0] == ' ' { + tmp = 41 + } else { + tmp = 72 + } + addMatch(id+uint(tmp)*n, l+5, l, matches) + has_found_match = true + if l+5 < max_length { + var s []byte = data[l+5:] + if data[0] == ' ' { + if l+8 < max_length && s[0] == ' ' && s[1] == 'o' && s[2] == 'f' && s[3] == ' ' { + addMatch(id+62*n, l+9, l, matches) + if l+12 < max_length && s[4] == 't' && s[5] == 'h' && s[6] == 'e' && s[7] == ' ' { + addMatch(id+73*n, l+13, l, matches) + } + } + } + } + } + } + } + } + + return has_found_match +} diff --git a/vendor/github.com/andybalholm/brotli/static_dict_lut.go b/vendor/github.com/andybalholm/brotli/static_dict_lut.go new file mode 100644 index 0000000000..b33963e967 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/static_dict_lut.go @@ -0,0 +1,75094 @@ +package brotli + +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Lookup table for static dictionary and transforms. */ + +type dictWord struct { + len byte + transform byte + idx uint16 +} + +const kDictNumBits int = 15 + +const kDictHashMul32 uint32 = 0x1E35A7BD + +var kStaticDictionaryBuckets = [32768]uint16{ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 6, + 0, + 0, + 0, + 0, + 0, + 20, + 0, + 0, + 0, + 21, + 0, + 22, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23, + 0, + 0, + 25, + 0, + 29, + 0, + 53, + 0, + 0, + 0, + 0, + 0, + 0, + 55, + 0, + 0, + 0, + 0, + 0, + 0, + 61, + 76, + 0, + 0, + 0, + 94, + 0, + 0, + 0, + 0, + 0, + 0, + 96, + 0, + 97, + 0, + 98, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 99, + 101, + 106, + 108, + 0, + 0, + 0, + 0, + 0, + 110, + 0, + 111, + 112, + 0, + 113, + 118, + 124, + 0, + 0, + 0, + 0, + 0, + 125, + 128, + 0, + 0, + 0, + 0, + 129, + 0, + 0, + 131, + 0, + 0, + 0, + 0, + 0, + 0, + 132, + 0, + 0, + 135, + 0, + 0, + 0, + 137, + 0, + 0, + 0, + 0, + 0, + 138, + 139, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 142, + 143, + 144, + 0, + 0, + 0, + 0, + 0, + 145, + 0, + 0, + 0, + 146, + 149, + 151, + 152, + 0, + 0, + 153, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 154, + 0, + 0, + 0, + 0, + 0, + 0, + 155, + 0, + 0, + 0, + 0, + 160, + 182, + 0, + 0, + 0, + 0, + 0, + 0, + 183, + 0, + 0, + 0, + 188, + 189, + 0, + 0, + 192, + 0, + 0, + 0, + 0, + 0, + 0, + 194, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 197, + 202, + 209, + 0, + 0, + 210, + 0, + 224, + 0, + 0, + 0, + 225, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 231, + 0, + 0, + 0, + 232, + 0, + 240, + 0, + 0, + 242, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 244, + 0, + 0, + 0, + 246, + 0, + 0, + 249, + 251, + 253, + 0, + 0, + 0, + 0, + 0, + 258, + 0, + 0, + 261, + 263, + 0, + 0, + 0, + 267, + 0, + 0, + 268, + 0, + 269, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 271, + 0, + 0, + 0, + 0, + 0, + 0, + 272, + 0, + 273, + 0, + 277, + 0, + 278, + 286, + 0, + 0, + 0, + 0, + 287, + 0, + 289, + 290, + 291, + 0, + 0, + 0, + 295, + 0, + 0, + 296, + 297, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 298, + 0, + 0, + 0, + 299, + 0, + 0, + 305, + 0, + 324, + 0, + 0, + 0, + 0, + 0, + 327, + 0, + 328, + 329, + 0, + 0, + 0, + 0, + 336, + 0, + 0, + 340, + 0, + 341, + 342, + 343, + 0, + 0, + 346, + 0, + 348, + 0, + 0, + 0, + 0, + 0, + 0, + 349, + 351, + 0, + 0, + 355, + 0, + 363, + 0, + 364, + 0, + 368, + 369, + 0, + 370, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 372, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 373, + 0, + 375, + 0, + 0, + 0, + 0, + 376, + 377, + 0, + 0, + 394, + 395, + 396, + 0, + 0, + 398, + 0, + 0, + 0, + 0, + 400, + 0, + 0, + 408, + 0, + 0, + 0, + 0, + 420, + 0, + 0, + 0, + 0, + 0, + 0, + 421, + 0, + 0, + 422, + 423, + 0, + 0, + 429, + 435, + 436, + 442, + 0, + 0, + 443, + 0, + 444, + 445, + 453, + 456, + 0, + 457, + 0, + 0, + 0, + 0, + 0, + 458, + 0, + 0, + 0, + 459, + 0, + 0, + 0, + 460, + 0, + 462, + 463, + 465, + 0, + 0, + 0, + 0, + 0, + 0, + 466, + 469, + 0, + 0, + 0, + 0, + 0, + 0, + 470, + 0, + 0, + 0, + 474, + 0, + 476, + 0, + 0, + 0, + 0, + 483, + 0, + 485, + 0, + 0, + 0, + 486, + 0, + 0, + 488, + 491, + 492, + 0, + 0, + 497, + 499, + 500, + 0, + 501, + 0, + 0, + 0, + 505, + 0, + 0, + 506, + 0, + 0, + 0, + 507, + 0, + 0, + 0, + 509, + 0, + 0, + 0, + 0, + 511, + 512, + 519, + 0, + 0, + 0, + 0, + 0, + 0, + 529, + 530, + 0, + 0, + 0, + 534, + 0, + 0, + 0, + 0, + 543, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 553, + 0, + 0, + 0, + 0, + 557, + 560, + 0, + 0, + 0, + 0, + 0, + 0, + 561, + 0, + 564, + 0, + 0, + 0, + 0, + 0, + 0, + 565, + 566, + 0, + 575, + 0, + 619, + 0, + 620, + 0, + 0, + 623, + 624, + 0, + 0, + 0, + 625, + 0, + 0, + 626, + 627, + 0, + 0, + 628, + 0, + 0, + 0, + 0, + 630, + 0, + 631, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 641, + 0, + 0, + 0, + 0, + 643, + 656, + 668, + 0, + 0, + 0, + 673, + 0, + 0, + 0, + 674, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 682, + 0, + 687, + 0, + 690, + 0, + 693, + 699, + 700, + 0, + 0, + 0, + 0, + 0, + 0, + 704, + 705, + 0, + 0, + 0, + 0, + 707, + 710, + 0, + 711, + 0, + 0, + 0, + 0, + 726, + 0, + 0, + 729, + 0, + 0, + 0, + 730, + 731, + 0, + 0, + 0, + 0, + 0, + 752, + 0, + 0, + 0, + 762, + 0, + 763, + 0, + 0, + 767, + 0, + 0, + 0, + 770, + 774, + 0, + 0, + 775, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 776, + 0, + 0, + 0, + 777, + 783, + 0, + 0, + 0, + 785, + 788, + 0, + 0, + 0, + 0, + 790, + 0, + 0, + 0, + 793, + 0, + 0, + 0, + 0, + 794, + 0, + 0, + 804, + 819, + 821, + 0, + 827, + 0, + 0, + 0, + 834, + 0, + 0, + 835, + 0, + 0, + 0, + 841, + 0, + 844, + 0, + 850, + 851, + 859, + 0, + 860, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 874, + 0, + 876, + 0, + 877, + 890, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 893, + 894, + 898, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 899, + 0, + 0, + 0, + 900, + 904, + 906, + 0, + 0, + 0, + 907, + 0, + 908, + 909, + 0, + 910, + 0, + 0, + 0, + 0, + 911, + 0, + 0, + 0, + 0, + 0, + 916, + 0, + 0, + 0, + 922, + 925, + 0, + 930, + 0, + 934, + 0, + 0, + 0, + 0, + 0, + 943, + 0, + 0, + 944, + 0, + 953, + 954, + 0, + 0, + 0, + 0, + 0, + 0, + 955, + 0, + 962, + 963, + 0, + 0, + 976, + 0, + 0, + 977, + 978, + 979, + 980, + 0, + 981, + 0, + 0, + 0, + 0, + 984, + 0, + 0, + 985, + 0, + 0, + 987, + 989, + 991, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 992, + 0, + 0, + 0, + 993, + 0, + 0, + 0, + 0, + 0, + 0, + 996, + 0, + 0, + 0, + 1000, + 0, + 0, + 0, + 0, + 0, + 1002, + 0, + 0, + 0, + 0, + 1005, + 1007, + 0, + 0, + 0, + 1009, + 0, + 0, + 0, + 1010, + 0, + 0, + 0, + 0, + 0, + 0, + 1011, + 0, + 1012, + 0, + 0, + 0, + 0, + 1014, + 1016, + 0, + 0, + 0, + 1020, + 0, + 1021, + 0, + 0, + 0, + 0, + 1022, + 0, + 0, + 0, + 1024, + 0, + 0, + 0, + 0, + 0, + 0, + 1025, + 0, + 0, + 1026, + 1027, + 0, + 0, + 0, + 0, + 0, + 1031, + 0, + 1033, + 0, + 0, + 0, + 0, + 1034, + 0, + 0, + 0, + 1037, + 1040, + 0, + 0, + 0, + 1042, + 1043, + 0, + 0, + 1053, + 0, + 1054, + 0, + 0, + 1057, + 0, + 0, + 0, + 1058, + 0, + 0, + 1060, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1061, + 0, + 0, + 1062, + 0, + 0, + 0, + 0, + 1063, + 0, + 0, + 0, + 0, + 1064, + 0, + 0, + 0, + 0, + 0, + 1065, + 0, + 0, + 0, + 0, + 1066, + 1067, + 0, + 0, + 0, + 1069, + 1070, + 1072, + 0, + 0, + 0, + 0, + 0, + 0, + 1073, + 0, + 1075, + 0, + 0, + 0, + 0, + 0, + 0, + 1080, + 1084, + 0, + 0, + 0, + 0, + 1088, + 0, + 0, + 0, + 0, + 0, + 0, + 1094, + 0, + 1095, + 0, + 1107, + 0, + 0, + 0, + 1112, + 1114, + 0, + 1119, + 0, + 1122, + 0, + 0, + 1126, + 0, + 1129, + 0, + 1130, + 0, + 0, + 0, + 0, + 0, + 1132, + 0, + 0, + 0, + 0, + 0, + 0, + 1144, + 0, + 0, + 1145, + 1146, + 0, + 1148, + 1149, + 0, + 0, + 1150, + 1151, + 0, + 0, + 0, + 0, + 1152, + 0, + 1153, + 0, + 0, + 0, + 0, + 0, + 1154, + 0, + 1163, + 0, + 0, + 0, + 1164, + 0, + 0, + 0, + 0, + 0, + 1165, + 0, + 1167, + 0, + 1170, + 0, + 0, + 0, + 0, + 0, + 1171, + 1172, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1173, + 1175, + 1177, + 0, + 1186, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1195, + 0, + 0, + 1221, + 0, + 0, + 1224, + 0, + 0, + 1227, + 0, + 0, + 0, + 0, + 0, + 1228, + 1229, + 0, + 0, + 1230, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1231, + 0, + 0, + 0, + 1233, + 0, + 0, + 1243, + 1244, + 1246, + 1248, + 0, + 0, + 0, + 0, + 1254, + 1255, + 1258, + 1259, + 0, + 0, + 0, + 1260, + 0, + 0, + 1261, + 0, + 0, + 0, + 1262, + 1264, + 0, + 0, + 1265, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1266, + 0, + 1267, + 0, + 0, + 0, + 0, + 1273, + 1274, + 1276, + 1289, + 0, + 0, + 1291, + 1292, + 1293, + 0, + 0, + 1294, + 1295, + 1296, + 0, + 0, + 0, + 0, + 1302, + 0, + 1304, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1311, + 1312, + 0, + 1314, + 0, + 1316, + 1320, + 1321, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1322, + 1323, + 1324, + 0, + 1335, + 0, + 1336, + 0, + 0, + 0, + 0, + 1341, + 1342, + 0, + 1346, + 0, + 1357, + 0, + 0, + 0, + 1358, + 1360, + 0, + 0, + 0, + 0, + 0, + 0, + 1361, + 0, + 0, + 0, + 1362, + 1365, + 0, + 1366, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1379, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1386, + 0, + 1388, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1395, + 0, + 0, + 0, + 0, + 1403, + 0, + 1405, + 0, + 0, + 1407, + 0, + 0, + 0, + 0, + 0, + 1408, + 1409, + 0, + 1410, + 0, + 0, + 0, + 1412, + 1413, + 1416, + 0, + 0, + 1429, + 1451, + 0, + 0, + 1454, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1455, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1456, + 0, + 0, + 0, + 0, + 1459, + 1460, + 1461, + 1475, + 0, + 0, + 0, + 0, + 0, + 0, + 1477, + 0, + 1480, + 0, + 1481, + 0, + 0, + 1486, + 0, + 0, + 1495, + 0, + 0, + 0, + 1496, + 0, + 0, + 1498, + 1499, + 1501, + 1520, + 1521, + 0, + 0, + 0, + 1526, + 0, + 0, + 0, + 0, + 1528, + 1529, + 0, + 1533, + 1536, + 0, + 0, + 0, + 1537, + 1538, + 1549, + 0, + 1550, + 1558, + 1559, + 1572, + 0, + 1573, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1575, + 0, + 0, + 0, + 0, + 0, + 1579, + 0, + 1599, + 0, + 1603, + 0, + 1604, + 0, + 1605, + 0, + 0, + 0, + 0, + 0, + 1608, + 1610, + 0, + 0, + 0, + 0, + 1611, + 0, + 1615, + 0, + 1616, + 1618, + 0, + 1619, + 0, + 0, + 1622, + 0, + 0, + 0, + 0, + 1634, + 0, + 0, + 0, + 1635, + 0, + 0, + 0, + 1641, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1643, + 0, + 0, + 0, + 1650, + 0, + 0, + 1652, + 0, + 0, + 0, + 0, + 0, + 1653, + 0, + 0, + 0, + 1654, + 0, + 0, + 0, + 0, + 1655, + 0, + 1662, + 0, + 0, + 1663, + 1664, + 0, + 0, + 1668, + 0, + 0, + 1669, + 1670, + 0, + 1672, + 1673, + 0, + 0, + 0, + 0, + 0, + 1674, + 0, + 0, + 0, + 1675, + 1676, + 1680, + 0, + 1682, + 0, + 0, + 1687, + 0, + 0, + 0, + 0, + 0, + 1704, + 0, + 0, + 1705, + 0, + 0, + 1721, + 0, + 0, + 0, + 0, + 1734, + 1735, + 0, + 0, + 0, + 0, + 1737, + 0, + 0, + 0, + 0, + 1739, + 0, + 0, + 1740, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1741, + 1743, + 0, + 0, + 0, + 0, + 1745, + 0, + 0, + 0, + 1749, + 0, + 0, + 0, + 1751, + 0, + 0, + 0, + 0, + 0, + 0, + 1760, + 0, + 0, + 0, + 0, + 1765, + 0, + 0, + 0, + 0, + 0, + 1784, + 0, + 1785, + 1787, + 0, + 0, + 0, + 0, + 1788, + 1789, + 0, + 0, + 0, + 0, + 1790, + 1791, + 1793, + 0, + 1798, + 1799, + 0, + 0, + 0, + 0, + 1801, + 0, + 1803, + 1805, + 0, + 0, + 0, + 1806, + 1811, + 0, + 1812, + 1814, + 0, + 1821, + 0, + 0, + 0, + 0, + 0, + 1822, + 1833, + 0, + 0, + 0, + 0, + 0, + 0, + 1848, + 0, + 0, + 0, + 0, + 0, + 0, + 1857, + 0, + 0, + 0, + 1859, + 0, + 0, + 0, + 0, + 1861, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1866, + 0, + 1921, + 1925, + 0, + 0, + 0, + 1929, + 1930, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1931, + 0, + 0, + 0, + 0, + 1932, + 0, + 0, + 0, + 1934, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1946, + 0, + 0, + 1948, + 0, + 0, + 0, + 0, + 1950, + 0, + 1957, + 0, + 1958, + 0, + 0, + 0, + 0, + 0, + 1965, + 1967, + 0, + 0, + 0, + 0, + 1968, + 0, + 1969, + 0, + 1971, + 1972, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1973, + 0, + 0, + 0, + 0, + 1975, + 0, + 0, + 0, + 0, + 1976, + 1979, + 0, + 1982, + 0, + 0, + 0, + 0, + 1984, + 1988, + 0, + 0, + 0, + 0, + 1990, + 2004, + 2008, + 0, + 0, + 0, + 2012, + 2013, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2015, + 0, + 2016, + 2017, + 0, + 0, + 0, + 0, + 2021, + 0, + 0, + 2025, + 0, + 0, + 0, + 0, + 0, + 2029, + 2036, + 2040, + 0, + 2042, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2043, + 0, + 0, + 0, + 0, + 0, + 2045, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2046, + 2047, + 0, + 2048, + 2049, + 0, + 2059, + 0, + 0, + 2063, + 0, + 2064, + 2065, + 0, + 0, + 2066, + 0, + 0, + 0, + 0, + 0, + 0, + 2069, + 0, + 0, + 0, + 0, + 2070, + 0, + 2071, + 0, + 2072, + 0, + 0, + 0, + 0, + 2080, + 2082, + 2083, + 0, + 0, + 0, + 0, + 0, + 2085, + 0, + 2086, + 2088, + 2089, + 2105, + 0, + 0, + 0, + 0, + 2107, + 0, + 0, + 2116, + 2117, + 0, + 2120, + 0, + 0, + 2122, + 0, + 0, + 0, + 0, + 0, + 2123, + 0, + 0, + 2125, + 2127, + 2128, + 0, + 0, + 0, + 2130, + 0, + 0, + 0, + 2137, + 2139, + 2140, + 2141, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2144, + 2145, + 0, + 0, + 2146, + 2149, + 0, + 0, + 0, + 0, + 2150, + 0, + 0, + 2151, + 2158, + 0, + 2159, + 0, + 2160, + 0, + 0, + 0, + 0, + 0, + 0, + 2161, + 2162, + 0, + 0, + 2194, + 2202, + 0, + 0, + 0, + 0, + 0, + 0, + 2205, + 2217, + 0, + 2220, + 0, + 2221, + 0, + 2222, + 2224, + 0, + 0, + 0, + 0, + 2237, + 0, + 0, + 0, + 0, + 0, + 2238, + 0, + 2239, + 2241, + 0, + 0, + 2242, + 0, + 0, + 0, + 0, + 0, + 2243, + 0, + 0, + 0, + 0, + 0, + 0, + 2252, + 0, + 0, + 2253, + 0, + 0, + 0, + 2257, + 2258, + 0, + 0, + 0, + 2260, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2262, + 0, + 2264, + 0, + 0, + 0, + 0, + 0, + 2269, + 2270, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2271, + 0, + 2273, + 0, + 0, + 0, + 0, + 2277, + 0, + 0, + 0, + 0, + 2278, + 0, + 0, + 0, + 0, + 2279, + 0, + 2280, + 0, + 2283, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2287, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2289, + 2290, + 0, + 0, + 0, + 0, + 2291, + 0, + 2292, + 0, + 0, + 0, + 2293, + 2295, + 2296, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2298, + 0, + 0, + 0, + 0, + 0, + 2303, + 0, + 2305, + 0, + 0, + 2306, + 0, + 2307, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2313, + 2314, + 2315, + 2316, + 0, + 0, + 2318, + 0, + 2319, + 0, + 2322, + 0, + 0, + 2323, + 0, + 2324, + 0, + 2326, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2335, + 0, + 2336, + 2338, + 2339, + 0, + 2340, + 0, + 0, + 0, + 2355, + 0, + 2375, + 0, + 2382, + 2386, + 0, + 2387, + 0, + 0, + 2394, + 0, + 0, + 0, + 0, + 2395, + 0, + 2397, + 0, + 0, + 0, + 0, + 0, + 2398, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2399, + 2402, + 2404, + 2408, + 2411, + 0, + 0, + 0, + 2413, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2415, + 0, + 0, + 2416, + 2417, + 2419, + 0, + 2420, + 0, + 0, + 0, + 0, + 0, + 2425, + 0, + 0, + 0, + 2426, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2427, + 2428, + 0, + 2429, + 0, + 0, + 2430, + 2434, + 0, + 2436, + 0, + 0, + 0, + 0, + 0, + 0, + 2441, + 2442, + 0, + 2445, + 0, + 0, + 2446, + 2457, + 0, + 2459, + 0, + 0, + 2462, + 0, + 2464, + 0, + 2477, + 0, + 2478, + 2486, + 0, + 0, + 0, + 2491, + 0, + 0, + 2493, + 0, + 0, + 2494, + 0, + 2495, + 0, + 2513, + 2523, + 0, + 0, + 0, + 0, + 2524, + 0, + 0, + 0, + 0, + 0, + 0, + 2528, + 2529, + 2530, + 0, + 0, + 2531, + 0, + 2533, + 0, + 0, + 2534, + 2535, + 0, + 2536, + 2537, + 0, + 2538, + 0, + 2539, + 2540, + 0, + 0, + 0, + 2545, + 2546, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2548, + 0, + 0, + 2549, + 0, + 2550, + 2555, + 0, + 0, + 0, + 0, + 0, + 2557, + 0, + 2560, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2561, + 0, + 2576, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2577, + 2578, + 0, + 0, + 0, + 2579, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2580, + 0, + 0, + 0, + 0, + 2581, + 0, + 0, + 0, + 0, + 2583, + 0, + 2584, + 0, + 2588, + 2590, + 0, + 0, + 0, + 2591, + 0, + 0, + 0, + 0, + 2593, + 2594, + 0, + 2595, + 0, + 2601, + 2602, + 0, + 0, + 2603, + 0, + 2605, + 0, + 0, + 0, + 2606, + 2607, + 2611, + 0, + 2615, + 0, + 0, + 0, + 2617, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2619, + 0, + 0, + 2620, + 0, + 0, + 0, + 2621, + 0, + 2623, + 0, + 2625, + 0, + 0, + 2628, + 2629, + 0, + 0, + 2635, + 2636, + 2637, + 0, + 0, + 2639, + 0, + 0, + 0, + 2642, + 0, + 0, + 0, + 0, + 2643, + 0, + 2644, + 0, + 2649, + 0, + 0, + 0, + 0, + 0, + 0, + 2655, + 2656, + 0, + 0, + 2657, + 0, + 0, + 0, + 0, + 0, + 2658, + 0, + 0, + 0, + 0, + 0, + 2659, + 0, + 0, + 0, + 0, + 2664, + 2685, + 0, + 2687, + 0, + 2688, + 0, + 0, + 2689, + 0, + 0, + 2694, + 0, + 2695, + 0, + 0, + 2698, + 0, + 2701, + 2706, + 0, + 0, + 0, + 2707, + 0, + 2709, + 2710, + 2711, + 0, + 0, + 0, + 2720, + 2730, + 2735, + 0, + 0, + 0, + 0, + 2738, + 2740, + 0, + 0, + 0, + 0, + 2747, + 0, + 0, + 0, + 0, + 0, + 0, + 2748, + 0, + 0, + 2749, + 0, + 0, + 0, + 0, + 0, + 2750, + 0, + 0, + 2752, + 2754, + 0, + 0, + 0, + 0, + 0, + 2758, + 0, + 0, + 0, + 0, + 2762, + 0, + 0, + 0, + 0, + 2763, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2764, + 2767, + 0, + 0, + 0, + 0, + 2768, + 0, + 0, + 2770, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2771, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2772, + 0, + 0, + 0, + 0, + 0, + 2773, + 2776, + 0, + 0, + 2783, + 0, + 0, + 2784, + 0, + 2789, + 0, + 2790, + 0, + 0, + 0, + 2792, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2793, + 2795, + 0, + 0, + 0, + 0, + 0, + 0, + 2796, + 0, + 0, + 0, + 0, + 0, + 0, + 2797, + 2799, + 0, + 0, + 0, + 0, + 2803, + 0, + 0, + 0, + 0, + 2806, + 0, + 2807, + 2808, + 2817, + 2819, + 0, + 0, + 0, + 0, + 0, + 2821, + 0, + 0, + 0, + 0, + 2822, + 2823, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2824, + 0, + 0, + 2828, + 0, + 2834, + 0, + 0, + 0, + 0, + 0, + 0, + 2836, + 0, + 2838, + 0, + 0, + 2839, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2841, + 0, + 0, + 0, + 2842, + 0, + 0, + 0, + 0, + 0, + 2843, + 2844, + 0, + 0, + 0, + 0, + 2846, + 0, + 0, + 2847, + 0, + 2849, + 0, + 2853, + 0, + 0, + 0, + 0, + 0, + 2857, + 0, + 0, + 0, + 0, + 2858, + 0, + 2859, + 0, + 0, + 2860, + 0, + 2862, + 2868, + 0, + 0, + 0, + 0, + 2875, + 0, + 2876, + 0, + 0, + 2877, + 2878, + 2884, + 2889, + 2890, + 0, + 0, + 2891, + 0, + 0, + 2892, + 0, + 0, + 0, + 2906, + 2912, + 0, + 2913, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2916, + 0, + 2934, + 0, + 0, + 0, + 0, + 0, + 2935, + 0, + 0, + 0, + 0, + 2939, + 0, + 2940, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2941, + 0, + 0, + 0, + 2946, + 0, + 2949, + 0, + 0, + 2950, + 2954, + 2955, + 0, + 0, + 0, + 2959, + 2961, + 0, + 0, + 2962, + 0, + 2963, + 0, + 0, + 0, + 0, + 0, + 0, + 2964, + 2965, + 2966, + 2967, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2969, + 0, + 0, + 0, + 0, + 0, + 2970, + 2975, + 0, + 2982, + 2983, + 2984, + 0, + 0, + 0, + 0, + 0, + 2989, + 0, + 0, + 2990, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2991, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2998, + 0, + 3000, + 3001, + 0, + 0, + 3002, + 0, + 0, + 0, + 3003, + 0, + 0, + 3012, + 0, + 0, + 3022, + 0, + 0, + 3024, + 0, + 0, + 3025, + 3027, + 0, + 0, + 0, + 3030, + 0, + 0, + 0, + 0, + 3034, + 3035, + 0, + 0, + 3036, + 0, + 3039, + 0, + 3049, + 0, + 0, + 3050, + 0, + 0, + 0, + 0, + 0, + 0, + 3051, + 0, + 3053, + 0, + 0, + 0, + 0, + 3057, + 0, + 3058, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3063, + 0, + 0, + 3073, + 3074, + 3078, + 3079, + 0, + 3080, + 3086, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3087, + 0, + 3092, + 0, + 3095, + 0, + 3099, + 0, + 0, + 0, + 3100, + 0, + 3101, + 3102, + 0, + 3122, + 0, + 0, + 0, + 3124, + 0, + 3125, + 0, + 0, + 0, + 0, + 0, + 0, + 3132, + 3134, + 0, + 0, + 3136, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3147, + 0, + 0, + 3149, + 0, + 0, + 0, + 0, + 0, + 3150, + 3151, + 3152, + 0, + 0, + 0, + 0, + 3158, + 0, + 0, + 3160, + 0, + 0, + 3161, + 0, + 0, + 3162, + 0, + 3163, + 3166, + 3168, + 0, + 0, + 3169, + 3170, + 0, + 0, + 3171, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3182, + 0, + 3184, + 0, + 0, + 3188, + 0, + 0, + 3194, + 0, + 0, + 0, + 0, + 0, + 0, + 3204, + 0, + 0, + 0, + 0, + 3209, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3216, + 3217, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3219, + 0, + 0, + 3220, + 3222, + 0, + 3223, + 0, + 0, + 0, + 0, + 3224, + 0, + 3225, + 3226, + 0, + 3228, + 3233, + 0, + 3239, + 3241, + 3242, + 0, + 0, + 3251, + 3252, + 3253, + 3255, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3260, + 0, + 0, + 3261, + 0, + 0, + 0, + 3267, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3271, + 0, + 0, + 0, + 3278, + 0, + 3282, + 0, + 0, + 0, + 3284, + 0, + 0, + 0, + 3285, + 3286, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3287, + 3292, + 0, + 0, + 0, + 0, + 3294, + 3296, + 0, + 0, + 3299, + 3300, + 3301, + 0, + 3302, + 0, + 0, + 0, + 0, + 0, + 3304, + 3306, + 0, + 0, + 0, + 0, + 0, + 0, + 3308, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3311, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3312, + 3314, + 3315, + 0, + 3318, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3319, + 0, + 0, + 0, + 0, + 0, + 3321, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3322, + 0, + 0, + 3324, + 3325, + 0, + 0, + 3326, + 0, + 0, + 3328, + 3329, + 3331, + 0, + 0, + 3335, + 0, + 0, + 3337, + 0, + 3338, + 0, + 0, + 0, + 0, + 3343, + 3347, + 0, + 0, + 0, + 3348, + 0, + 0, + 3351, + 0, + 0, + 0, + 0, + 0, + 0, + 3354, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3355, + 0, + 0, + 3365, + 3366, + 3367, + 0, + 0, + 0, + 0, + 0, + 0, + 3368, + 3369, + 0, + 3370, + 0, + 0, + 3373, + 0, + 0, + 3376, + 0, + 0, + 3377, + 0, + 3379, + 3387, + 0, + 0, + 0, + 0, + 0, + 3390, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3402, + 0, + 3403, + 3436, + 3437, + 3439, + 0, + 0, + 3441, + 0, + 0, + 0, + 3442, + 0, + 0, + 3449, + 0, + 0, + 0, + 3450, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3451, + 0, + 0, + 3452, + 0, + 3453, + 3456, + 0, + 3457, + 0, + 0, + 3458, + 0, + 3459, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3460, + 0, + 0, + 3469, + 3470, + 0, + 0, + 3475, + 0, + 0, + 0, + 3480, + 3487, + 3489, + 0, + 3490, + 0, + 0, + 3491, + 3499, + 0, + 3500, + 0, + 0, + 3501, + 0, + 0, + 0, + 3502, + 0, + 3514, + 0, + 0, + 0, + 3516, + 3517, + 0, + 0, + 0, + 3518, + 0, + 0, + 0, + 0, + 3520, + 3521, + 3522, + 0, + 0, + 3526, + 3530, + 0, + 0, + 0, + 0, + 3531, + 0, + 0, + 0, + 0, + 3536, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3539, + 3541, + 0, + 0, + 3542, + 3544, + 0, + 3547, + 3548, + 0, + 0, + 3550, + 0, + 3553, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3554, + 0, + 3555, + 0, + 3558, + 0, + 3559, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3563, + 0, + 3581, + 0, + 0, + 0, + 3599, + 0, + 0, + 0, + 3600, + 0, + 3601, + 0, + 3602, + 3603, + 0, + 0, + 3606, + 3608, + 0, + 3610, + 3611, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3612, + 3616, + 3619, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3624, + 3628, + 0, + 3629, + 3634, + 3635, + 0, + 0, + 0, + 0, + 0, + 0, + 3636, + 0, + 3637, + 0, + 0, + 3638, + 3651, + 0, + 0, + 0, + 0, + 0, + 0, + 3652, + 3653, + 0, + 0, + 0, + 0, + 3656, + 3657, + 0, + 0, + 0, + 0, + 0, + 3658, + 0, + 0, + 0, + 0, + 3659, + 0, + 3661, + 3663, + 3664, + 0, + 3665, + 0, + 3692, + 0, + 0, + 0, + 3694, + 3696, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3698, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3700, + 0, + 0, + 3701, + 0, + 0, + 0, + 3708, + 3709, + 0, + 0, + 0, + 3711, + 3712, + 0, + 0, + 0, + 0, + 0, + 3723, + 0, + 3724, + 3725, + 0, + 0, + 3726, + 0, + 0, + 0, + 0, + 0, + 0, + 3728, + 3729, + 0, + 3734, + 3735, + 3737, + 0, + 0, + 0, + 3743, + 0, + 3745, + 0, + 0, + 3746, + 0, + 0, + 3747, + 3748, + 0, + 3757, + 0, + 3759, + 3766, + 3767, + 0, + 3768, + 0, + 0, + 0, + 0, + 3769, + 0, + 0, + 3771, + 0, + 3774, + 0, + 0, + 0, + 0, + 0, + 0, + 3775, + 0, + 0, + 0, + 0, + 0, + 0, + 3776, + 0, + 3777, + 3786, + 0, + 3788, + 3789, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3791, + 0, + 3811, + 0, + 0, + 0, + 0, + 0, + 3814, + 3815, + 3816, + 3820, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3821, + 0, + 0, + 3825, + 0, + 0, + 0, + 0, + 3835, + 0, + 0, + 3848, + 3849, + 0, + 0, + 0, + 0, + 3850, + 3851, + 3853, + 0, + 0, + 0, + 0, + 3859, + 0, + 3860, + 3862, + 0, + 0, + 0, + 0, + 0, + 3863, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3873, + 0, + 3874, + 0, + 3875, + 3886, + 0, + 3887, + 0, + 0, + 0, + 0, + 3892, + 3913, + 0, + 3914, + 0, + 0, + 0, + 3925, + 3931, + 0, + 0, + 0, + 0, + 3934, + 3941, + 3942, + 0, + 0, + 0, + 0, + 3943, + 0, + 0, + 0, + 3944, + 0, + 0, + 0, + 0, + 0, + 3945, + 0, + 3947, + 0, + 0, + 0, + 3956, + 3957, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3958, + 0, + 3959, + 3965, + 0, + 0, + 0, + 0, + 3966, + 0, + 0, + 0, + 3967, + 0, + 0, + 0, + 3968, + 3974, + 0, + 0, + 0, + 0, + 0, + 3975, + 3977, + 3978, + 0, + 0, + 0, + 0, + 3980, + 0, + 3985, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3986, + 4011, + 0, + 0, + 4017, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4018, + 0, + 0, + 0, + 0, + 4019, + 0, + 4023, + 0, + 0, + 0, + 4027, + 4028, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4031, + 4034, + 0, + 0, + 4035, + 4037, + 4039, + 4040, + 0, + 0, + 0, + 0, + 0, + 4059, + 0, + 4060, + 4061, + 0, + 4062, + 4063, + 4066, + 0, + 0, + 4072, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4088, + 0, + 0, + 0, + 0, + 0, + 4091, + 0, + 0, + 0, + 0, + 4094, + 4095, + 0, + 0, + 4096, + 0, + 0, + 0, + 0, + 0, + 4098, + 4099, + 0, + 0, + 0, + 4101, + 0, + 4104, + 0, + 0, + 0, + 4105, + 4108, + 0, + 4113, + 0, + 0, + 4115, + 4116, + 0, + 4126, + 0, + 0, + 4127, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4128, + 4132, + 4133, + 0, + 4134, + 0, + 0, + 0, + 4137, + 0, + 0, + 4141, + 0, + 0, + 0, + 0, + 4144, + 4146, + 4147, + 0, + 0, + 0, + 0, + 4148, + 0, + 0, + 4311, + 0, + 0, + 0, + 4314, + 4329, + 0, + 4331, + 4332, + 0, + 4333, + 0, + 4334, + 0, + 0, + 0, + 4335, + 0, + 4336, + 0, + 0, + 0, + 4337, + 0, + 0, + 0, + 4342, + 4345, + 4346, + 4350, + 0, + 4351, + 4352, + 0, + 4354, + 4355, + 0, + 0, + 4364, + 0, + 0, + 0, + 0, + 4369, + 0, + 0, + 0, + 4373, + 0, + 4374, + 0, + 0, + 0, + 0, + 4377, + 0, + 0, + 0, + 0, + 4378, + 0, + 0, + 0, + 4380, + 0, + 0, + 0, + 4381, + 4382, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4384, + 0, + 0, + 0, + 0, + 4385, + 0, + 0, + 0, + 4386, + 0, + 0, + 0, + 4391, + 4398, + 0, + 0, + 0, + 0, + 4407, + 4409, + 0, + 0, + 0, + 0, + 4410, + 0, + 0, + 4411, + 0, + 4414, + 4415, + 4418, + 0, + 4427, + 4428, + 4430, + 0, + 4431, + 0, + 4448, + 0, + 0, + 0, + 0, + 0, + 4449, + 0, + 0, + 0, + 4451, + 4452, + 0, + 4453, + 4454, + 0, + 4456, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4459, + 0, + 4463, + 0, + 0, + 0, + 0, + 0, + 4466, + 0, + 4467, + 0, + 4469, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4470, + 4471, + 0, + 4473, + 0, + 0, + 4475, + 0, + 0, + 0, + 0, + 4477, + 4478, + 0, + 0, + 0, + 4479, + 4481, + 0, + 4482, + 0, + 4484, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4486, + 0, + 0, + 4488, + 0, + 0, + 4497, + 0, + 4508, + 0, + 0, + 4510, + 4511, + 0, + 4520, + 4523, + 0, + 4524, + 0, + 4525, + 0, + 4527, + 0, + 0, + 4528, + 0, + 0, + 0, + 0, + 4530, + 0, + 4531, + 0, + 0, + 4532, + 0, + 0, + 0, + 4533, + 0, + 0, + 0, + 0, + 0, + 4535, + 0, + 0, + 0, + 4536, + 0, + 0, + 0, + 0, + 0, + 4541, + 4543, + 4544, + 4545, + 4547, + 0, + 4548, + 0, + 0, + 0, + 0, + 4550, + 4551, + 0, + 4553, + 0, + 0, + 0, + 0, + 4562, + 0, + 0, + 4571, + 0, + 0, + 0, + 4574, + 0, + 0, + 0, + 4575, + 0, + 4576, + 0, + 4577, + 0, + 0, + 0, + 4581, + 0, + 0, + 0, + 0, + 0, + 4582, + 0, + 0, + 4586, + 0, + 0, + 0, + 4588, + 0, + 0, + 4597, + 0, + 4598, + 0, + 0, + 0, + 0, + 4616, + 4617, + 0, + 4618, + 0, + 0, + 0, + 0, + 4619, + 0, + 4620, + 0, + 0, + 4621, + 0, + 4624, + 0, + 0, + 0, + 0, + 0, + 4625, + 0, + 0, + 0, + 0, + 4657, + 0, + 4659, + 0, + 4667, + 0, + 0, + 0, + 4668, + 4670, + 0, + 4672, + 0, + 0, + 0, + 0, + 0, + 4673, + 4676, + 0, + 0, + 0, + 0, + 4687, + 0, + 0, + 0, + 0, + 4697, + 0, + 0, + 0, + 0, + 4699, + 0, + 4701, + 0, + 0, + 0, + 0, + 4702, + 0, + 0, + 4706, + 0, + 0, + 4713, + 0, + 0, + 0, + 4714, + 4715, + 4716, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4717, + 0, + 0, + 4720, + 0, + 4721, + 4729, + 4735, + 0, + 0, + 0, + 4737, + 0, + 0, + 0, + 4739, + 0, + 0, + 0, + 4740, + 0, + 0, + 0, + 4741, + 0, + 0, + 0, + 0, + 0, + 4742, + 0, + 4745, + 4746, + 4747, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4748, + 0, + 0, + 0, + 4749, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4751, + 4786, + 0, + 4787, + 0, + 4788, + 4796, + 0, + 0, + 4797, + 4798, + 0, + 4799, + 4806, + 4807, + 0, + 0, + 0, + 0, + 4809, + 4810, + 0, + 0, + 0, + 0, + 0, + 0, + 4811, + 0, + 0, + 0, + 0, + 0, + 4812, + 0, + 4813, + 0, + 0, + 4815, + 0, + 4821, + 4822, + 0, + 0, + 0, + 0, + 4823, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4824, + 0, + 0, + 0, + 0, + 4826, + 0, + 0, + 0, + 4828, + 0, + 4829, + 0, + 0, + 0, + 4843, + 0, + 0, + 4847, + 0, + 4853, + 4855, + 4858, + 0, + 0, + 0, + 0, + 0, + 4859, + 0, + 4864, + 0, + 0, + 4879, + 0, + 0, + 0, + 0, + 4880, + 0, + 0, + 0, + 0, + 4881, + 0, + 4882, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4883, + 0, + 0, + 0, + 0, + 4884, + 0, + 0, + 0, + 0, + 0, + 4886, + 4887, + 4888, + 4894, + 4896, + 0, + 4902, + 0, + 0, + 4905, + 0, + 0, + 4915, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4916, + 4917, + 4919, + 4921, + 0, + 0, + 0, + 0, + 0, + 4926, + 0, + 0, + 0, + 0, + 4927, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4929, + 0, + 4930, + 4931, + 0, + 4938, + 0, + 4952, + 0, + 4953, + 4957, + 4960, + 4964, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5019, + 5020, + 5022, + 0, + 0, + 0, + 0, + 0, + 5023, + 0, + 0, + 0, + 5024, + 0, + 0, + 0, + 5025, + 0, + 0, + 0, + 0, + 5028, + 0, + 0, + 0, + 0, + 5029, + 5030, + 5031, + 0, + 5033, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5034, + 5035, + 0, + 5036, + 0, + 0, + 5037, + 0, + 0, + 0, + 0, + 5038, + 0, + 0, + 5039, + 0, + 0, + 0, + 5041, + 5042, + 0, + 0, + 0, + 0, + 5044, + 5049, + 5054, + 0, + 5055, + 0, + 5057, + 0, + 0, + 0, + 5060, + 0, + 0, + 0, + 0, + 0, + 5063, + 0, + 5064, + 5065, + 0, + 5067, + 0, + 0, + 0, + 5068, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5076, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5077, + 0, + 0, + 5078, + 5080, + 0, + 0, + 5083, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5085, + 0, + 0, + 0, + 0, + 0, + 0, + 5098, + 5099, + 5101, + 5105, + 5107, + 0, + 5108, + 0, + 5109, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5110, + 0, + 0, + 0, + 0, + 0, + 5117, + 5118, + 0, + 5121, + 0, + 5122, + 0, + 0, + 5130, + 0, + 0, + 0, + 5137, + 0, + 0, + 0, + 5148, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5151, + 5154, + 0, + 0, + 0, + 5155, + 0, + 0, + 5156, + 5159, + 5161, + 0, + 0, + 0, + 0, + 5162, + 0, + 0, + 0, + 0, + 5163, + 5164, + 0, + 5166, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5167, + 0, + 0, + 0, + 5172, + 0, + 0, + 0, + 0, + 0, + 0, + 5178, + 5179, + 0, + 0, + 5190, + 0, + 0, + 5191, + 5192, + 5194, + 0, + 0, + 5198, + 5201, + 0, + 0, + 0, + 0, + 0, + 5203, + 0, + 5206, + 5209, + 0, + 0, + 0, + 0, + 0, + 0, + 5213, + 0, + 5214, + 5216, + 0, + 0, + 0, + 0, + 0, + 5217, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5218, + 5219, + 0, + 5231, + 0, + 0, + 5244, + 5249, + 0, + 5254, + 0, + 5255, + 0, + 0, + 5257, + 0, + 0, + 0, + 0, + 0, + 5258, + 0, + 5260, + 5270, + 0, + 5277, + 0, + 0, + 0, + 0, + 0, + 0, + 5280, + 5281, + 5282, + 5283, + 0, + 0, + 0, + 0, + 0, + 5284, + 0, + 5285, + 0, + 0, + 0, + 0, + 0, + 5287, + 5288, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5289, + 5291, + 0, + 0, + 5294, + 0, + 0, + 5295, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5304, + 0, + 0, + 5306, + 5307, + 5308, + 0, + 5309, + 0, + 0, + 5310, + 0, + 0, + 0, + 0, + 5311, + 5312, + 0, + 5313, + 0, + 0, + 0, + 0, + 0, + 5316, + 0, + 0, + 0, + 5317, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5325, + 0, + 0, + 0, + 0, + 0, + 0, + 5326, + 0, + 5327, + 5329, + 0, + 5332, + 0, + 0, + 0, + 0, + 5338, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5340, + 0, + 0, + 5341, + 0, + 0, + 0, + 5342, + 0, + 5343, + 5344, + 0, + 0, + 5345, + 0, + 0, + 0, + 0, + 0, + 0, + 5347, + 5348, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5349, + 0, + 5350, + 0, + 5354, + 0, + 0, + 0, + 0, + 5358, + 0, + 0, + 5359, + 0, + 0, + 5361, + 0, + 0, + 5365, + 0, + 5367, + 0, + 5373, + 0, + 0, + 0, + 5379, + 0, + 0, + 0, + 5380, + 0, + 0, + 0, + 5382, + 0, + 5384, + 0, + 0, + 0, + 0, + 0, + 0, + 5385, + 0, + 0, + 0, + 0, + 5387, + 0, + 0, + 0, + 0, + 0, + 0, + 5388, + 5390, + 5393, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5396, + 0, + 0, + 0, + 0, + 5397, + 5402, + 0, + 0, + 0, + 0, + 0, + 5403, + 0, + 0, + 0, + 5404, + 5405, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5406, + 0, + 0, + 0, + 0, + 5410, + 0, + 0, + 5411, + 0, + 5415, + 0, + 0, + 0, + 0, + 5416, + 5434, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5438, + 0, + 5440, + 0, + 0, + 0, + 0, + 0, + 0, + 5441, + 5442, + 0, + 0, + 0, + 5443, + 5444, + 5447, + 0, + 0, + 5448, + 5449, + 5451, + 0, + 0, + 0, + 5456, + 5457, + 0, + 0, + 0, + 5459, + 0, + 0, + 0, + 5461, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5464, + 0, + 5466, + 0, + 0, + 5467, + 0, + 5470, + 0, + 0, + 5473, + 0, + 0, + 5474, + 0, + 0, + 5476, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5477, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5484, + 0, + 0, + 5485, + 5486, + 0, + 0, + 0, + 0, + 0, + 5488, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5489, + 0, + 0, + 0, + 0, + 0, + 5507, + 0, + 0, + 0, + 5510, + 0, + 5511, + 0, + 0, + 5512, + 0, + 0, + 0, + 5513, + 0, + 5515, + 0, + 0, + 5516, + 5517, + 0, + 5518, + 0, + 0, + 5522, + 0, + 0, + 0, + 0, + 0, + 5534, + 5535, + 0, + 0, + 5536, + 0, + 5538, + 0, + 0, + 5543, + 0, + 5544, + 0, + 0, + 5545, + 0, + 5547, + 0, + 5557, + 0, + 0, + 5558, + 0, + 5560, + 5567, + 0, + 0, + 0, + 0, + 5568, + 0, + 0, + 0, + 5571, + 5573, + 0, + 5574, + 0, + 5575, + 0, + 0, + 0, + 0, + 5577, + 0, + 0, + 5598, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5600, + 5609, + 0, + 0, + 0, + 0, + 5610, + 0, + 0, + 5612, + 0, + 5624, + 0, + 5625, + 0, + 0, + 0, + 5629, + 0, + 5641, + 0, + 5642, + 5643, + 0, + 0, + 0, + 0, + 0, + 0, + 5651, + 0, + 0, + 0, + 5652, + 5653, + 0, + 5661, + 5662, + 5678, + 0, + 5679, + 0, + 0, + 0, + 0, + 5685, + 5686, + 0, + 0, + 0, + 0, + 0, + 5690, + 5692, + 0, + 5703, + 0, + 0, + 0, + 0, + 0, + 5706, + 0, + 0, + 0, + 0, + 5707, + 0, + 0, + 0, + 0, + 0, + 0, + 5708, + 0, + 0, + 5709, + 0, + 5710, + 0, + 0, + 0, + 5712, + 0, + 5733, + 0, + 5734, + 5735, + 0, + 0, + 5744, + 5751, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5752, + 0, + 5754, + 0, + 0, + 0, + 0, + 0, + 0, + 5757, + 5758, + 0, + 5760, + 5761, + 0, + 0, + 0, + 0, + 5763, + 5764, + 5765, + 0, + 5766, + 0, + 5767, + 5768, + 0, + 5770, + 0, + 0, + 0, + 0, + 5776, + 5780, + 0, + 0, + 0, + 0, + 5782, + 0, + 0, + 0, + 0, + 5784, + 0, + 0, + 5788, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5797, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5799, + 0, + 0, + 5801, + 0, + 0, + 0, + 5811, + 0, + 0, + 0, + 0, + 0, + 0, + 5816, + 0, + 0, + 5827, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5830, + 5831, + 0, + 0, + 5832, + 0, + 0, + 5833, + 0, + 5835, + 5844, + 5845, + 0, + 5846, + 0, + 0, + 0, + 0, + 0, + 5850, + 0, + 0, + 0, + 0, + 0, + 5852, + 0, + 5855, + 5857, + 0, + 0, + 5859, + 0, + 5861, + 0, + 0, + 5863, + 0, + 5865, + 0, + 0, + 0, + 5873, + 5875, + 0, + 0, + 0, + 5877, + 0, + 5879, + 0, + 0, + 0, + 5888, + 0, + 0, + 5889, + 5891, + 0, + 5894, + 0, + 0, + 0, + 0, + 0, + 0, + 5895, + 0, + 5897, + 0, + 0, + 0, + 0, + 0, + 0, + 5907, + 0, + 5911, + 0, + 0, + 5912, + 0, + 5913, + 5922, + 5924, + 0, + 5927, + 5928, + 0, + 0, + 0, + 0, + 5929, + 5930, + 0, + 5933, + 0, + 0, + 0, + 0, + 5949, + 0, + 0, + 5951, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5953, + 0, + 0, + 5954, + 0, + 5959, + 5960, + 5961, + 0, + 5964, + 0, + 0, + 0, + 5976, + 5978, + 5987, + 5990, + 0, + 0, + 0, + 0, + 0, + 5991, + 0, + 5992, + 0, + 0, + 0, + 5994, + 5995, + 0, + 0, + 5996, + 0, + 0, + 6001, + 6003, + 0, + 0, + 0, + 0, + 6007, + 0, + 0, + 0, + 0, + 0, + 6008, + 0, + 0, + 6009, + 0, + 6010, + 0, + 0, + 0, + 6011, + 6015, + 0, + 6017, + 0, + 6019, + 0, + 6023, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6025, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6026, + 0, + 6030, + 0, + 0, + 6032, + 0, + 0, + 0, + 6033, + 6038, + 6040, + 0, + 0, + 0, + 6041, + 6045, + 0, + 0, + 6046, + 0, + 0, + 6053, + 0, + 0, + 6054, + 0, + 6055, + 0, + 0, + 0, + 0, + 0, + 0, + 6057, + 0, + 6063, + 0, + 0, + 0, + 6064, + 0, + 6066, + 6071, + 6072, + 0, + 0, + 0, + 0, + 0, + 0, + 6075, + 6076, + 0, + 0, + 6077, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6078, + 6079, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6080, + 0, + 6083, + 0, + 0, + 0, + 0, + 0, + 6084, + 0, + 0, + 6088, + 0, + 6089, + 0, + 0, + 6093, + 6105, + 0, + 0, + 6107, + 0, + 6110, + 0, + 0, + 0, + 6111, + 6125, + 6126, + 0, + 0, + 0, + 6129, + 0, + 0, + 0, + 0, + 6130, + 0, + 0, + 0, + 6131, + 6134, + 0, + 0, + 0, + 0, + 0, + 0, + 6142, + 0, + 0, + 0, + 0, + 0, + 6144, + 0, + 0, + 6146, + 6151, + 6153, + 0, + 6156, + 0, + 6163, + 0, + 6180, + 6181, + 0, + 0, + 0, + 0, + 0, + 6182, + 0, + 0, + 0, + 0, + 6184, + 6195, + 0, + 0, + 6206, + 0, + 6208, + 0, + 0, + 6212, + 6213, + 6214, + 0, + 6215, + 0, + 0, + 0, + 6228, + 0, + 0, + 0, + 6234, + 0, + 0, + 0, + 0, + 0, + 0, + 6235, + 6240, + 0, + 6242, + 6243, + 6244, + 0, + 6250, + 6255, + 0, + 0, + 0, + 0, + 0, + 6257, + 0, + 0, + 0, + 6258, + 6278, + 0, + 6284, + 0, + 0, + 0, + 6285, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6286, + 0, + 0, + 0, + 6320, + 0, + 0, + 6322, + 6332, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6334, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6335, + 0, + 0, + 6337, + 0, + 6338, + 0, + 6339, + 6340, + 0, + 0, + 6356, + 6357, + 6369, + 0, + 0, + 0, + 6370, + 6371, + 6372, + 0, + 6373, + 0, + 0, + 0, + 0, + 0, + 6376, + 0, + 0, + 0, + 0, + 0, + 6382, + 6383, + 6384, + 0, + 0, + 0, + 0, + 6386, + 0, + 6389, + 6397, + 6400, + 6411, + 0, + 6414, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6415, + 6416, + 0, + 0, + 0, + 0, + 0, + 0, + 6417, + 0, + 0, + 0, + 0, + 6418, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6420, + 0, + 6421, + 6423, + 6425, + 0, + 6429, + 6430, + 0, + 6433, + 6438, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6439, + 6440, + 0, + 0, + 6441, + 0, + 0, + 6444, + 0, + 0, + 0, + 0, + 6446, + 0, + 0, + 0, + 0, + 6447, + 6448, + 0, + 0, + 6450, + 0, + 0, + 0, + 6454, + 0, + 0, + 6455, + 0, + 6461, + 0, + 0, + 0, + 0, + 0, + 0, + 6462, + 0, + 0, + 6463, + 0, + 6464, + 0, + 6465, + 6467, + 0, + 0, + 0, + 6468, + 0, + 6479, + 6480, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6481, + 0, + 0, + 6485, + 6487, + 0, + 0, + 0, + 0, + 0, + 0, + 6493, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6494, + 6495, + 6496, + 0, + 0, + 0, + 0, + 0, + 6498, + 0, + 0, + 0, + 6507, + 6508, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6511, + 6512, + 0, + 0, + 0, + 0, + 6513, + 0, + 0, + 0, + 6514, + 0, + 0, + 0, + 0, + 0, + 6516, + 0, + 0, + 6517, + 6518, + 0, + 0, + 0, + 6519, + 6520, + 6521, + 0, + 6523, + 0, + 0, + 0, + 0, + 6524, + 6528, + 0, + 6530, + 0, + 0, + 6532, + 0, + 6578, + 0, + 0, + 0, + 6583, + 0, + 6584, + 0, + 0, + 0, + 6587, + 0, + 0, + 0, + 6590, + 0, + 6591, + 0, + 0, + 0, + 0, + 0, + 6592, + 0, + 0, + 0, + 0, + 6593, + 6594, + 0, + 0, + 0, + 0, + 0, + 6599, + 6600, + 0, + 0, + 6601, + 6602, + 6604, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6608, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6610, + 6611, + 0, + 6615, + 0, + 6616, + 6618, + 6620, + 0, + 6637, + 0, + 0, + 0, + 0, + 6639, + 0, + 0, + 0, + 0, + 6641, + 0, + 6642, + 0, + 0, + 0, + 6647, + 0, + 6660, + 6663, + 0, + 6664, + 0, + 6666, + 6669, + 0, + 6675, + 6676, + 6677, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6678, + 0, + 0, + 0, + 6679, + 0, + 6680, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6693, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6704, + 6705, + 6706, + 0, + 0, + 6711, + 6713, + 0, + 0, + 0, + 0, + 0, + 6716, + 0, + 0, + 0, + 6717, + 0, + 6719, + 6724, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6725, + 6726, + 0, + 0, + 0, + 0, + 0, + 6728, + 6729, + 6735, + 0, + 6737, + 6742, + 0, + 0, + 6743, + 6750, + 0, + 6751, + 0, + 0, + 6752, + 6753, + 0, + 0, + 0, + 0, + 0, + 0, + 6754, + 0, + 0, + 0, + 0, + 0, + 6756, + 0, + 0, + 0, + 0, + 0, + 0, + 6763, + 0, + 0, + 6764, + 6765, + 0, + 0, + 0, + 6770, + 0, + 0, + 0, + 6776, + 6780, + 0, + 6781, + 0, + 0, + 0, + 6783, + 0, + 6784, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6785, + 0, + 0, + 0, + 6792, + 0, + 0, + 0, + 6793, + 0, + 0, + 6802, + 0, + 0, + 0, + 0, + 0, + 6803, + 0, + 0, + 0, + 6804, + 0, + 0, + 0, + 6812, + 0, + 0, + 6823, + 0, + 6824, + 6839, + 0, + 0, + 0, + 0, + 6852, + 0, + 0, + 6854, + 0, + 6856, + 6857, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6867, + 0, + 6868, + 6870, + 6872, + 0, + 0, + 0, + 6873, + 6874, + 0, + 0, + 0, + 0, + 0, + 6875, + 0, + 0, + 6877, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6878, + 0, + 0, + 0, + 6879, + 0, + 6880, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6887, + 0, + 6888, + 6891, + 6893, + 0, + 6895, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6899, + 0, + 0, + 0, + 0, + 6901, + 0, + 0, + 0, + 0, + 6910, + 0, + 6911, + 0, + 0, + 6912, + 0, + 0, + 6913, + 6914, + 0, + 0, + 0, + 6915, + 0, + 0, + 0, + 6916, + 6919, + 0, + 0, + 0, + 0, + 0, + 0, + 6924, + 0, + 6925, + 0, + 0, + 0, + 6926, + 6927, + 6928, + 0, + 6929, + 0, + 6930, + 0, + 0, + 6931, + 6935, + 0, + 6936, + 0, + 0, + 0, + 0, + 6939, + 6940, + 6941, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6942, + 6948, + 6949, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6952, + 6954, + 6963, + 6965, + 6966, + 0, + 0, + 6967, + 6968, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6969, + 0, + 0, + 6970, + 6979, + 0, + 0, + 6980, + 0, + 0, + 6983, + 0, + 0, + 0, + 0, + 0, + 6984, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6988, + 6990, + 6992, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 6995, + 0, + 0, + 0, + 7012, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7019, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7021, + 0, + 0, + 7022, + 7023, + 7028, + 0, + 7030, + 7033, + 0, + 0, + 0, + 0, + 0, + 0, + 7038, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7039, + 0, + 0, + 0, + 0, + 0, + 7046, + 0, + 7047, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7048, + 7052, + 0, + 0, + 0, + 0, + 0, + 7054, + 0, + 7060, + 0, + 0, + 0, + 0, + 7061, + 0, + 7065, + 0, + 0, + 0, + 0, + 7067, + 7069, + 0, + 7070, + 7071, + 7072, + 0, + 0, + 7078, + 0, + 7080, + 7081, + 0, + 7083, + 0, + 0, + 0, + 7084, + 7087, + 7088, + 0, + 0, + 7090, + 0, + 7093, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7107, + 0, + 0, + 7108, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7110, + 0, + 7114, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7115, + 0, + 7116, + 0, + 0, + 0, + 0, + 0, + 7117, + 0, + 0, + 7118, + 0, + 0, + 7124, + 0, + 7125, + 0, + 0, + 7126, + 0, + 0, + 0, + 0, + 7128, + 0, + 0, + 0, + 0, + 0, + 7129, + 0, + 7130, + 0, + 7132, + 7133, + 0, + 0, + 7134, + 0, + 0, + 7139, + 0, + 7148, + 7150, + 0, + 0, + 0, + 0, + 7152, + 0, + 0, + 0, + 7153, + 7156, + 7157, + 0, + 0, + 0, + 0, + 0, + 7158, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7163, + 7165, + 7169, + 0, + 7171, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7172, + 0, + 7173, + 7181, + 0, + 0, + 0, + 0, + 0, + 7182, + 7185, + 0, + 0, + 0, + 0, + 7187, + 0, + 7201, + 7204, + 0, + 0, + 0, + 0, + 0, + 7206, + 7207, + 0, + 0, + 0, + 0, + 7211, + 7216, + 0, + 7218, + 0, + 0, + 0, + 0, + 7226, + 7228, + 7230, + 7232, + 7233, + 7235, + 7237, + 0, + 0, + 0, + 0, + 7238, + 7241, + 0, + 7242, + 0, + 0, + 7247, + 0, + 0, + 0, + 7266, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7289, + 0, + 0, + 7290, + 7291, + 0, + 0, + 7292, + 0, + 7297, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7300, + 0, + 7301, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7302, + 0, + 0, + 0, + 0, + 7305, + 0, + 0, + 0, + 0, + 7307, + 0, + 7308, + 0, + 7310, + 0, + 7335, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7337, + 0, + 7343, + 7347, + 0, + 0, + 0, + 0, + 0, + 7348, + 0, + 7349, + 7350, + 7352, + 7354, + 0, + 0, + 0, + 0, + 7357, + 0, + 7358, + 7366, + 0, + 7367, + 7368, + 0, + 0, + 7373, + 0, + 0, + 0, + 7374, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7376, + 0, + 0, + 0, + 7377, + 0, + 0, + 0, + 0, + 0, + 7378, + 0, + 7379, + 7380, + 0, + 0, + 0, + 0, + 0, + 7383, + 0, + 0, + 7386, + 0, + 0, + 0, + 0, + 7398, + 0, + 0, + 0, + 7399, + 7400, + 0, + 7401, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7402, + 0, + 0, + 0, + 0, + 0, + 7405, + 0, + 0, + 0, + 0, + 0, + 7406, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7421, + 7427, + 7429, + 0, + 0, + 0, + 7435, + 0, + 0, + 7436, + 0, + 0, + 0, + 7437, + 0, + 0, + 0, + 0, + 0, + 0, + 7438, + 7443, + 0, + 7446, + 0, + 7448, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7456, + 0, + 0, + 0, + 0, + 0, + 7457, + 0, + 0, + 7461, + 0, + 0, + 0, + 0, + 0, + 7462, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7463, + 7466, + 7472, + 0, + 7476, + 0, + 0, + 7490, + 0, + 7491, + 0, + 0, + 7493, + 0, + 0, + 0, + 7498, + 7499, + 0, + 0, + 7508, + 0, + 0, + 0, + 0, + 0, + 7512, + 0, + 0, + 0, + 7513, + 7514, + 7516, + 0, + 0, + 0, + 0, + 7518, + 0, + 0, + 7519, + 7521, + 7522, + 0, + 0, + 0, + 7526, + 0, + 0, + 7529, + 0, + 0, + 7531, + 0, + 7536, + 0, + 7538, + 0, + 7539, + 0, + 0, + 7541, + 7542, + 7546, + 0, + 0, + 0, + 0, + 0, + 7547, + 0, + 7548, + 0, + 0, + 0, + 0, + 0, + 7550, + 0, + 0, + 7552, + 7553, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7554, + 7563, + 0, + 7573, + 0, + 0, + 0, + 0, + 0, + 0, + 7574, + 7576, + 0, + 7578, + 7581, + 7583, + 0, + 0, + 0, + 7584, + 0, + 7587, + 0, + 0, + 0, + 0, + 0, + 7589, + 0, + 0, + 0, + 7594, + 0, + 0, + 7595, + 0, + 0, + 7600, + 7602, + 7610, + 0, + 0, + 0, + 0, + 0, + 7612, + 0, + 7613, + 7614, + 0, + 0, + 7615, + 0, + 0, + 7616, + 0, + 7620, + 0, + 7621, + 7622, + 0, + 7623, + 0, + 0, + 0, + 0, + 7626, + 0, + 0, + 0, + 0, + 7627, + 7629, + 7631, + 0, + 0, + 7633, + 0, + 0, + 0, + 0, + 0, + 7639, + 0, + 7640, + 7642, + 0, + 0, + 7643, + 0, + 0, + 0, + 0, + 7644, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7645, + 0, + 0, + 0, + 0, + 0, + 7661, + 7662, + 7663, + 7665, + 0, + 7666, + 0, + 7667, + 0, + 7684, + 7688, + 7690, + 0, + 7691, + 0, + 0, + 0, + 0, + 0, + 0, + 7692, + 0, + 0, + 7700, + 0, + 7707, + 0, + 7708, + 0, + 7709, + 0, + 7721, + 0, + 0, + 0, + 7722, + 0, + 7724, + 0, + 0, + 0, + 0, + 0, + 0, + 7729, + 7731, + 0, + 7732, + 0, + 7733, + 7735, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7739, + 0, + 0, + 7741, + 7745, + 0, + 7748, + 0, + 0, + 0, + 7751, + 0, + 0, + 0, + 7752, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7753, + 0, + 0, + 7756, + 0, + 7757, + 0, + 7759, + 0, + 7760, + 0, + 0, + 0, + 0, + 7761, + 7768, + 0, + 0, + 7769, + 0, + 0, + 7770, + 0, + 0, + 7771, + 0, + 0, + 7772, + 0, + 0, + 7773, + 0, + 0, + 0, + 0, + 0, + 7778, + 7783, + 0, + 0, + 0, + 0, + 0, + 7784, + 7785, + 0, + 7790, + 0, + 0, + 0, + 0, + 7792, + 0, + 7798, + 0, + 0, + 0, + 0, + 0, + 7799, + 0, + 7810, + 0, + 0, + 7813, + 0, + 7814, + 0, + 7816, + 0, + 7818, + 7824, + 7825, + 7826, + 0, + 7828, + 7830, + 0, + 0, + 0, + 7840, + 0, + 7842, + 0, + 7843, + 0, + 0, + 0, + 0, + 7844, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7846, + 0, + 0, + 0, + 0, + 0, + 7856, + 7857, + 7858, + 7862, + 0, + 7865, + 0, + 0, + 7866, + 0, + 0, + 7913, + 0, + 0, + 0, + 0, + 7914, + 0, + 0, + 7915, + 7917, + 7918, + 7919, + 0, + 7920, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7921, + 7922, + 0, + 7924, + 0, + 0, + 7925, + 0, + 0, + 7927, + 0, + 7930, + 7935, + 0, + 0, + 7937, + 0, + 0, + 0, + 0, + 0, + 0, + 7939, + 0, + 7940, + 0, + 0, + 0, + 0, + 0, + 7941, + 0, + 0, + 0, + 0, + 7945, + 0, + 0, + 0, + 0, + 7949, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7950, + 0, + 7953, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7968, + 0, + 0, + 0, + 0, + 7969, + 7972, + 7992, + 0, + 7993, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 7994, + 0, + 0, + 0, + 0, + 8007, + 8008, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8010, + 0, + 0, + 0, + 8012, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8018, + 0, + 8028, + 8029, + 0, + 0, + 8030, + 0, + 0, + 8032, + 8033, + 0, + 0, + 8034, + 8036, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8037, + 0, + 0, + 0, + 8043, + 8052, + 8059, + 8060, + 0, + 0, + 8061, + 0, + 0, + 0, + 8062, + 0, + 8063, + 0, + 8064, + 0, + 8066, + 8068, + 0, + 0, + 0, + 8080, + 8081, + 0, + 8089, + 0, + 0, + 0, + 0, + 0, + 8092, + 0, + 0, + 0, + 0, + 0, + 0, + 8093, + 8110, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8111, + 0, + 0, + 0, + 0, + 0, + 8112, + 8115, + 0, + 8117, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8120, + 8121, + 8122, + 8128, + 8129, + 8130, + 8131, + 0, + 0, + 8139, + 0, + 0, + 8144, + 0, + 0, + 0, + 0, + 8145, + 8146, + 8153, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8154, + 0, + 8157, + 8160, + 8162, + 0, + 8164, + 8165, + 0, + 0, + 0, + 0, + 8166, + 8167, + 0, + 0, + 8179, + 0, + 0, + 0, + 8185, + 0, + 0, + 0, + 8186, + 0, + 0, + 8187, + 0, + 0, + 0, + 8188, + 0, + 0, + 0, + 0, + 0, + 8204, + 0, + 0, + 0, + 0, + 8210, + 0, + 0, + 0, + 0, + 0, + 8213, + 0, + 8214, + 0, + 0, + 8215, + 0, + 0, + 0, + 0, + 0, + 0, + 8218, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8219, + 0, + 8221, + 0, + 0, + 8222, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8225, + 0, + 0, + 0, + 8233, + 0, + 0, + 8242, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8247, + 0, + 8248, + 8252, + 0, + 8256, + 8257, + 0, + 0, + 8261, + 0, + 8264, + 8265, + 0, + 0, + 0, + 0, + 8267, + 0, + 0, + 0, + 8269, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8270, + 0, + 0, + 0, + 8278, + 0, + 8279, + 8283, + 0, + 0, + 8285, + 8286, + 8289, + 8292, + 0, + 0, + 0, + 0, + 8293, + 8295, + 8299, + 8300, + 8301, + 0, + 0, + 0, + 0, + 0, + 0, + 8304, + 8307, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8321, + 0, + 0, + 0, + 8322, + 8323, + 8325, + 8326, + 8327, + 0, + 0, + 8332, + 8338, + 0, + 0, + 8340, + 0, + 0, + 0, + 0, + 0, + 8350, + 0, + 0, + 8351, + 0, + 8354, + 8355, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8360, + 8372, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8377, + 0, + 0, + 0, + 0, + 8380, + 0, + 0, + 0, + 8383, + 0, + 8384, + 0, + 0, + 0, + 0, + 8386, + 8392, + 0, + 0, + 8394, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8396, + 8397, + 0, + 8398, + 0, + 8399, + 0, + 0, + 0, + 0, + 0, + 8400, + 0, + 8401, + 8410, + 8411, + 0, + 8412, + 8413, + 8422, + 0, + 0, + 0, + 0, + 8423, + 0, + 0, + 0, + 0, + 8424, + 0, + 0, + 8425, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8441, + 8442, + 0, + 0, + 0, + 0, + 0, + 0, + 8443, + 0, + 0, + 8444, + 0, + 8447, + 0, + 0, + 0, + 0, + 8451, + 0, + 8458, + 0, + 8462, + 0, + 0, + 8468, + 0, + 8469, + 0, + 0, + 0, + 8470, + 0, + 8473, + 8479, + 8480, + 0, + 0, + 0, + 0, + 8481, + 8483, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8484, + 0, + 0, + 8490, + 0, + 0, + 0, + 0, + 0, + 0, + 8491, + 8493, + 8494, + 0, + 8528, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8530, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8534, + 8538, + 8540, + 0, + 0, + 8541, + 0, + 0, + 8545, + 0, + 8557, + 0, + 0, + 8569, + 8570, + 0, + 0, + 8571, + 8574, + 8575, + 8579, + 0, + 8583, + 0, + 0, + 0, + 0, + 8591, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8606, + 0, + 8607, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8608, + 0, + 0, + 8609, + 0, + 0, + 0, + 8610, + 0, + 0, + 0, + 8611, + 0, + 0, + 8613, + 8617, + 8621, + 0, + 0, + 8622, + 0, + 8623, + 0, + 8624, + 8625, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8637, + 8638, + 8639, + 8650, + 0, + 0, + 0, + 0, + 8652, + 8654, + 8655, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8656, + 0, + 0, + 0, + 0, + 0, + 8657, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8658, + 0, + 0, + 8659, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8660, + 0, + 0, + 0, + 0, + 0, + 0, + 8661, + 8663, + 8664, + 0, + 0, + 0, + 0, + 8665, + 0, + 8669, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8671, + 8674, + 0, + 8684, + 0, + 8686, + 0, + 0, + 0, + 8689, + 0, + 0, + 0, + 8690, + 0, + 8706, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8710, + 0, + 8711, + 8713, + 8714, + 8724, + 8727, + 8728, + 8733, + 8736, + 0, + 8737, + 8739, + 0, + 0, + 0, + 0, + 8742, + 8743, + 8745, + 8754, + 0, + 0, + 0, + 0, + 8756, + 0, + 0, + 0, + 0, + 0, + 0, + 8757, + 8760, + 0, + 0, + 0, + 0, + 0, + 8762, + 8763, + 8764, + 0, + 8766, + 8769, + 8770, + 8773, + 0, + 8774, + 0, + 8779, + 0, + 0, + 0, + 0, + 8780, + 0, + 0, + 8781, + 0, + 0, + 8783, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8784, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8785, + 0, + 0, + 0, + 0, + 8786, + 0, + 0, + 0, + 0, + 8788, + 8790, + 0, + 0, + 0, + 8803, + 0, + 8813, + 8814, + 0, + 0, + 0, + 0, + 0, + 8815, + 8816, + 0, + 0, + 0, + 0, + 8818, + 0, + 0, + 0, + 0, + 8822, + 8828, + 8829, + 0, + 8831, + 0, + 0, + 0, + 0, + 8833, + 0, + 0, + 0, + 8834, + 0, + 0, + 0, + 8835, + 0, + 8836, + 0, + 0, + 0, + 8837, + 0, + 0, + 0, + 0, + 0, + 0, + 8838, + 8839, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8840, + 0, + 0, + 0, + 8841, + 0, + 8842, + 0, + 0, + 0, + 8846, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8847, + 0, + 8848, + 0, + 0, + 8864, + 0, + 0, + 8866, + 0, + 0, + 8870, + 8872, + 0, + 0, + 8873, + 8874, + 0, + 0, + 0, + 0, + 0, + 0, + 8875, + 0, + 8876, + 0, + 0, + 0, + 0, + 8896, + 8900, + 0, + 0, + 0, + 0, + 8901, + 0, + 0, + 0, + 0, + 0, + 8904, + 0, + 8907, + 0, + 0, + 0, + 0, + 8911, + 8912, + 8913, + 0, + 0, + 0, + 8914, + 0, + 8915, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8916, + 0, + 0, + 0, + 8929, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8930, + 0, + 8932, + 0, + 8943, + 0, + 0, + 0, + 8945, + 8947, + 0, + 0, + 0, + 0, + 8949, + 0, + 8950, + 0, + 8954, + 8957, + 0, + 0, + 8970, + 0, + 0, + 0, + 0, + 8971, + 0, + 8996, + 0, + 0, + 0, + 0, + 8997, + 9000, + 0, + 0, + 0, + 0, + 9001, + 9002, + 0, + 9004, + 9009, + 9024, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9027, + 9082, + 0, + 0, + 9083, + 9089, + 0, + 0, + 0, + 0, + 0, + 0, + 9090, + 0, + 0, + 0, + 9092, + 0, + 0, + 9093, + 0, + 9095, + 0, + 0, + 9096, + 9097, + 9101, + 9102, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9112, + 0, + 0, + 0, + 0, + 0, + 0, + 9114, + 0, + 0, + 9120, + 0, + 9121, + 9122, + 0, + 0, + 0, + 9123, + 9124, + 0, + 0, + 9125, + 0, + 0, + 9126, + 0, + 9127, + 0, + 0, + 9129, + 9131, + 0, + 0, + 0, + 9132, + 0, + 0, + 9136, + 0, + 9144, + 0, + 0, + 9148, + 0, + 0, + 0, + 0, + 0, + 0, + 9149, + 0, + 9152, + 9163, + 0, + 0, + 9165, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9166, + 0, + 9169, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9170, + 0, + 0, + 0, + 0, + 9172, + 0, + 9174, + 9175, + 9176, + 0, + 9177, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9186, + 0, + 9187, + 0, + 0, + 0, + 9188, + 9189, + 0, + 0, + 9190, + 0, + 0, + 0, + 0, + 9191, + 0, + 0, + 0, + 9193, + 0, + 0, + 0, + 0, + 9197, + 9198, + 0, + 0, + 0, + 9208, + 9211, + 0, + 0, + 0, + 0, + 9216, + 9217, + 0, + 9220, + 0, + 0, + 0, + 0, + 9221, + 9222, + 9223, + 0, + 9224, + 9225, + 0, + 0, + 9227, + 0, + 9228, + 9229, + 0, + 0, + 9230, + 0, + 9232, + 0, + 9233, + 0, + 0, + 0, + 0, + 0, + 9234, + 9235, + 0, + 0, + 9237, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9238, + 9240, + 0, + 0, + 9241, + 0, + 0, + 0, + 0, + 9244, + 0, + 0, + 0, + 0, + 9247, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9248, + 0, + 0, + 0, + 9249, + 0, + 0, + 0, + 0, + 0, + 9250, + 0, + 0, + 0, + 0, + 9251, + 0, + 0, + 9252, + 9255, + 0, + 0, + 0, + 9256, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9257, + 0, + 0, + 9258, + 0, + 0, + 0, + 0, + 0, + 0, + 9259, + 0, + 0, + 0, + 0, + 0, + 9262, + 9263, + 0, + 0, + 9265, + 9266, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9268, + 9271, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9273, + 0, + 0, + 0, + 9276, + 9277, + 9279, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9280, + 0, + 0, + 9293, + 0, + 0, + 0, + 0, + 0, + 9297, + 9301, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9308, + 9309, + 9313, + 9321, + 9322, + 0, + 9326, + 9327, + 0, + 0, + 9477, + 0, + 9479, + 0, + 0, + 0, + 0, + 9482, + 0, + 0, + 0, + 9483, + 0, + 9484, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9485, + 0, + 0, + 9486, + 0, + 0, + 0, + 9489, + 0, + 0, + 0, + 0, + 9490, + 9491, + 0, + 0, + 0, + 0, + 9493, + 0, + 9495, + 9496, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9500, + 0, + 9502, + 0, + 0, + 0, + 0, + 0, + 9504, + 9507, + 0, + 9509, + 0, + 9511, + 0, + 0, + 9513, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9515, + 0, + 0, + 0, + 0, + 0, + 0, + 9516, + 9517, + 0, + 0, + 0, + 0, + 9532, + 0, + 0, + 9533, + 0, + 0, + 9538, + 0, + 9539, + 9540, + 0, + 0, + 0, + 0, + 9541, + 0, + 0, + 0, + 9542, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9544, + 9545, + 0, + 9546, + 0, + 0, + 0, + 0, + 0, + 0, + 9547, + 9548, + 0, + 0, + 0, + 9550, + 0, + 9557, + 0, + 9558, + 0, + 9561, + 0, + 9563, + 9570, + 0, + 9572, + 9574, + 9575, + 0, + 0, + 0, + 9577, + 9592, + 0, + 0, + 9596, + 0, + 0, + 0, + 9598, + 0, + 9600, + 0, + 9601, + 0, + 0, + 0, + 0, + 0, + 0, + 9608, + 0, + 9638, + 9639, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9641, + 0, + 0, + 9643, + 9644, + 9645, + 9646, + 0, + 0, + 0, + 9648, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9650, + 9654, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9655, + 0, + 0, + 0, + 0, + 0, + 9656, + 0, + 9657, + 0, + 0, + 0, + 0, + 9658, + 0, + 0, + 9659, + 0, + 0, + 9664, + 0, + 0, + 9665, + 0, + 9667, + 9669, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9671, + 0, + 9673, + 9681, + 0, + 0, + 0, + 0, + 9682, + 9683, + 9684, + 0, + 0, + 0, + 0, + 9686, + 9698, + 0, + 0, + 9700, + 9701, + 9702, + 0, + 9703, + 9717, + 0, + 0, + 0, + 0, + 9718, + 0, + 9726, + 0, + 0, + 0, + 0, + 9727, + 0, + 0, + 0, + 9728, + 0, + 9742, + 0, + 9744, + 0, + 0, + 0, + 9750, + 0, + 9754, + 9755, + 0, + 0, + 0, + 0, + 0, + 9756, + 0, + 9757, + 9768, + 0, + 9769, + 0, + 0, + 0, + 9770, + 9771, + 0, + 9773, + 0, + 9774, + 0, + 9775, + 0, + 0, + 0, + 9776, + 9777, + 9784, + 0, + 0, + 0, + 9786, + 0, + 9789, + 0, + 0, + 0, + 0, + 9793, + 9794, + 0, + 0, + 0, + 9808, + 0, + 0, + 0, + 0, + 0, + 9811, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9812, + 0, + 9820, + 0, + 9823, + 0, + 9828, + 0, + 0, + 0, + 0, + 9830, + 0, + 0, + 9833, + 9836, + 0, + 0, + 0, + 9840, + 0, + 0, + 0, + 9841, + 0, + 0, + 9842, + 0, + 9845, + 0, + 0, + 0, + 9847, + 9848, + 0, + 0, + 9855, + 0, + 0, + 0, + 0, + 0, + 0, + 9856, + 9863, + 9865, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9866, + 9867, + 9868, + 9873, + 9875, + 0, + 0, + 0, + 0, + 0, + 0, + 9880, + 0, + 9886, + 0, + 0, + 0, + 9887, + 0, + 0, + 9891, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9906, + 9907, + 9908, + 0, + 0, + 0, + 9909, + 0, + 0, + 0, + 0, + 0, + 0, + 9910, + 0, + 0, + 0, + 0, + 9913, + 0, + 0, + 0, + 0, + 9914, + 0, + 0, + 0, + 0, + 0, + 9922, + 0, + 0, + 0, + 0, + 9923, + 9925, + 0, + 0, + 0, + 0, + 0, + 0, + 9930, + 0, + 0, + 0, + 9931, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9932, + 0, + 9939, + 0, + 0, + 9940, + 9962, + 9966, + 0, + 9969, + 9970, + 0, + 0, + 9974, + 0, + 9979, + 9981, + 9982, + 0, + 0, + 0, + 9985, + 0, + 0, + 0, + 0, + 0, + 0, + 9987, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 9988, + 9993, + 0, + 0, + 9994, + 0, + 0, + 0, + 9997, + 0, + 10004, + 0, + 0, + 0, + 0, + 0, + 10007, + 10019, + 10020, + 10022, + 0, + 0, + 0, + 10031, + 0, + 0, + 0, + 0, + 0, + 10032, + 0, + 0, + 10034, + 0, + 10036, + 0, + 0, + 0, + 0, + 10038, + 0, + 10039, + 10040, + 10041, + 10042, + 0, + 0, + 0, + 0, + 0, + 10043, + 0, + 0, + 0, + 0, + 0, + 10045, + 10054, + 0, + 0, + 0, + 0, + 10055, + 0, + 0, + 10057, + 10058, + 0, + 0, + 0, + 0, + 0, + 0, + 10059, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10060, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10063, + 0, + 10066, + 0, + 0, + 0, + 10070, + 0, + 10072, + 0, + 0, + 10076, + 10077, + 0, + 0, + 10084, + 0, + 10087, + 10090, + 10091, + 0, + 0, + 0, + 10094, + 10097, + 0, + 0, + 0, + 0, + 0, + 0, + 10098, + 0, + 0, + 0, + 0, + 0, + 0, + 10103, + 0, + 10104, + 0, + 10108, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10120, + 0, + 0, + 0, + 10122, + 0, + 0, + 10125, + 0, + 0, + 0, + 0, + 10127, + 10128, + 0, + 0, + 10134, + 0, + 10135, + 10136, + 0, + 10137, + 0, + 0, + 10147, + 0, + 10149, + 10150, + 0, + 0, + 10156, + 0, + 10158, + 10159, + 10160, + 10168, + 0, + 0, + 10171, + 0, + 10173, + 0, + 0, + 0, + 10176, + 0, + 0, + 0, + 0, + 10177, + 0, + 0, + 0, + 0, + 10178, + 0, + 0, + 0, + 0, + 10194, + 0, + 10202, + 0, + 0, + 10203, + 10204, + 0, + 10205, + 10206, + 0, + 10207, + 0, + 0, + 0, + 0, + 10209, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10213, + 0, + 0, + 0, + 0, + 0, + 0, + 10217, + 0, + 10229, + 0, + 10230, + 10231, + 0, + 0, + 10232, + 0, + 0, + 10237, + 10238, + 10244, + 0, + 0, + 0, + 0, + 0, + 10250, + 0, + 10252, + 0, + 0, + 0, + 0, + 0, + 0, + 10255, + 0, + 0, + 10257, + 0, + 0, + 0, + 0, + 0, + 0, + 10258, + 0, + 10259, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10260, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10284, + 10288, + 10289, + 0, + 0, + 0, + 10290, + 0, + 10296, + 0, + 0, + 0, + 0, + 0, + 10297, + 0, + 0, + 0, + 0, + 0, + 0, + 10298, + 0, + 0, + 0, + 0, + 10299, + 10303, + 0, + 0, + 0, + 0, + 0, + 10306, + 0, + 0, + 0, + 10307, + 0, + 10308, + 0, + 0, + 0, + 0, + 10311, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10315, + 10317, + 0, + 0, + 0, + 10318, + 10319, + 0, + 10321, + 0, + 10326, + 0, + 10328, + 0, + 0, + 0, + 0, + 10329, + 0, + 0, + 10331, + 0, + 10332, + 0, + 0, + 0, + 0, + 0, + 0, + 10334, + 0, + 0, + 10335, + 10338, + 0, + 0, + 0, + 0, + 0, + 10339, + 10349, + 0, + 0, + 0, + 0, + 0, + 0, + 10351, + 0, + 10353, + 0, + 0, + 0, + 0, + 0, + 0, + 10362, + 0, + 10368, + 0, + 10369, + 0, + 0, + 0, + 10372, + 10373, + 0, + 0, + 0, + 0, + 0, + 10374, + 0, + 0, + 0, + 10375, + 0, + 10376, + 0, + 0, + 10386, + 10388, + 10390, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10391, + 0, + 0, + 10392, + 10394, + 0, + 0, + 10396, + 0, + 10397, + 0, + 10403, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10404, + 0, + 10405, + 10410, + 0, + 0, + 10411, + 0, + 10412, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10421, + 10422, + 10423, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10425, + 0, + 0, + 10427, + 0, + 0, + 10430, + 0, + 0, + 0, + 0, + 0, + 10432, + 0, + 10433, + 10434, + 0, + 0, + 0, + 0, + 10436, + 10437, + 0, + 10438, + 0, + 10439, + 0, + 10444, + 10446, + 0, + 0, + 0, + 0, + 0, + 10448, + 0, + 0, + 0, + 0, + 0, + 10449, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10451, + 0, + 10453, + 0, + 0, + 0, + 10454, + 10457, + 0, + 0, + 10459, + 0, + 10469, + 0, + 0, + 0, + 0, + 0, + 10472, + 10481, + 0, + 0, + 0, + 0, + 0, + 10482, + 10483, + 0, + 10492, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10499, + 0, + 0, + 0, + 10502, + 0, + 0, + 10510, + 0, + 10521, + 10524, + 0, + 0, + 10525, + 10526, + 10528, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10530, + 0, + 0, + 0, + 0, + 10533, + 0, + 10534, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10535, + 10536, + 0, + 0, + 10544, + 0, + 10553, + 10556, + 0, + 10557, + 10559, + 0, + 0, + 0, + 0, + 0, + 10562, + 10563, + 10564, + 0, + 10565, + 0, + 0, + 0, + 10566, + 0, + 10567, + 0, + 0, + 0, + 0, + 10575, + 0, + 0, + 10576, + 0, + 10578, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10585, + 10586, + 10587, + 10589, + 0, + 10590, + 0, + 0, + 10594, + 0, + 0, + 0, + 0, + 0, + 10598, + 0, + 0, + 10601, + 0, + 0, + 0, + 10602, + 0, + 10603, + 0, + 10604, + 0, + 10605, + 0, + 0, + 10607, + 0, + 10626, + 0, + 10627, + 0, + 0, + 0, + 0, + 0, + 10629, + 10630, + 10631, + 0, + 0, + 0, + 10646, + 0, + 0, + 0, + 10647, + 0, + 10650, + 0, + 10651, + 0, + 0, + 0, + 10652, + 10653, + 10655, + 0, + 10658, + 0, + 0, + 10659, + 0, + 10667, + 0, + 0, + 0, + 0, + 10669, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10670, + 0, + 0, + 0, + 10671, + 0, + 0, + 0, + 0, + 10672, + 10673, + 0, + 10674, + 0, + 0, + 0, + 10676, + 0, + 0, + 0, + 0, + 0, + 0, + 10678, + 0, + 10682, + 0, + 0, + 10692, + 0, + 10697, + 0, + 0, + 0, + 0, + 10698, + 0, + 0, + 0, + 10700, + 0, + 0, + 0, + 0, + 0, + 10703, + 0, + 10704, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10705, + 0, + 10715, + 10718, + 10720, + 0, + 0, + 10722, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10723, + 0, + 0, + 0, + 0, + 10726, + 0, + 0, + 0, + 0, + 0, + 10727, + 10730, + 10743, + 0, + 0, + 0, + 0, + 0, + 0, + 10744, + 0, + 0, + 10745, + 0, + 0, + 0, + 0, + 0, + 0, + 10748, + 0, + 0, + 0, + 0, + 10750, + 0, + 0, + 10752, + 10753, + 0, + 0, + 0, + 10756, + 0, + 0, + 0, + 0, + 0, + 0, + 10758, + 0, + 0, + 0, + 10759, + 0, + 10769, + 0, + 0, + 10772, + 0, + 0, + 0, + 0, + 0, + 0, + 10773, + 0, + 0, + 0, + 10777, + 0, + 0, + 10779, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10780, + 10784, + 0, + 0, + 0, + 10789, + 0, + 0, + 0, + 10791, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10795, + 0, + 0, + 10796, + 0, + 10808, + 0, + 10809, + 0, + 0, + 0, + 10810, + 0, + 0, + 0, + 10812, + 0, + 0, + 10814, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10815, + 0, + 0, + 0, + 0, + 10816, + 10817, + 0, + 0, + 0, + 0, + 10819, + 0, + 10820, + 0, + 0, + 0, + 0, + 10821, + 10822, + 10823, + 0, + 10826, + 10849, + 0, + 0, + 0, + 0, + 10850, + 0, + 0, + 10852, + 0, + 10853, + 0, + 0, + 10856, + 0, + 0, + 10857, + 10858, + 10859, + 10860, + 0, + 0, + 0, + 0, + 0, + 0, + 10863, + 0, + 10866, + 10867, + 10872, + 10890, + 0, + 0, + 10891, + 10892, + 0, + 0, + 0, + 0, + 0, + 10893, + 0, + 0, + 0, + 10896, + 10899, + 0, + 0, + 10900, + 10902, + 0, + 0, + 0, + 0, + 0, + 10903, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10905, + 0, + 10906, + 0, + 0, + 0, + 0, + 10908, + 10911, + 0, + 10912, + 0, + 0, + 10916, + 0, + 0, + 0, + 0, + 0, + 10917, + 0, + 10918, + 0, + 0, + 0, + 10923, + 0, + 0, + 0, + 0, + 0, + 10924, + 0, + 0, + 10928, + 10929, + 0, + 0, + 10930, + 0, + 0, + 0, + 10932, + 0, + 0, + 0, + 0, + 10939, + 0, + 0, + 10945, + 0, + 0, + 0, + 10947, + 0, + 0, + 10948, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10958, + 0, + 10960, + 10962, + 0, + 0, + 10964, + 0, + 0, + 0, + 10966, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10967, + 0, + 0, + 0, + 10968, + 0, + 0, + 0, + 10973, + 0, + 0, + 0, + 0, + 0, + 10975, + 0, + 0, + 0, + 10976, + 10978, + 0, + 0, + 10982, + 10984, + 10987, + 0, + 0, + 10988, + 0, + 10989, + 0, + 0, + 10991, + 0, + 0, + 0, + 0, + 10992, + 0, + 0, + 0, + 10993, + 0, + 10995, + 0, + 0, + 0, + 10996, + 10997, + 0, + 0, + 0, + 10998, + 0, + 10999, + 0, + 11001, + 0, + 0, + 0, + 0, + 0, + 0, + 11010, + 11012, + 0, + 11013, + 11016, + 11017, + 0, + 0, + 11019, + 11020, + 11021, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11022, + 0, + 0, + 11023, + 11029, + 0, + 0, + 0, + 0, + 11031, + 0, + 0, + 0, + 11034, + 0, + 0, + 0, + 0, + 11055, + 0, + 0, + 0, + 0, + 0, + 11056, + 11060, + 0, + 0, + 0, + 0, + 0, + 0, + 11061, + 0, + 0, + 11064, + 11065, + 0, + 11066, + 0, + 11069, + 0, + 11085, + 0, + 0, + 0, + 0, + 0, + 11086, + 0, + 0, + 0, + 11088, + 0, + 0, + 0, + 11094, + 0, + 0, + 0, + 11095, + 11096, + 0, + 0, + 0, + 0, + 0, + 0, + 11097, + 11098, + 0, + 0, + 0, + 0, + 0, + 0, + 11099, + 0, + 0, + 11102, + 11108, + 0, + 0, + 0, + 11109, + 0, + 11114, + 11119, + 0, + 11131, + 0, + 0, + 0, + 11142, + 0, + 0, + 11143, + 0, + 11146, + 0, + 11147, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11148, + 0, + 11149, + 11152, + 11153, + 11154, + 0, + 11156, + 0, + 11157, + 0, + 0, + 0, + 11158, + 0, + 0, + 11159, + 11160, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11163, + 0, + 0, + 11164, + 11166, + 0, + 0, + 0, + 11172, + 11174, + 0, + 0, + 0, + 11176, + 0, + 0, + 0, + 0, + 0, + 11182, + 11183, + 0, + 0, + 0, + 11184, + 11187, + 0, + 0, + 11188, + 11189, + 0, + 0, + 0, + 0, + 0, + 0, + 11194, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11200, + 11202, + 0, + 0, + 0, + 0, + 0, + 0, + 11203, + 0, + 11204, + 0, + 0, + 0, + 0, + 0, + 11205, + 0, + 0, + 0, + 11206, + 0, + 11207, + 0, + 0, + 11209, + 0, + 11211, + 0, + 11214, + 0, + 0, + 11231, + 0, + 0, + 0, + 11293, + 11295, + 0, + 0, + 11296, + 11297, + 11302, + 0, + 0, + 0, + 11307, + 0, + 0, + 0, + 0, + 11309, + 11310, + 0, + 11311, + 0, + 0, + 0, + 11313, + 0, + 11314, + 0, + 0, + 0, + 0, + 11334, + 0, + 11338, + 0, + 0, + 0, + 11339, + 0, + 0, + 0, + 0, + 0, + 11340, + 0, + 11341, + 11342, + 0, + 11344, + 0, + 11345, + 0, + 0, + 0, + 11348, + 11349, + 0, + 0, + 11350, + 0, + 0, + 0, + 11355, + 0, + 0, + 0, + 0, + 0, + 0, + 11356, + 0, + 11357, + 11370, + 0, + 0, + 11371, + 0, + 11374, + 11376, + 0, + 0, + 0, + 11377, + 0, + 0, + 11378, + 11383, + 0, + 11386, + 11399, + 0, + 11400, + 11406, + 0, + 0, + 0, + 11408, + 0, + 0, + 11409, + 11412, + 0, + 0, + 0, + 0, + 11417, + 0, + 0, + 0, + 11418, + 0, + 11421, + 0, + 11426, + 11429, + 0, + 0, + 0, + 0, + 0, + 11430, + 0, + 11437, + 0, + 11438, + 0, + 0, + 0, + 0, + 0, + 11440, + 11453, + 0, + 0, + 0, + 0, + 0, + 0, + 11454, + 0, + 0, + 0, + 0, + 11455, + 0, + 0, + 11456, + 11460, + 11461, + 11463, + 0, + 11469, + 0, + 11473, + 0, + 0, + 0, + 0, + 11474, + 0, + 0, + 0, + 11475, + 0, + 11476, + 11477, + 11480, + 0, + 0, + 0, + 0, + 11481, + 0, + 0, + 11484, + 0, + 0, + 11487, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11497, + 0, + 0, + 11502, + 0, + 11509, + 0, + 0, + 11510, + 11511, + 11513, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11515, + 0, + 0, + 0, + 0, + 11516, + 0, + 11520, + 11521, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11529, + 11530, + 11531, + 11534, + 0, + 0, + 11543, + 0, + 0, + 0, + 0, + 0, + 11547, + 0, + 11548, + 0, + 0, + 0, + 0, + 0, + 11552, + 11556, + 0, + 11557, + 0, + 0, + 11559, + 0, + 11560, + 0, + 0, + 0, + 0, + 0, + 0, + 11561, + 0, + 0, + 11563, + 11564, + 0, + 11565, + 0, + 0, + 0, + 0, + 11567, + 0, + 0, + 0, + 11569, + 0, + 11574, + 0, + 11575, + 0, + 0, + 0, + 11577, + 0, + 11578, + 0, + 0, + 0, + 11580, + 11581, + 0, + 0, + 0, + 11582, + 11584, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11587, + 0, + 11588, + 11591, + 0, + 11595, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11596, + 0, + 11597, + 0, + 0, + 0, + 0, + 11598, + 11601, + 0, + 0, + 0, + 11602, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11603, + 11604, + 0, + 11606, + 0, + 0, + 11608, + 0, + 0, + 0, + 0, + 11610, + 0, + 0, + 11611, + 0, + 0, + 0, + 0, + 11613, + 0, + 11622, + 0, + 0, + 0, + 11623, + 0, + 0, + 0, + 0, + 11625, + 0, + 0, + 11626, + 11627, + 11628, + 11630, + 0, + 0, + 0, + 0, + 0, + 0, + 11639, + 0, + 0, + 11646, + 0, + 11648, + 11649, + 0, + 11650, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11651, + 0, + 0, + 11652, + 11653, + 11656, + 0, + 0, + 11677, + 11679, + 0, + 0, + 0, + 0, + 11680, + 0, + 0, + 11681, + 0, + 11685, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11688, + 0, + 0, + 0, + 11716, + 0, + 11719, + 0, + 0, + 0, + 0, + 0, + 11721, + 0, + 0, + 11724, + 11743, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11745, + 11748, + 11750, + 0, + 0, + 0, + 0, + 0, + 11751, + 0, + 0, + 0, + 11752, + 11754, + 0, + 11755, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11759, + 0, + 0, + 0, + 0, + 0, + 0, + 11760, + 0, + 0, + 0, + 11761, + 0, + 0, + 0, + 0, + 0, + 0, + 11766, + 11767, + 0, + 11772, + 11773, + 0, + 11774, + 0, + 0, + 11775, + 0, + 11777, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11778, + 11780, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11783, + 0, + 11784, + 0, + 0, + 0, + 11785, + 0, + 0, + 0, + 11786, + 0, + 0, + 0, + 0, + 11788, + 0, + 0, + 11789, + 11791, + 11792, + 0, + 0, + 0, + 0, + 11795, + 11834, + 11835, + 11836, + 0, + 0, + 11837, + 0, + 0, + 0, + 11838, + 0, + 0, + 11846, + 11851, + 0, + 11852, + 0, + 11869, + 0, + 0, + 0, + 11871, + 0, + 0, + 0, + 11872, + 11874, + 0, + 0, + 0, + 0, + 0, + 0, + 11875, + 0, + 11876, + 11877, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11883, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11884, + 0, + 11885, + 0, + 11886, + 0, + 0, + 11887, + 0, + 11894, + 11895, + 11897, + 11909, + 11910, + 0, + 11912, + 11918, + 0, + 0, + 11920, + 0, + 11922, + 11924, + 11927, + 11928, + 0, + 0, + 0, + 0, + 11929, + 0, + 11934, + 0, + 0, + 0, + 0, + 0, + 11941, + 11943, + 11944, + 0, + 11945, + 0, + 0, + 0, + 0, + 11948, + 11949, + 0, + 0, + 0, + 0, + 11953, + 0, + 11954, + 0, + 11955, + 0, + 11956, + 0, + 0, + 0, + 0, + 0, + 11957, + 0, + 0, + 11959, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 11961, + 0, + 0, + 0, + 0, + 0, + 11978, + 0, + 0, + 0, + 11979, + 11980, + 11986, + 11987, + 0, + 11992, + 0, + 0, + 0, + 0, + 0, + 11993, + 0, + 0, + 0, + 11994, + 0, + 11999, + 12004, + 12005, + 12006, + 0, + 0, + 0, + 0, + 0, + 12011, + 0, + 0, + 12012, + 12014, + 0, + 0, + 12015, + 0, + 0, + 12019, + 12028, + 0, + 0, + 12029, + 0, + 0, + 12032, + 12033, + 0, + 0, + 0, + 0, + 12034, + 0, + 12041, + 12043, + 0, + 0, + 12044, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12046, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12054, + 12055, + 0, + 12056, + 0, + 0, + 0, + 12060, + 12064, + 0, + 0, + 0, + 0, + 0, + 12065, + 12067, + 12068, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12074, + 0, + 0, + 0, + 12075, + 12076, + 0, + 0, + 0, + 12079, + 0, + 12081, + 12086, + 12087, + 0, + 0, + 12088, + 0, + 0, + 0, + 0, + 12089, + 0, + 12092, + 0, + 0, + 0, + 0, + 12097, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12098, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12102, + 12103, + 12104, + 12111, + 0, + 0, + 12114, + 12116, + 0, + 0, + 0, + 12118, + 0, + 0, + 0, + 12119, + 12120, + 12128, + 0, + 0, + 0, + 0, + 12130, + 0, + 0, + 0, + 0, + 0, + 0, + 12131, + 0, + 0, + 0, + 12132, + 12134, + 0, + 0, + 0, + 0, + 12137, + 0, + 12139, + 0, + 12141, + 0, + 0, + 12142, + 0, + 0, + 0, + 12144, + 0, + 0, + 0, + 0, + 0, + 12145, + 0, + 12148, + 0, + 12153, + 0, + 0, + 0, + 0, + 12154, + 12171, + 12173, + 0, + 0, + 0, + 12175, + 0, + 0, + 0, + 0, + 12178, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12183, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12184, + 0, + 0, + 0, + 12186, + 0, + 0, + 0, + 0, + 0, + 12187, + 12188, + 0, + 0, + 12189, + 0, + 12196, + 0, + 12197, + 0, + 0, + 12198, + 0, + 12201, + 0, + 0, + 0, + 0, + 12203, + 0, + 12209, + 0, + 0, + 0, + 0, + 12210, + 12211, + 12212, + 12213, + 0, + 12217, + 12218, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12222, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12223, + 0, + 0, + 12229, + 0, + 0, + 0, + 0, + 12233, + 0, + 0, + 0, + 0, + 12234, + 0, + 0, + 12236, + 12242, + 0, + 0, + 0, + 12243, + 0, + 0, + 0, + 12244, + 12253, + 0, + 12254, + 12256, + 0, + 12257, + 0, + 0, + 12275, + 0, + 0, + 0, + 0, + 0, + 12277, + 0, + 0, + 0, + 0, + 0, + 12278, + 0, + 12289, + 0, + 0, + 12290, + 0, + 12292, + 12293, + 0, + 0, + 12294, + 0, + 12295, + 0, + 0, + 12296, + 0, + 12297, + 0, + 12298, + 0, + 0, + 0, + 0, + 12301, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12309, + 0, + 12338, + 12340, + 0, + 0, + 0, + 0, + 12341, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12342, + 12343, + 0, + 12344, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12345, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12346, + 0, + 0, + 0, + 0, + 12348, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12350, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12351, + 0, + 12355, + 12356, + 12357, + 0, + 0, + 12367, + 12370, + 12371, + 0, + 0, + 0, + 0, + 0, + 12372, + 12376, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12379, + 0, + 12382, + 0, + 12383, + 0, + 0, + 12384, + 0, + 0, + 0, + 0, + 12393, + 0, + 0, + 12394, + 0, + 0, + 0, + 0, + 12398, + 12403, + 0, + 0, + 12404, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12410, + 0, + 0, + 0, + 12411, + 0, + 0, + 0, + 12412, + 0, + 0, + 0, + 0, + 12420, + 0, + 12421, + 0, + 0, + 0, + 0, + 0, + 12423, + 0, + 12425, + 12429, + 0, + 0, + 0, + 12431, + 12432, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12434, + 0, + 0, + 0, + 0, + 0, + 12435, + 12436, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12437, + 0, + 0, + 0, + 0, + 0, + 12438, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12445, + 0, + 0, + 0, + 12450, + 12451, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12452, + 12475, + 0, + 0, + 12493, + 12494, + 0, + 0, + 0, + 12495, + 0, + 0, + 0, + 0, + 12496, + 12502, + 12509, + 0, + 0, + 0, + 0, + 12510, + 0, + 12512, + 12513, + 0, + 0, + 0, + 0, + 12514, + 0, + 0, + 0, + 12515, + 0, + 12520, + 0, + 0, + 0, + 12524, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12527, + 0, + 0, + 0, + 12528, + 0, + 0, + 0, + 12529, + 0, + 0, + 0, + 0, + 0, + 12530, + 0, + 12535, + 0, + 0, + 12536, + 0, + 12538, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12540, + 0, + 12548, + 0, + 0, + 0, + 0, + 0, + 12550, + 0, + 0, + 0, + 12551, + 12552, + 0, + 0, + 0, + 12554, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12555, + 0, + 0, + 12562, + 0, + 12565, + 0, + 12566, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12569, + 0, + 0, + 0, + 12571, + 12574, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12577, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12578, + 12579, + 12603, + 0, + 12608, + 0, + 0, + 12611, + 0, + 12612, + 0, + 12615, + 0, + 12625, + 0, + 0, + 0, + 0, + 12627, + 12646, + 0, + 12648, + 0, + 0, + 12657, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12670, + 0, + 0, + 12671, + 0, + 12673, + 12677, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12679, + 0, + 12681, + 0, + 12682, + 12693, + 0, + 12694, + 0, + 12697, + 0, + 12701, + 0, + 0, + 0, + 12703, + 12704, + 0, + 0, + 0, + 0, + 12707, + 12737, + 0, + 0, + 12739, + 0, + 0, + 12740, + 0, + 0, + 12742, + 12743, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12745, + 0, + 12746, + 12747, + 0, + 12748, + 0, + 0, + 12759, + 12767, + 0, + 0, + 0, + 0, + 12773, + 0, + 12774, + 12778, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12779, + 0, + 0, + 0, + 0, + 0, + 12780, + 12793, + 0, + 12824, + 0, + 12825, + 0, + 12836, + 0, + 0, + 0, + 0, + 12839, + 0, + 12842, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12843, + 12845, + 0, + 12846, + 0, + 0, + 0, + 0, + 12847, + 0, + 0, + 12850, + 12852, + 12853, + 0, + 0, + 0, + 12854, + 0, + 0, + 0, + 12855, + 0, + 12856, + 0, + 12858, + 0, + 0, + 12859, + 0, + 12862, + 0, + 12863, + 0, + 0, + 12866, + 0, + 12869, + 12872, + 12873, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12875, + 0, + 12877, + 0, + 0, + 12878, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12884, + 12885, + 12888, + 0, + 12889, + 0, + 0, + 0, + 0, + 12893, + 0, + 0, + 0, + 12895, + 12896, + 12898, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12902, + 0, + 12909, + 12910, + 0, + 12926, + 0, + 12928, + 0, + 0, + 0, + 12929, + 0, + 12930, + 0, + 0, + 0, + 0, + 12931, + 0, + 12932, + 12933, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12934, + 0, + 12942, + 0, + 0, + 0, + 0, + 12944, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12946, + 0, + 0, + 12948, + 0, + 0, + 12949, + 0, + 0, + 0, + 0, + 12950, + 0, + 0, + 0, + 0, + 12951, + 0, + 12952, + 0, + 12953, + 0, + 0, + 0, + 12954, + 12958, + 12959, + 0, + 0, + 0, + 0, + 0, + 12960, + 12964, + 0, + 0, + 0, + 0, + 0, + 12966, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 12970, + 0, + 12971, + 0, + 0, + 0, + 0, + 0, + 0, + 12972, + 0, + 0, + 12982, + 0, + 0, + 0, + 12984, + 12985, + 0, + 12986, + 12996, + 12997, + 13001, + 13002, + 0, + 0, + 0, + 0, + 13004, + 0, + 0, + 13005, + 0, + 0, + 13007, + 13009, + 0, + 13017, + 0, + 0, + 0, + 13020, + 0, + 13021, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13022, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13024, + 13027, + 0, + 0, + 0, + 0, + 0, + 13028, + 0, + 0, + 13029, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13032, + 0, + 13037, + 0, + 0, + 0, + 0, + 0, + 0, + 13040, + 0, + 0, + 13041, + 0, + 0, + 0, + 13043, + 13044, + 13046, + 0, + 0, + 0, + 0, + 13047, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13049, + 13054, + 0, + 13056, + 0, + 0, + 13060, + 13061, + 0, + 0, + 0, + 0, + 0, + 13067, + 0, + 0, + 13068, + 0, + 13071, + 0, + 0, + 0, + 0, + 0, + 13077, + 13078, + 0, + 0, + 0, + 0, + 0, + 13079, + 13080, + 13081, + 0, + 13082, + 0, + 0, + 0, + 13085, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13086, + 0, + 13087, + 13088, + 0, + 0, + 0, + 0, + 0, + 13094, + 0, + 13099, + 0, + 13100, + 0, + 0, + 0, + 13101, + 0, + 13125, + 13126, + 13128, + 13129, + 0, + 0, + 13130, + 0, + 13131, + 0, + 0, + 0, + 0, + 0, + 0, + 13134, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13150, + 0, + 13168, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13169, + 0, + 0, + 13170, + 0, + 0, + 0, + 0, + 13174, + 0, + 0, + 0, + 13176, + 0, + 0, + 0, + 0, + 0, + 13177, + 0, + 13178, + 13183, + 13187, + 0, + 0, + 0, + 13189, + 0, + 0, + 13190, + 0, + 0, + 13191, + 0, + 0, + 13206, + 0, + 0, + 0, + 13207, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13212, + 0, + 0, + 13219, + 13232, + 0, + 0, + 0, + 13241, + 0, + 13249, + 13253, + 0, + 0, + 0, + 0, + 0, + 13255, + 13259, + 0, + 13260, + 13261, + 0, + 13262, + 0, + 13272, + 0, + 0, + 0, + 0, + 13276, + 0, + 0, + 0, + 0, + 13277, + 13299, + 0, + 0, + 13301, + 13302, + 0, + 0, + 13303, + 0, + 0, + 13305, + 0, + 13310, + 0, + 0, + 0, + 13311, + 0, + 0, + 0, + 0, + 13325, + 0, + 13328, + 0, + 0, + 0, + 13329, + 0, + 0, + 0, + 0, + 0, + 0, + 13330, + 0, + 0, + 13331, + 0, + 13335, + 0, + 0, + 13342, + 0, + 0, + 0, + 0, + 0, + 13343, + 0, + 13354, + 0, + 13362, + 0, + 13366, + 13367, + 13369, + 0, + 0, + 13371, + 13372, + 0, + 13373, + 13374, + 0, + 13376, + 0, + 13380, + 13381, + 13386, + 0, + 13387, + 13388, + 0, + 13389, + 13391, + 13395, + 0, + 0, + 0, + 0, + 0, + 13401, + 13409, + 0, + 13410, + 0, + 0, + 0, + 0, + 13420, + 0, + 0, + 0, + 0, + 0, + 13422, + 0, + 0, + 0, + 0, + 13423, + 0, + 0, + 0, + 0, + 13425, + 0, + 0, + 0, + 0, + 0, + 13427, + 0, + 0, + 0, + 13428, + 0, + 0, + 13430, + 13438, + 0, + 13439, + 0, + 13445, + 0, + 13448, + 13449, + 0, + 0, + 0, + 0, + 0, + 0, + 13451, + 0, + 13457, + 0, + 0, + 0, + 0, + 13458, + 13459, + 0, + 13460, + 0, + 0, + 0, + 0, + 13464, + 13465, + 13466, + 13470, + 0, + 13471, + 13472, + 13474, + 13475, + 0, + 13476, + 0, + 0, + 13478, + 13479, + 0, + 13481, + 0, + 0, + 0, + 0, + 13487, + 0, + 13490, + 0, + 13493, + 0, + 0, + 13494, + 0, + 0, + 13495, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13496, + 13497, + 0, + 13500, + 0, + 0, + 13516, + 13522, + 0, + 0, + 13525, + 13528, + 0, + 0, + 0, + 13530, + 13535, + 0, + 13537, + 13539, + 0, + 13540, + 0, + 13543, + 0, + 13544, + 0, + 0, + 0, + 0, + 0, + 0, + 13545, + 0, + 0, + 0, + 0, + 0, + 0, + 13547, + 0, + 0, + 0, + 13549, + 13555, + 0, + 0, + 0, + 13556, + 13557, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13558, + 0, + 13563, + 0, + 0, + 0, + 0, + 13564, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13566, + 0, + 0, + 0, + 0, + 0, + 0, + 13569, + 0, + 0, + 13571, + 0, + 0, + 0, + 0, + 13573, + 0, + 0, + 0, + 0, + 0, + 0, + 13578, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13581, + 0, + 13586, + 0, + 13595, + 0, + 13600, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13601, + 13603, + 0, + 13604, + 13605, + 13606, + 13607, + 0, + 0, + 13617, + 13618, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13623, + 0, + 13625, + 13627, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13629, + 0, + 0, + 0, + 13634, + 0, + 0, + 0, + 13638, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13654, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13656, + 0, + 13659, + 0, + 0, + 13660, + 0, + 0, + 13662, + 0, + 0, + 0, + 13663, + 0, + 13664, + 0, + 0, + 0, + 0, + 0, + 13668, + 0, + 13669, + 13671, + 0, + 0, + 13672, + 0, + 0, + 0, + 0, + 0, + 0, + 13675, + 13685, + 0, + 13686, + 0, + 0, + 0, + 13687, + 0, + 0, + 0, + 13692, + 13694, + 13697, + 0, + 0, + 0, + 13702, + 0, + 0, + 0, + 0, + 0, + 13705, + 0, + 0, + 0, + 0, + 13707, + 0, + 0, + 0, + 13714, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13715, + 0, + 13716, + 13717, + 0, + 0, + 13719, + 13724, + 13730, + 13731, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13732, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13734, + 0, + 13736, + 0, + 0, + 13737, + 13738, + 13747, + 0, + 13751, + 0, + 0, + 13752, + 0, + 0, + 0, + 13753, + 0, + 13757, + 0, + 0, + 13762, + 13763, + 0, + 13764, + 13765, + 0, + 13766, + 0, + 0, + 13767, + 0, + 0, + 0, + 13768, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13769, + 0, + 0, + 13772, + 0, + 13775, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13776, + 13778, + 13787, + 0, + 0, + 0, + 13797, + 0, + 13798, + 0, + 13801, + 0, + 13804, + 13806, + 0, + 0, + 0, + 0, + 13816, + 13817, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13834, + 0, + 13836, + 0, + 0, + 13838, + 0, + 0, + 13839, + 0, + 13840, + 0, + 0, + 0, + 0, + 13842, + 0, + 0, + 0, + 0, + 0, + 0, + 13843, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13845, + 0, + 0, + 0, + 0, + 0, + 13858, + 0, + 0, + 13860, + 0, + 0, + 13861, + 0, + 0, + 13862, + 13863, + 0, + 13868, + 0, + 13869, + 13870, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13872, + 0, + 0, + 0, + 0, + 13873, + 13878, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13886, + 0, + 13888, + 13889, + 13890, + 0, + 0, + 13891, + 13894, + 0, + 13897, + 13899, + 13900, + 13904, + 0, + 0, + 13906, + 0, + 0, + 0, + 13909, + 0, + 0, + 0, + 13910, + 0, + 0, + 0, + 13911, + 0, + 0, + 0, + 0, + 0, + 13912, + 13917, + 0, + 0, + 0, + 0, + 13918, + 0, + 13919, + 0, + 0, + 13920, + 0, + 0, + 0, + 13921, + 0, + 0, + 13922, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13924, + 0, + 13927, + 0, + 0, + 0, + 0, + 0, + 13932, + 0, + 13933, + 0, + 13934, + 0, + 0, + 13935, + 0, + 13944, + 0, + 0, + 0, + 13954, + 0, + 0, + 13955, + 0, + 0, + 0, + 0, + 13956, + 0, + 13957, + 0, + 13967, + 13969, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 13970, + 13990, + 0, + 13991, + 13994, + 0, + 13995, + 0, + 0, + 0, + 0, + 13996, + 0, + 0, + 13999, + 0, + 0, + 0, + 14018, + 0, + 14019, + 0, + 14021, + 0, + 0, + 0, + 0, + 0, + 0, + 14041, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14043, + 0, + 0, + 0, + 0, + 14046, + 0, + 0, + 0, + 14048, + 14049, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14051, + 0, + 0, + 14052, + 14056, + 0, + 14063, + 0, + 14064, + 14066, + 0, + 0, + 14067, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14068, + 0, + 0, + 0, + 14072, + 0, + 14074, + 14075, + 0, + 14076, + 14079, + 14085, + 14086, + 14087, + 14093, + 0, + 0, + 0, + 0, + 14095, + 0, + 0, + 0, + 0, + 0, + 0, + 14096, + 14097, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14098, + 0, + 14102, + 0, + 0, + 0, + 0, + 0, + 14103, + 0, + 0, + 0, + 14104, + 0, + 0, + 14105, + 0, + 0, + 0, + 14107, + 14108, + 0, + 0, + 14109, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14117, + 0, + 0, + 0, + 0, + 14118, + 0, + 0, + 0, + 0, + 14119, + 0, + 0, + 14120, + 0, + 0, + 14121, + 0, + 14122, + 14127, + 0, + 14128, + 14136, + 0, + 0, + 14138, + 0, + 14140, + 0, + 0, + 0, + 14141, + 14142, + 0, + 0, + 0, + 0, + 14146, + 0, + 0, + 14149, + 0, + 14151, + 0, + 0, + 0, + 14152, + 0, + 0, + 14153, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14154, + 0, + 14156, + 14157, + 0, + 0, + 14159, + 0, + 14161, + 0, + 0, + 0, + 0, + 14162, + 0, + 0, + 0, + 0, + 0, + 0, + 14163, + 0, + 0, + 14173, + 0, + 0, + 0, + 0, + 0, + 0, + 14174, + 0, + 0, + 14176, + 0, + 0, + 14178, + 0, + 0, + 14179, + 14181, + 0, + 0, + 14182, + 14185, + 14187, + 0, + 14190, + 0, + 0, + 14197, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14198, + 0, + 0, + 0, + 0, + 0, + 0, + 14199, + 14200, + 0, + 0, + 0, + 14204, + 0, + 0, + 14208, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14231, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14234, + 0, + 0, + 14235, + 0, + 0, + 0, + 14240, + 14241, + 0, + 0, + 0, + 14246, + 0, + 0, + 0, + 14247, + 0, + 14250, + 0, + 0, + 14251, + 0, + 0, + 14254, + 0, + 0, + 14256, + 0, + 0, + 0, + 14260, + 0, + 14261, + 0, + 0, + 0, + 0, + 14262, + 14267, + 14269, + 0, + 0, + 14277, + 0, + 0, + 14278, + 0, + 14279, + 14282, + 0, + 0, + 0, + 14283, + 0, + 0, + 0, + 14284, + 14285, + 0, + 0, + 0, + 0, + 14286, + 0, + 0, + 0, + 14288, + 0, + 0, + 0, + 14289, + 0, + 14290, + 0, + 14293, + 14301, + 14302, + 14304, + 14305, + 0, + 14307, + 0, + 14308, + 14309, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14311, + 14312, + 0, + 0, + 14317, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14318, + 0, + 0, + 0, + 0, + 14320, + 0, + 0, + 0, + 0, + 14321, + 14322, + 0, + 0, + 0, + 0, + 0, + 14326, + 14329, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14330, + 14331, + 0, + 0, + 0, + 0, + 14332, + 0, + 0, + 0, + 14333, + 0, + 0, + 14337, + 14340, + 0, + 14341, + 0, + 0, + 14342, + 0, + 14345, + 14346, + 0, + 0, + 14347, + 0, + 14362, + 0, + 0, + 0, + 0, + 0, + 14364, + 14365, + 14371, + 0, + 14373, + 0, + 0, + 14374, + 0, + 14379, + 0, + 14400, + 0, + 0, + 0, + 0, + 0, + 14401, + 0, + 0, + 14405, + 0, + 14406, + 0, + 14408, + 14409, + 0, + 0, + 0, + 14417, + 0, + 0, + 14424, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14430, + 0, + 0, + 0, + 14431, + 0, + 0, + 14435, + 0, + 14440, + 0, + 0, + 0, + 0, + 0, + 0, + 14442, + 0, + 0, + 14443, + 0, + 0, + 0, + 0, + 0, + 14446, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14454, + 0, + 14457, + 0, + 14460, + 0, + 0, + 14466, + 0, + 0, + 0, + 0, + 0, + 14467, + 0, + 0, + 0, + 0, + 0, + 0, + 14469, + 0, + 14477, + 0, + 0, + 0, + 0, + 0, + 0, + 14478, + 14482, + 0, + 0, + 0, + 14483, + 0, + 0, + 0, + 14485, + 14486, + 0, + 0, + 0, + 14487, + 14488, + 14489, + 14492, + 14493, + 14494, + 14495, + 14496, + 14497, + 0, + 14499, + 0, + 14501, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14502, + 0, + 14507, + 14512, + 14513, + 14514, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14515, + 14526, + 14530, + 0, + 14537, + 0, + 14544, + 0, + 14547, + 0, + 0, + 14548, + 14550, + 14551, + 0, + 0, + 14552, + 0, + 0, + 0, + 14553, + 0, + 14554, + 0, + 0, + 0, + 0, + 14556, + 14564, + 0, + 0, + 14565, + 14566, + 0, + 0, + 0, + 0, + 0, + 0, + 14568, + 0, + 0, + 14569, + 0, + 0, + 0, + 14571, + 14576, + 0, + 0, + 14577, + 14578, + 14579, + 0, + 0, + 14580, + 0, + 0, + 0, + 0, + 14582, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14583, + 0, + 0, + 0, + 0, + 0, + 14587, + 0, + 14588, + 0, + 0, + 14600, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14601, + 0, + 0, + 14604, + 14605, + 14611, + 0, + 14613, + 0, + 0, + 0, + 0, + 14615, + 0, + 0, + 0, + 0, + 0, + 0, + 14627, + 0, + 14628, + 0, + 0, + 0, + 0, + 14631, + 0, + 14633, + 14634, + 0, + 0, + 0, + 0, + 14635, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14636, + 0, + 0, + 14639, + 14642, + 0, + 0, + 0, + 0, + 14644, + 0, + 0, + 0, + 0, + 14645, + 14646, + 0, + 14653, + 0, + 0, + 14654, + 0, + 14658, + 0, + 14661, + 0, + 0, + 0, + 14665, + 0, + 0, + 0, + 14668, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14669, + 0, + 0, + 14670, + 0, + 0, + 0, + 14680, + 0, + 0, + 14681, + 0, + 0, + 0, + 0, + 0, + 14682, + 14683, + 0, + 0, + 0, + 0, + 14686, + 0, + 0, + 0, + 0, + 14687, + 14697, + 0, + 0, + 0, + 0, + 14699, + 14705, + 14711, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14712, + 0, + 0, + 0, + 14713, + 0, + 0, + 0, + 0, + 14719, + 0, + 14720, + 14721, + 14726, + 0, + 0, + 0, + 14728, + 14729, + 0, + 0, + 0, + 0, + 14731, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14733, + 14736, + 14737, + 0, + 0, + 14740, + 14742, + 0, + 0, + 0, + 14744, + 14753, + 0, + 0, + 0, + 0, + 14755, + 14758, + 14760, + 0, + 0, + 0, + 0, + 0, + 14761, + 14762, + 14765, + 14771, + 0, + 14772, + 0, + 14773, + 14774, + 0, + 0, + 14775, + 0, + 0, + 14776, + 0, + 0, + 0, + 0, + 14777, + 0, + 14779, + 0, + 0, + 14782, + 0, + 0, + 14785, + 14786, + 14788, + 0, + 0, + 0, + 0, + 0, + 14795, + 0, + 0, + 0, + 0, + 0, + 0, + 14798, + 0, + 14803, + 14804, + 14806, + 0, + 0, + 0, + 14809, + 0, + 0, + 0, + 0, + 0, + 0, + 14810, + 0, + 0, + 0, + 0, + 14811, + 0, + 14812, + 0, + 0, + 0, + 0, + 0, + 14815, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14816, + 0, + 14818, + 0, + 0, + 0, + 0, + 0, + 0, + 14819, + 0, + 14820, + 0, + 14823, + 0, + 0, + 0, + 14824, + 0, + 0, + 14826, + 14827, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14830, + 0, + 0, + 0, + 0, + 0, + 14833, + 0, + 14845, + 0, + 0, + 0, + 0, + 0, + 14846, + 0, + 0, + 14847, + 14871, + 0, + 14873, + 0, + 14876, + 0, + 14877, + 14878, + 14880, + 0, + 0, + 0, + 0, + 0, + 14881, + 0, + 14882, + 14894, + 0, + 0, + 0, + 0, + 14895, + 0, + 14907, + 0, + 14908, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14911, + 0, + 0, + 0, + 0, + 14920, + 0, + 0, + 14931, + 0, + 14932, + 14934, + 14935, + 0, + 0, + 14936, + 0, + 14945, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 14947, + 0, + 0, + 14948, + 14949, + 14951, + 0, + 0, + 14952, + 0, + 0, + 0, + 14964, + 14973, + 0, + 0, + 14990, + 0, + 0, + 0, + 0, + 14995, + 0, + 0, + 14998, + 15001, + 0, + 0, + 15002, + 15020, + 0, + 0, + 0, + 0, + 0, + 0, + 15021, + 0, + 15022, + 0, + 0, + 0, + 0, + 15023, + 0, + 0, + 15025, + 15029, + 15033, + 0, + 0, + 0, + 15034, + 0, + 0, + 0, + 15035, + 0, + 0, + 0, + 0, + 0, + 15043, + 15044, + 0, + 0, + 0, + 15045, + 15046, + 15048, + 15050, + 0, + 15065, + 0, + 0, + 0, + 0, + 15066, + 0, + 0, + 15075, + 15082, + 15084, + 0, + 0, + 15085, + 15086, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15088, + 0, + 0, + 0, + 15089, + 0, + 0, + 0, + 0, + 15094, + 0, + 15096, + 0, + 15097, + 0, + 15100, + 0, + 0, + 15102, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15105, + 0, + 0, + 15106, + 0, + 15109, + 15113, + 0, + 0, + 0, + 15115, + 0, + 15118, + 0, + 0, + 0, + 0, + 0, + 0, + 15119, + 0, + 0, + 15120, + 0, + 0, + 0, + 0, + 0, + 15123, + 15129, + 0, + 0, + 0, + 15130, + 0, + 15131, + 0, + 0, + 15134, + 0, + 15135, + 0, + 0, + 0, + 15137, + 15138, + 0, + 0, + 0, + 0, + 0, + 0, + 15139, + 0, + 0, + 0, + 0, + 0, + 15140, + 0, + 0, + 15154, + 15162, + 0, + 15169, + 15170, + 0, + 15175, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15177, + 0, + 15178, + 15179, + 0, + 0, + 0, + 0, + 0, + 15183, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15185, + 15187, + 0, + 15194, + 15195, + 15196, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15204, + 0, + 0, + 0, + 0, + 15206, + 0, + 0, + 0, + 0, + 0, + 15207, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15213, + 0, + 15214, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15232, + 0, + 0, + 0, + 0, + 15234, + 0, + 15238, + 15240, + 0, + 15248, + 0, + 0, + 0, + 0, + 15250, + 15251, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15252, + 0, + 0, + 0, + 15255, + 15262, + 15266, + 0, + 0, + 0, + 15267, + 0, + 0, + 0, + 15277, + 15279, + 0, + 0, + 0, + 15280, + 15281, + 15282, + 0, + 0, + 0, + 0, + 0, + 15285, + 0, + 0, + 0, + 0, + 15289, + 0, + 0, + 15291, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15296, + 15297, + 0, + 0, + 15304, + 0, + 0, + 0, + 0, + 15306, + 0, + 0, + 0, + 0, + 0, + 0, + 15307, + 15308, + 0, + 15309, + 0, + 0, + 15311, + 0, + 0, + 15312, + 15313, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15314, + 15317, + 0, + 0, + 0, + 15318, + 15319, + 0, + 0, + 0, + 0, + 15320, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15321, + 0, + 0, + 0, + 0, + 0, + 15324, + 0, + 15325, + 15326, + 0, + 15330, + 0, + 0, + 0, + 0, + 15334, + 0, + 15335, + 0, + 15341, + 0, + 0, + 15342, + 0, + 0, + 15343, + 15344, + 0, + 0, + 0, + 0, + 15345, + 0, + 0, + 0, + 0, + 15347, + 0, + 0, + 15348, + 15349, + 15350, + 0, + 15356, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15357, + 0, + 15358, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15359, + 15360, + 15364, + 0, + 15380, + 0, + 0, + 0, + 0, + 0, + 15392, + 0, + 0, + 15393, + 0, + 15395, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15396, + 0, + 0, + 15397, + 15398, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15399, + 0, + 15400, + 0, + 0, + 0, + 15402, + 0, + 15405, + 15410, + 0, + 0, + 0, + 0, + 15411, + 0, + 0, + 0, + 15412, + 0, + 15416, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15428, + 0, + 15435, + 0, + 0, + 15438, + 0, + 0, + 0, + 0, + 15439, + 0, + 0, + 0, + 15440, + 0, + 0, + 0, + 15441, + 15449, + 15451, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15452, + 0, + 0, + 15455, + 0, + 0, + 0, + 15456, + 0, + 0, + 15458, + 0, + 15460, + 15461, + 0, + 0, + 0, + 0, + 0, + 15462, + 15464, + 0, + 15465, + 0, + 0, + 15466, + 0, + 0, + 15467, + 0, + 0, + 0, + 0, + 0, + 15468, + 0, + 0, + 0, + 0, + 15481, + 0, + 0, + 15484, + 0, + 15485, + 15486, + 0, + 0, + 0, + 15487, + 0, + 0, + 0, + 0, + 0, + 15488, + 0, + 15492, + 15498, + 0, + 0, + 0, + 15499, + 0, + 0, + 0, + 15500, + 0, + 15501, + 0, + 0, + 15512, + 0, + 15522, + 0, + 0, + 0, + 15524, + 0, + 15525, + 15526, + 0, + 0, + 15527, + 0, + 0, + 15545, + 15546, + 0, + 15548, + 15552, + 0, + 15553, + 0, + 0, + 0, + 15554, + 0, + 15555, + 0, + 15557, + 15565, + 15573, + 15577, + 15578, + 0, + 15582, + 0, + 15583, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15586, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15588, + 0, + 0, + 0, + 0, + 0, + 15589, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15593, + 15594, + 0, + 0, + 0, + 0, + 15595, + 0, + 0, + 0, + 0, + 0, + 0, + 15596, + 0, + 0, + 0, + 15597, + 0, + 0, + 0, + 0, + 15600, + 0, + 0, + 15601, + 0, + 0, + 0, + 0, + 15602, + 15603, + 0, + 0, + 0, + 0, + 0, + 0, + 15604, + 0, + 15609, + 0, + 0, + 15612, + 0, + 0, + 15613, + 0, + 0, + 15615, + 15617, + 15618, + 0, + 0, + 15620, + 0, + 15636, + 15637, + 0, + 0, + 15649, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15650, + 0, + 0, + 15651, + 0, + 0, + 0, + 15656, + 0, + 15658, + 0, + 0, + 0, + 15664, + 0, + 0, + 15665, + 0, + 0, + 15668, + 0, + 0, + 0, + 0, + 0, + 15669, + 0, + 0, + 15674, + 0, + 0, + 15675, + 0, + 0, + 0, + 0, + 15676, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15677, + 0, + 0, + 0, + 0, + 15678, + 0, + 0, + 0, + 0, + 0, + 15679, + 0, + 0, + 15681, + 0, + 15686, + 0, + 0, + 0, + 0, + 15687, + 0, + 15688, + 0, + 0, + 15690, + 0, + 0, + 0, + 15697, + 0, + 15699, + 15700, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15701, + 0, + 15702, + 15703, + 0, + 15704, + 0, + 15705, + 0, + 15707, + 0, + 15709, + 0, + 15712, + 15716, + 0, + 15717, + 0, + 15718, + 15720, + 0, + 0, + 0, + 0, + 0, + 15724, + 0, + 0, + 0, + 15725, + 0, + 15726, + 0, + 0, + 0, + 15740, + 0, + 15745, + 15746, + 0, + 0, + 15747, + 0, + 15748, + 0, + 0, + 0, + 0, + 0, + 15749, + 0, + 0, + 0, + 15752, + 0, + 15753, + 0, + 0, + 0, + 0, + 0, + 0, + 15759, + 0, + 0, + 0, + 15765, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15767, + 0, + 0, + 0, + 15771, + 0, + 0, + 15784, + 0, + 0, + 0, + 0, + 15785, + 15790, + 15791, + 0, + 0, + 15792, + 0, + 0, + 0, + 15807, + 0, + 15811, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15818, + 0, + 0, + 0, + 15819, + 0, + 0, + 0, + 0, + 15821, + 0, + 0, + 0, + 0, + 0, + 15822, + 15824, + 0, + 0, + 15827, + 0, + 0, + 15829, + 15831, + 0, + 15832, + 0, + 0, + 15833, + 0, + 15835, + 15838, + 15839, + 15843, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15844, + 0, + 0, + 0, + 0, + 15845, + 15851, + 15856, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15858, + 15860, + 0, + 15861, + 0, + 0, + 0, + 15864, + 0, + 0, + 0, + 0, + 15865, + 0, + 0, + 0, + 0, + 0, + 0, + 15866, + 0, + 15872, + 0, + 0, + 15876, + 0, + 0, + 0, + 0, + 15877, + 15878, + 15883, + 15885, + 0, + 0, + 15888, + 0, + 0, + 0, + 0, + 0, + 15889, + 15890, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15892, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15893, + 0, + 0, + 15894, + 0, + 0, + 0, + 15895, + 0, + 15896, + 15897, + 0, + 15898, + 15901, + 15902, + 0, + 15911, + 15915, + 0, + 15916, + 0, + 15924, + 15935, + 0, + 15937, + 0, + 0, + 0, + 0, + 0, + 15950, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15958, + 0, + 0, + 0, + 15961, + 0, + 0, + 15966, + 0, + 15967, + 0, + 0, + 15977, + 0, + 0, + 15978, + 0, + 0, + 15981, + 15982, + 15983, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 15986, + 0, + 0, + 0, + 15990, + 0, + 15991, + 15995, + 15998, + 0, + 15999, + 0, + 16000, + 0, + 0, + 0, + 0, + 16008, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16009, + 16011, + 0, + 16013, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16014, + 0, + 0, + 16015, + 16023, + 16024, + 16025, + 0, + 0, + 16026, + 0, + 16030, + 0, + 16032, + 0, + 16033, + 0, + 0, + 0, + 0, + 0, + 0, + 16035, + 16036, + 16037, + 0, + 0, + 0, + 0, + 0, + 16039, + 0, + 0, + 0, + 0, + 16041, + 0, + 0, + 0, + 0, + 0, + 16043, + 16044, + 0, + 0, + 16047, + 0, + 0, + 0, + 16048, + 0, + 0, + 16049, + 16050, + 16052, + 0, + 0, + 0, + 0, + 0, + 16055, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16056, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16058, + 16060, + 16061, + 0, + 0, + 16063, + 0, + 0, + 16064, + 0, + 0, + 0, + 16067, + 16068, + 0, + 0, + 16069, + 16078, + 0, + 0, + 0, + 16079, + 0, + 0, + 0, + 16080, + 0, + 16081, + 0, + 0, + 0, + 16088, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16089, + 16093, + 0, + 16097, + 0, + 16103, + 0, + 16104, + 16105, + 0, + 0, + 16256, + 0, + 0, + 16259, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16260, + 16261, + 0, + 0, + 16262, + 0, + 0, + 16263, + 0, + 16268, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16269, + 0, + 0, + 16270, + 16273, + 0, + 16274, + 0, + 0, + 0, + 0, + 16275, + 16276, + 16277, + 16280, + 0, + 0, + 0, + 16281, + 16284, + 0, + 0, + 0, + 16286, + 0, + 16289, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16290, + 0, + 0, + 0, + 0, + 16291, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16292, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16293, + 16295, + 16297, + 0, + 16302, + 0, + 16304, + 0, + 16305, + 0, + 16306, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16307, + 16308, + 16312, + 0, + 0, + 0, + 0, + 0, + 0, + 16313, + 16315, + 0, + 16318, + 0, + 0, + 0, + 16321, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16326, + 16333, + 16336, + 0, + 0, + 0, + 0, + 16337, + 16340, + 0, + 0, + 0, + 0, + 0, + 16345, + 0, + 0, + 16346, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16347, + 0, + 0, + 16348, + 0, + 0, + 0, + 0, + 16349, + 0, + 0, + 0, + 16350, + 0, + 16357, + 0, + 0, + 0, + 0, + 16359, + 16360, + 0, + 0, + 0, + 0, + 16362, + 16363, + 16364, + 16365, + 0, + 0, + 16366, + 0, + 0, + 0, + 0, + 16367, + 16368, + 0, + 16369, + 16374, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16376, + 0, + 0, + 0, + 0, + 16378, + 16379, + 0, + 16380, + 0, + 0, + 0, + 16381, + 16383, + 0, + 0, + 0, + 0, + 0, + 16390, + 0, + 0, + 0, + 16399, + 0, + 16402, + 16404, + 16406, + 16407, + 0, + 0, + 0, + 16409, + 16411, + 0, + 0, + 0, + 0, + 16412, + 0, + 16413, + 16415, + 16423, + 0, + 0, + 0, + 0, + 0, + 16424, + 0, + 0, + 0, + 16428, + 16434, + 16435, + 16449, + 0, + 16450, + 16451, + 0, + 0, + 0, + 16453, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16454, + 0, + 0, + 16456, + 16458, + 0, + 0, + 16459, + 0, + 0, + 16460, + 0, + 0, + 0, + 0, + 16462, + 0, + 16463, + 0, + 0, + 16466, + 0, + 0, + 0, + 0, + 0, + 16479, + 0, + 0, + 16480, + 0, + 16481, + 16484, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16485, + 0, + 0, + 0, + 0, + 0, + 0, + 16489, + 0, + 0, + 0, + 0, + 0, + 16491, + 0, + 0, + 16498, + 0, + 0, + 16503, + 0, + 16505, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16506, + 0, + 0, + 0, + 16508, + 16509, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16511, + 16513, + 0, + 0, + 0, + 16516, + 0, + 16517, + 0, + 16519, + 0, + 16529, + 0, + 0, + 16531, + 0, + 0, + 0, + 0, + 0, + 0, + 16534, + 0, + 0, + 16541, + 16542, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16543, + 16547, + 16548, + 0, + 0, + 0, + 16551, + 0, + 16552, + 0, + 0, + 0, + 16553, + 0, + 0, + 16558, + 0, + 0, + 16562, + 16565, + 0, + 0, + 0, + 16570, + 0, + 0, + 0, + 16573, + 16585, + 0, + 0, + 0, + 16586, + 16587, + 16595, + 0, + 16596, + 0, + 16598, + 0, + 0, + 0, + 16600, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16601, + 0, + 0, + 0, + 0, + 16603, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16604, + 16612, + 0, + 0, + 0, + 0, + 16613, + 0, + 16618, + 0, + 0, + 0, + 16640, + 0, + 0, + 16641, + 0, + 0, + 0, + 0, + 0, + 0, + 16645, + 0, + 0, + 0, + 0, + 16646, + 0, + 0, + 0, + 0, + 0, + 0, + 16651, + 0, + 0, + 0, + 0, + 16653, + 16654, + 0, + 0, + 0, + 16655, + 0, + 0, + 16656, + 16667, + 0, + 0, + 0, + 0, + 16671, + 0, + 16672, + 0, + 0, + 0, + 16673, + 0, + 0, + 0, + 0, + 0, + 16676, + 0, + 16686, + 0, + 0, + 0, + 0, + 16689, + 0, + 16690, + 0, + 16692, + 0, + 16693, + 0, + 16694, + 0, + 16696, + 0, + 0, + 0, + 16705, + 0, + 0, + 0, + 0, + 0, + 0, + 16707, + 0, + 0, + 0, + 16709, + 0, + 0, + 0, + 0, + 16711, + 0, + 16712, + 16713, + 0, + 0, + 0, + 16715, + 0, + 0, + 0, + 0, + 16716, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16718, + 16724, + 0, + 0, + 16726, + 16727, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16728, + 0, + 16729, + 0, + 0, + 16730, + 0, + 0, + 0, + 0, + 0, + 16731, + 0, + 0, + 0, + 16732, + 0, + 0, + 0, + 0, + 16734, + 16738, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16743, + 0, + 0, + 16745, + 0, + 0, + 0, + 0, + 0, + 16749, + 0, + 16752, + 0, + 0, + 0, + 0, + 16756, + 0, + 0, + 16758, + 0, + 16759, + 0, + 0, + 0, + 0, + 0, + 16760, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16762, + 0, + 16769, + 0, + 16770, + 0, + 16772, + 0, + 0, + 0, + 16777, + 16780, + 0, + 0, + 0, + 0, + 0, + 0, + 16781, + 0, + 0, + 16782, + 0, + 16784, + 0, + 0, + 16785, + 16787, + 16792, + 0, + 0, + 16794, + 0, + 0, + 0, + 16798, + 0, + 0, + 16809, + 0, + 0, + 16814, + 16816, + 16817, + 0, + 16819, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16820, + 0, + 0, + 16836, + 16839, + 0, + 0, + 16841, + 16851, + 16857, + 0, + 0, + 16858, + 16859, + 0, + 0, + 16860, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16862, + 0, + 16863, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16864, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16876, + 0, + 16881, + 16882, + 0, + 16885, + 16886, + 0, + 16887, + 0, + 0, + 0, + 16889, + 16891, + 0, + 0, + 0, + 0, + 0, + 16894, + 16895, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 16897, + 0, + 16898, + 0, + 0, + 0, + 0, + 0, + 16913, + 0, + 0, + 16924, + 16925, + 16926, + 0, + 0, + 16927, + 0, + 0, + 0, + 16937, + 16938, + 0, + 0, + 0, + 16940, + 16941, + 0, + 0, + 0, + 16942, + 16945, + 0, + 16946, + 16949, + 16950, + 0, + 0, + 0, + 16952, + 16955, + 0, + 0, + 0, + 16965, + 0, + 16969, + 0, + 0, + 16975, + 0, + 0, + 16976, + 0, + 0, + 0, + 0, + 16978, + 0, + 0, + 16981, + 0, + 16983, + 16989, + 0, + 0, + 0, + 0, + 16990, + 0, + 0, + 16991, + 0, + 0, + 0, + 16993, + 0, + 16994, + 16996, + 17000, + 0, + 0, + 0, + 0, + 0, + 17002, + 17004, + 0, + 17006, + 0, + 0, + 17007, + 0, + 0, + 0, + 0, + 17008, + 17013, + 17014, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17021, + 0, + 17031, + 0, + 0, + 0, + 0, + 0, + 17033, + 17036, + 0, + 17038, + 0, + 0, + 17039, + 0, + 17045, + 0, + 0, + 17046, + 17047, + 0, + 0, + 0, + 0, + 17048, + 0, + 17049, + 17050, + 0, + 17051, + 17053, + 0, + 17054, + 0, + 17055, + 0, + 0, + 0, + 0, + 0, + 17063, + 0, + 0, + 17064, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17065, + 0, + 0, + 17068, + 0, + 0, + 0, + 0, + 0, + 17072, + 0, + 0, + 0, + 0, + 0, + 0, + 17073, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17074, + 0, + 17080, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17081, + 17083, + 17084, + 0, + 0, + 0, + 17085, + 0, + 0, + 0, + 0, + 17092, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17093, + 0, + 17095, + 17102, + 0, + 0, + 0, + 0, + 0, + 0, + 17103, + 0, + 0, + 17105, + 0, + 17107, + 0, + 0, + 0, + 0, + 17114, + 0, + 0, + 0, + 0, + 0, + 17115, + 17125, + 17127, + 0, + 0, + 17128, + 0, + 0, + 0, + 17129, + 17130, + 0, + 17131, + 0, + 0, + 0, + 0, + 0, + 17132, + 17135, + 17145, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17146, + 0, + 17147, + 0, + 17148, + 0, + 0, + 0, + 0, + 0, + 0, + 17149, + 17150, + 0, + 17151, + 17153, + 0, + 17155, + 0, + 0, + 0, + 0, + 17163, + 17171, + 0, + 17174, + 0, + 0, + 0, + 0, + 17179, + 0, + 0, + 17182, + 17185, + 0, + 0, + 0, + 0, + 0, + 17186, + 0, + 0, + 17188, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17189, + 17191, + 0, + 17194, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17195, + 17196, + 17203, + 17204, + 0, + 0, + 17205, + 17217, + 0, + 0, + 0, + 0, + 0, + 17218, + 0, + 0, + 0, + 0, + 17219, + 0, + 17220, + 0, + 17221, + 0, + 0, + 17230, + 0, + 0, + 0, + 0, + 0, + 17236, + 0, + 17238, + 17239, + 0, + 0, + 0, + 17241, + 17244, + 0, + 0, + 17245, + 0, + 17248, + 0, + 0, + 17251, + 0, + 17252, + 0, + 0, + 17264, + 0, + 17266, + 0, + 0, + 0, + 17268, + 0, + 0, + 0, + 0, + 17271, + 17272, + 0, + 17273, + 0, + 17295, + 0, + 17302, + 0, + 17305, + 0, + 0, + 0, + 17306, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17308, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17309, + 0, + 17310, + 17313, + 0, + 0, + 0, + 0, + 17314, + 17315, + 0, + 17317, + 0, + 0, + 0, + 0, + 17318, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17320, + 0, + 0, + 0, + 0, + 0, + 0, + 17334, + 0, + 17344, + 17348, + 0, + 0, + 0, + 17350, + 17351, + 0, + 0, + 17353, + 0, + 0, + 17354, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17355, + 0, + 0, + 0, + 0, + 0, + 0, + 17356, + 17357, + 0, + 0, + 17359, + 0, + 0, + 0, + 17371, + 0, + 17372, + 0, + 0, + 0, + 17393, + 0, + 0, + 0, + 0, + 17394, + 0, + 0, + 0, + 0, + 0, + 17395, + 0, + 0, + 17399, + 0, + 0, + 0, + 17401, + 17417, + 0, + 17418, + 0, + 17419, + 0, + 0, + 0, + 0, + 0, + 17422, + 17423, + 0, + 0, + 0, + 0, + 0, + 17424, + 0, + 0, + 0, + 0, + 0, + 17428, + 17429, + 17433, + 0, + 0, + 0, + 17437, + 0, + 0, + 17441, + 0, + 0, + 17442, + 0, + 0, + 17453, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17454, + 17456, + 17462, + 0, + 0, + 17466, + 0, + 0, + 17468, + 0, + 0, + 17469, + 0, + 0, + 0, + 0, + 17470, + 0, + 17475, + 0, + 0, + 0, + 0, + 0, + 17479, + 0, + 0, + 0, + 17483, + 17484, + 0, + 17485, + 0, + 17486, + 0, + 17491, + 17492, + 0, + 0, + 17493, + 0, + 17494, + 17495, + 0, + 0, + 0, + 17496, + 0, + 0, + 0, + 17497, + 0, + 0, + 0, + 17502, + 0, + 0, + 0, + 0, + 0, + 17503, + 0, + 17505, + 0, + 17507, + 0, + 0, + 0, + 17512, + 17513, + 17514, + 0, + 0, + 17515, + 0, + 0, + 0, + 17519, + 0, + 0, + 0, + 17522, + 0, + 0, + 17523, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17527, + 0, + 0, + 0, + 17528, + 0, + 0, + 0, + 17534, + 0, + 0, + 0, + 0, + 17536, + 0, + 0, + 0, + 17539, + 0, + 17540, + 17543, + 17549, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17556, + 0, + 0, + 17558, + 0, + 17559, + 0, + 0, + 17560, + 0, + 0, + 0, + 17563, + 0, + 0, + 0, + 0, + 0, + 0, + 17564, + 0, + 0, + 17565, + 17566, + 0, + 17567, + 0, + 0, + 0, + 0, + 0, + 0, + 17569, + 17570, + 0, + 17575, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17581, + 0, + 0, + 0, + 17582, + 17583, + 0, + 17586, + 0, + 0, + 17587, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17588, + 0, + 0, + 0, + 0, + 17596, + 17597, + 0, + 0, + 17598, + 17600, + 0, + 0, + 0, + 0, + 0, + 0, + 17601, + 0, + 0, + 0, + 17604, + 0, + 0, + 17605, + 0, + 0, + 17607, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17612, + 0, + 0, + 17618, + 0, + 17621, + 17622, + 0, + 0, + 0, + 0, + 17623, + 0, + 0, + 17624, + 0, + 0, + 17630, + 0, + 0, + 17631, + 17633, + 17634, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17635, + 0, + 0, + 17636, + 0, + 0, + 17637, + 0, + 17638, + 0, + 17640, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17641, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17643, + 0, + 0, + 0, + 0, + 17645, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17646, + 17662, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17663, + 17664, + 0, + 17665, + 17666, + 0, + 0, + 0, + 17669, + 17671, + 17673, + 0, + 17679, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17684, + 0, + 0, + 0, + 17686, + 0, + 17714, + 0, + 0, + 17720, + 17722, + 17726, + 0, + 0, + 17728, + 0, + 0, + 17729, + 0, + 0, + 0, + 17732, + 0, + 17733, + 0, + 17734, + 0, + 0, + 0, + 17735, + 0, + 0, + 0, + 0, + 17737, + 0, + 0, + 0, + 0, + 17739, + 0, + 0, + 0, + 17741, + 17742, + 0, + 0, + 0, + 0, + 17743, + 17744, + 17745, + 0, + 0, + 0, + 17749, + 0, + 17750, + 17751, + 17752, + 17754, + 17761, + 17762, + 0, + 17763, + 0, + 17766, + 0, + 17772, + 0, + 0, + 0, + 0, + 0, + 17775, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17776, + 0, + 0, + 17777, + 0, + 0, + 17778, + 17779, + 0, + 17782, + 17783, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17784, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17821, + 0, + 0, + 0, + 17822, + 0, + 0, + 0, + 17823, + 17825, + 0, + 0, + 0, + 0, + 0, + 17826, + 17831, + 17832, + 17833, + 0, + 0, + 17845, + 0, + 0, + 0, + 17846, + 0, + 0, + 0, + 17848, + 17850, + 17854, + 0, + 17855, + 0, + 0, + 17859, + 0, + 0, + 0, + 0, + 0, + 0, + 17860, + 17861, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 17870, + 17871, + 0, + 0, + 0, + 0, + 0, + 0, + 17872, + 0, + 0, + 0, + 17879, + 0, + 0, + 0, + 17881, + 17883, + 0, + 17884, + 0, + 17885, + 0, + 0, + 17886, + 0, + 0, + 17887, + 17891, + 17953, + 0, + 0, + 0, + 0, + 17954, + 0, + 0, + 17955, + 0, + 17968, + 0, + 0, + 17972, + 0, + 0, + 0, + 0, + 0, + 17974, + 0, + 0, + 0, + 0, + 17976, + 17978, + 0, + 0, + 17983, + 0, + 0, + 0, + 0, + 18003, + 0, + 0, + 0, + 0, + 0, + 18007, + 0, + 0, + 0, + 0, + 0, + 18009, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18010, + 0, + 0, + 0, + 0, + 0, + 0, + 18012, + 0, + 0, + 18014, + 0, + 0, + 0, + 18015, + 0, + 0, + 0, + 18016, + 0, + 18017, + 0, + 0, + 0, + 18030, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18031, + 0, + 0, + 18036, + 18037, + 18038, + 0, + 0, + 18049, + 18056, + 0, + 18057, + 18058, + 0, + 18059, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18062, + 0, + 0, + 0, + 0, + 18064, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18067, + 0, + 0, + 0, + 18068, + 0, + 0, + 18075, + 0, + 0, + 18078, + 18093, + 18094, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18097, + 0, + 0, + 0, + 0, + 0, + 18098, + 18100, + 0, + 0, + 0, + 18108, + 0, + 18111, + 0, + 0, + 18112, + 0, + 18113, + 0, + 0, + 18115, + 18116, + 0, + 18118, + 0, + 0, + 0, + 0, + 18121, + 0, + 0, + 0, + 0, + 18123, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18124, + 0, + 0, + 0, + 0, + 18125, + 18126, + 0, + 18127, + 0, + 0, + 18128, + 18135, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18150, + 0, + 0, + 0, + 0, + 0, + 18151, + 18152, + 0, + 0, + 18156, + 18164, + 0, + 18166, + 18171, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18172, + 18183, + 0, + 18184, + 0, + 0, + 0, + 0, + 18185, + 0, + 18187, + 0, + 0, + 0, + 0, + 0, + 18188, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18189, + 0, + 0, + 18190, + 0, + 0, + 18191, + 18192, + 0, + 0, + 18194, + 18195, + 18196, + 0, + 0, + 0, + 18197, + 0, + 18203, + 0, + 18204, + 0, + 0, + 0, + 0, + 18205, + 0, + 0, + 0, + 18207, + 18208, + 0, + 0, + 18214, + 0, + 0, + 0, + 18215, + 18216, + 0, + 0, + 0, + 18220, + 0, + 0, + 18222, + 0, + 0, + 0, + 0, + 0, + 18223, + 0, + 18225, + 18231, + 0, + 18234, + 0, + 18235, + 0, + 0, + 0, + 0, + 18240, + 0, + 0, + 18241, + 18242, + 0, + 0, + 0, + 0, + 0, + 18243, + 18251, + 0, + 18253, + 0, + 18254, + 0, + 0, + 0, + 18266, + 0, + 0, + 0, + 0, + 0, + 0, + 18269, + 18270, + 18271, + 18273, + 18281, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18282, + 0, + 18283, + 0, + 18284, + 0, + 0, + 0, + 0, + 0, + 0, + 18285, + 0, + 18287, + 18289, + 0, + 0, + 18290, + 0, + 0, + 0, + 0, + 18308, + 0, + 0, + 0, + 18310, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18311, + 0, + 18312, + 18313, + 0, + 18315, + 0, + 0, + 18316, + 18320, + 0, + 18331, + 0, + 18332, + 0, + 18336, + 0, + 0, + 0, + 0, + 18337, + 0, + 18340, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18341, + 0, + 18344, + 18345, + 0, + 18346, + 0, + 0, + 0, + 0, + 0, + 18348, + 0, + 18351, + 0, + 0, + 18356, + 0, + 0, + 0, + 0, + 0, + 0, + 18357, + 0, + 0, + 0, + 0, + 0, + 18367, + 0, + 0, + 0, + 18368, + 0, + 18369, + 0, + 18370, + 18371, + 0, + 0, + 0, + 18437, + 18444, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18445, + 18450, + 0, + 0, + 0, + 0, + 18451, + 0, + 18452, + 0, + 0, + 0, + 18453, + 0, + 0, + 0, + 0, + 0, + 18455, + 0, + 0, + 0, + 18456, + 0, + 18457, + 0, + 18460, + 0, + 0, + 18461, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18466, + 0, + 0, + 18467, + 0, + 0, + 0, + 0, + 18473, + 0, + 0, + 0, + 18476, + 0, + 18477, + 0, + 0, + 0, + 18478, + 18479, + 18480, + 0, + 0, + 0, + 18485, + 0, + 0, + 0, + 18486, + 0, + 0, + 0, + 0, + 0, + 0, + 18488, + 18490, + 0, + 0, + 0, + 0, + 0, + 0, + 18491, + 0, + 0, + 0, + 0, + 0, + 18495, + 0, + 0, + 18496, + 0, + 0, + 0, + 0, + 0, + 0, + 18505, + 0, + 18521, + 0, + 18522, + 18523, + 0, + 0, + 0, + 18525, + 18526, + 0, + 0, + 0, + 0, + 0, + 18527, + 0, + 0, + 0, + 0, + 18532, + 18533, + 0, + 18534, + 0, + 0, + 0, + 0, + 0, + 0, + 18535, + 18537, + 0, + 18538, + 0, + 0, + 0, + 0, + 0, + 0, + 18540, + 18541, + 18542, + 18543, + 0, + 18546, + 0, + 0, + 0, + 0, + 18553, + 18556, + 0, + 0, + 18558, + 0, + 0, + 18569, + 18571, + 0, + 0, + 0, + 18572, + 0, + 18574, + 0, + 0, + 0, + 0, + 18586, + 0, + 0, + 0, + 0, + 0, + 18588, + 0, + 0, + 18589, + 0, + 0, + 0, + 0, + 0, + 0, + 18590, + 0, + 18592, + 0, + 0, + 0, + 0, + 18594, + 0, + 0, + 0, + 18596, + 0, + 0, + 18597, + 18598, + 0, + 0, + 18601, + 0, + 0, + 0, + 0, + 18602, + 0, + 0, + 0, + 18603, + 18604, + 0, + 18605, + 0, + 0, + 0, + 0, + 18608, + 0, + 0, + 18611, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18612, + 0, + 18616, + 0, + 0, + 18617, + 18619, + 0, + 0, + 0, + 18628, + 0, + 0, + 0, + 18629, + 0, + 0, + 18630, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18631, + 0, + 18632, + 0, + 0, + 18635, + 18637, + 0, + 0, + 0, + 0, + 0, + 0, + 18641, + 18643, + 18648, + 0, + 18652, + 0, + 0, + 18653, + 0, + 18655, + 18656, + 0, + 0, + 0, + 18657, + 0, + 0, + 18666, + 18674, + 0, + 0, + 0, + 0, + 18677, + 18684, + 18685, + 0, + 0, + 18686, + 0, + 0, + 18690, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18695, + 18696, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18697, + 0, + 0, + 18700, + 0, + 0, + 0, + 0, + 0, + 0, + 18702, + 0, + 18708, + 0, + 0, + 18709, + 0, + 18710, + 0, + 0, + 18711, + 0, + 18714, + 0, + 0, + 18718, + 0, + 0, + 0, + 0, + 0, + 0, + 18719, + 0, + 0, + 18722, + 0, + 18726, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18731, + 0, + 0, + 0, + 0, + 0, + 18739, + 18741, + 0, + 0, + 18742, + 0, + 18743, + 18744, + 18746, + 18748, + 0, + 18752, + 18753, + 0, + 0, + 18754, + 18763, + 0, + 18765, + 0, + 0, + 0, + 18766, + 0, + 0, + 0, + 18769, + 0, + 0, + 0, + 0, + 0, + 18773, + 18778, + 18779, + 18781, + 0, + 0, + 18784, + 18787, + 0, + 18788, + 0, + 18793, + 0, + 0, + 0, + 0, + 0, + 0, + 18795, + 0, + 0, + 18800, + 0, + 0, + 0, + 0, + 0, + 18801, + 18804, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18806, + 0, + 0, + 0, + 18811, + 18815, + 18816, + 0, + 0, + 0, + 0, + 18825, + 0, + 0, + 18827, + 18829, + 0, + 0, + 18830, + 0, + 0, + 0, + 0, + 18831, + 0, + 0, + 18832, + 0, + 0, + 0, + 0, + 18833, + 0, + 18840, + 0, + 18841, + 0, + 18842, + 0, + 0, + 0, + 0, + 18843, + 0, + 18844, + 0, + 0, + 0, + 0, + 0, + 0, + 18845, + 18846, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18848, + 0, + 0, + 0, + 18853, + 18860, + 0, + 0, + 18862, + 18866, + 0, + 0, + 18867, + 18869, + 0, + 0, + 18874, + 18881, + 18891, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18892, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18895, + 0, + 18896, + 0, + 0, + 0, + 18900, + 0, + 0, + 0, + 18901, + 0, + 18902, + 18915, + 18916, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18919, + 0, + 0, + 0, + 0, + 0, + 18920, + 0, + 0, + 0, + 18921, + 18929, + 0, + 0, + 0, + 0, + 18930, + 0, + 0, + 0, + 0, + 0, + 0, + 18932, + 0, + 0, + 0, + 0, + 18934, + 18942, + 0, + 0, + 0, + 18951, + 18957, + 0, + 0, + 0, + 0, + 18958, + 0, + 0, + 0, + 0, + 18959, + 18960, + 0, + 0, + 18961, + 0, + 0, + 18962, + 0, + 0, + 0, + 0, + 18963, + 18964, + 0, + 0, + 0, + 18965, + 0, + 18967, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18968, + 0, + 18969, + 0, + 18970, + 18973, + 18976, + 0, + 0, + 0, + 0, + 0, + 0, + 18977, + 0, + 0, + 0, + 18981, + 0, + 0, + 0, + 18990, + 0, + 18998, + 0, + 0, + 0, + 0, + 0, + 18999, + 19003, + 0, + 0, + 19005, + 0, + 0, + 0, + 19006, + 0, + 0, + 0, + 0, + 0, + 0, + 19008, + 19011, + 0, + 0, + 19018, + 0, + 0, + 19019, + 0, + 19024, + 0, + 19031, + 19032, + 0, + 19039, + 0, + 19041, + 19050, + 0, + 0, + 0, + 19051, + 19055, + 19056, + 0, + 19059, + 19063, + 19064, + 0, + 0, + 19088, + 0, + 0, + 0, + 19093, + 19094, + 0, + 0, + 0, + 0, + 19095, + 0, + 19096, + 0, + 0, + 0, + 19097, + 0, + 0, + 19098, + 0, + 19099, + 19100, + 0, + 0, + 19103, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19111, + 0, + 0, + 0, + 0, + 0, + 0, + 19112, + 0, + 0, + 0, + 19116, + 19117, + 0, + 19121, + 19122, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19123, + 19124, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19125, + 19126, + 0, + 19128, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19129, + 19130, + 19131, + 19132, + 0, + 0, + 19146, + 0, + 0, + 19147, + 19156, + 19158, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19182, + 19185, + 0, + 0, + 19187, + 0, + 0, + 0, + 19193, + 0, + 0, + 0, + 0, + 0, + 19194, + 0, + 19197, + 0, + 0, + 0, + 0, + 19198, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19202, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19203, + 0, + 19205, + 19210, + 0, + 0, + 0, + 19213, + 0, + 19218, + 0, + 0, + 0, + 19223, + 19229, + 0, + 0, + 19230, + 0, + 0, + 19231, + 19232, + 19233, + 19239, + 0, + 0, + 0, + 0, + 0, + 19240, + 0, + 19248, + 19249, + 0, + 0, + 0, + 0, + 19254, + 0, + 19256, + 19258, + 19259, + 0, + 0, + 19261, + 0, + 19266, + 0, + 0, + 0, + 19272, + 0, + 19278, + 19281, + 19282, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19283, + 0, + 0, + 19284, + 0, + 0, + 19285, + 19287, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19288, + 19291, + 0, + 19292, + 0, + 0, + 0, + 0, + 19297, + 0, + 19298, + 0, + 0, + 0, + 0, + 19302, + 19303, + 0, + 0, + 0, + 0, + 19304, + 19305, + 0, + 0, + 0, + 0, + 19314, + 0, + 0, + 19315, + 0, + 0, + 19321, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19322, + 0, + 19333, + 0, + 19334, + 19335, + 0, + 19336, + 19337, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19346, + 0, + 0, + 19353, + 0, + 19354, + 19362, + 0, + 19366, + 19367, + 0, + 0, + 19369, + 0, + 19375, + 0, + 19377, + 19380, + 19388, + 0, + 0, + 0, + 0, + 0, + 19389, + 19390, + 0, + 0, + 0, + 0, + 19392, + 0, + 0, + 0, + 0, + 0, + 19402, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19412, + 0, + 0, + 19413, + 19422, + 0, + 19424, + 0, + 0, + 0, + 19425, + 0, + 0, + 0, + 19428, + 0, + 0, + 0, + 0, + 19431, + 0, + 0, + 0, + 0, + 0, + 19432, + 0, + 0, + 0, + 0, + 0, + 19448, + 19459, + 0, + 0, + 19461, + 0, + 19462, + 19463, + 0, + 19467, + 19474, + 19482, + 0, + 0, + 0, + 0, + 19494, + 0, + 0, + 0, + 0, + 19501, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19502, + 19504, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19505, + 0, + 0, + 0, + 0, + 19506, + 19507, + 0, + 0, + 0, + 19508, + 0, + 0, + 19511, + 0, + 0, + 19514, + 0, + 19515, + 0, + 19516, + 0, + 19518, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19530, + 0, + 19537, + 19538, + 0, + 19543, + 19546, + 0, + 19547, + 19551, + 0, + 0, + 0, + 0, + 0, + 0, + 19552, + 19553, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19555, + 0, + 0, + 19556, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19560, + 19561, + 0, + 0, + 19562, + 0, + 0, + 0, + 0, + 0, + 0, + 19565, + 19567, + 0, + 19568, + 0, + 0, + 0, + 19569, + 19570, + 0, + 19578, + 0, + 0, + 0, + 0, + 19580, + 0, + 0, + 0, + 0, + 19581, + 19584, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19585, + 19586, + 0, + 0, + 0, + 19587, + 19588, + 0, + 19589, + 0, + 0, + 0, + 0, + 0, + 0, + 19592, + 19593, + 19599, + 0, + 19600, + 0, + 0, + 19604, + 0, + 0, + 19605, + 0, + 19606, + 19608, + 19610, + 0, + 19613, + 19614, + 0, + 0, + 0, + 0, + 0, + 0, + 19616, + 19617, + 0, + 0, + 19618, + 0, + 0, + 19619, + 0, + 0, + 0, + 19620, + 19621, + 19631, + 0, + 0, + 19632, + 19634, + 19636, + 0, + 19643, + 0, + 0, + 19644, + 19658, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19659, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19675, + 19677, + 0, + 0, + 0, + 0, + 19679, + 0, + 19683, + 0, + 19684, + 0, + 0, + 0, + 0, + 0, + 0, + 19687, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19688, + 19689, + 19692, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19695, + 19697, + 0, + 0, + 0, + 0, + 0, + 19698, + 19699, + 0, + 0, + 19700, + 0, + 19702, + 0, + 0, + 19703, + 0, + 0, + 0, + 0, + 0, + 0, + 19704, + 19708, + 0, + 19710, + 0, + 19713, + 0, + 0, + 0, + 19715, + 0, + 0, + 0, + 0, + 19718, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19720, + 0, + 19722, + 0, + 0, + 19725, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19730, + 0, + 0, + 0, + 0, + 0, + 19731, + 0, + 19734, + 19735, + 19739, + 0, + 0, + 19740, + 0, + 19741, + 0, + 0, + 0, + 19746, + 0, + 0, + 19747, + 0, + 19771, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19772, + 19775, + 0, + 0, + 0, + 0, + 0, + 0, + 19778, + 0, + 0, + 0, + 0, + 0, + 19779, + 0, + 0, + 19780, + 19790, + 0, + 19791, + 0, + 0, + 19792, + 0, + 0, + 0, + 19793, + 0, + 0, + 19796, + 19797, + 0, + 0, + 0, + 19799, + 0, + 0, + 0, + 19801, + 0, + 0, + 0, + 0, + 19803, + 0, + 19804, + 0, + 19805, + 0, + 0, + 19807, + 0, + 0, + 0, + 19808, + 0, + 0, + 0, + 0, + 0, + 0, + 19809, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19816, + 0, + 19821, + 0, + 19822, + 19830, + 19831, + 0, + 0, + 0, + 19833, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19838, + 0, + 0, + 0, + 0, + 19839, + 0, + 0, + 19843, + 0, + 0, + 0, + 0, + 19845, + 0, + 0, + 0, + 0, + 19847, + 0, + 0, + 19848, + 0, + 19849, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19851, + 0, + 0, + 0, + 19854, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19864, + 0, + 19865, + 0, + 19866, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19868, + 0, + 0, + 19870, + 0, + 0, + 19871, + 0, + 0, + 19872, + 19873, + 19875, + 0, + 19880, + 19882, + 19884, + 0, + 0, + 19885, + 19886, + 19888, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19890, + 19892, + 19893, + 0, + 0, + 19894, + 0, + 0, + 0, + 19895, + 0, + 19896, + 19902, + 0, + 0, + 19903, + 0, + 0, + 19905, + 0, + 0, + 0, + 19906, + 0, + 19908, + 0, + 19909, + 19911, + 0, + 0, + 0, + 19913, + 19920, + 0, + 19938, + 19939, + 19940, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19942, + 0, + 19943, + 0, + 19945, + 0, + 0, + 0, + 19951, + 19952, + 19954, + 19960, + 0, + 19965, + 0, + 19971, + 0, + 0, + 0, + 0, + 0, + 19975, + 0, + 19976, + 0, + 19990, + 0, + 0, + 19991, + 0, + 19993, + 0, + 19995, + 0, + 0, + 0, + 19998, + 19999, + 20001, + 0, + 20003, + 20005, + 0, + 20011, + 20012, + 0, + 0, + 0, + 0, + 0, + 0, + 20014, + 0, + 20020, + 0, + 0, + 0, + 0, + 20021, + 0, + 0, + 0, + 0, + 0, + 20023, + 20024, + 0, + 0, + 0, + 0, + 0, + 20025, + 0, + 0, + 20027, + 0, + 0, + 20029, + 0, + 0, + 20032, + 0, + 0, + 0, + 0, + 20044, + 20045, + 0, + 20048, + 20049, + 0, + 0, + 20050, + 0, + 20052, + 0, + 0, + 20054, + 20057, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20059, + 0, + 0, + 20061, + 0, + 20062, + 0, + 20064, + 0, + 0, + 20066, + 0, + 0, + 20067, + 0, + 0, + 0, + 0, + 20069, + 0, + 0, + 0, + 0, + 0, + 0, + 20070, + 20071, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20072, + 0, + 0, + 20073, + 20074, + 0, + 0, + 0, + 0, + 0, + 20075, + 0, + 20078, + 0, + 0, + 0, + 0, + 20080, + 0, + 20081, + 0, + 0, + 0, + 0, + 0, + 0, + 20095, + 0, + 20098, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20107, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20112, + 0, + 0, + 0, + 20113, + 20114, + 0, + 0, + 0, + 20115, + 20123, + 20124, + 0, + 0, + 0, + 20131, + 20133, + 20134, + 0, + 0, + 0, + 0, + 20136, + 0, + 0, + 20137, + 20138, + 20150, + 0, + 20152, + 0, + 0, + 0, + 20153, + 0, + 0, + 20154, + 0, + 0, + 0, + 20158, + 0, + 20163, + 0, + 0, + 20164, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20166, + 0, + 20168, + 0, + 20170, + 0, + 20175, + 0, + 0, + 20178, + 0, + 0, + 0, + 0, + 20223, + 0, + 0, + 0, + 0, + 20224, + 0, + 20226, + 0, + 0, + 20230, + 0, + 20231, + 0, + 0, + 0, + 0, + 20232, + 0, + 0, + 20233, + 20234, + 0, + 20244, + 0, + 20247, + 0, + 0, + 0, + 0, + 0, + 0, + 20249, + 0, + 0, + 0, + 20250, + 0, + 0, + 0, + 0, + 20251, + 0, + 20253, + 0, + 20254, + 0, + 0, + 0, + 0, + 20256, + 0, + 0, + 20264, + 0, + 0, + 0, + 0, + 20266, + 0, + 0, + 0, + 20278, + 0, + 0, + 20279, + 20282, + 0, + 0, + 0, + 0, + 0, + 20283, + 0, + 20284, + 0, + 20285, + 0, + 20287, + 20290, + 0, + 0, + 0, + 0, + 20292, + 0, + 0, + 0, + 0, + 20293, + 20297, + 0, + 0, + 0, + 0, + 0, + 0, + 20299, + 0, + 20300, + 20303, + 0, + 0, + 0, + 0, + 0, + 0, + 20307, + 0, + 0, + 20308, + 0, + 20309, + 0, + 20310, + 0, + 0, + 0, + 0, + 0, + 0, + 20312, + 0, + 0, + 0, + 20314, + 0, + 0, + 0, + 0, + 20315, + 20316, + 0, + 20322, + 0, + 0, + 0, + 0, + 0, + 0, + 20339, + 0, + 0, + 0, + 20342, + 0, + 0, + 0, + 0, + 20352, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20362, + 0, + 0, + 20365, + 0, + 20375, + 20377, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20378, + 20379, + 0, + 20380, + 0, + 0, + 20381, + 0, + 20382, + 0, + 20383, + 0, + 20388, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20390, + 20392, + 20393, + 0, + 0, + 20395, + 0, + 0, + 0, + 0, + 0, + 20396, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20398, + 20415, + 0, + 0, + 0, + 20417, + 0, + 0, + 20420, + 0, + 0, + 20426, + 20428, + 0, + 20431, + 0, + 0, + 20432, + 0, + 20433, + 20434, + 20435, + 0, + 0, + 0, + 0, + 20440, + 0, + 0, + 0, + 0, + 0, + 20442, + 0, + 20443, + 0, + 20446, + 0, + 0, + 0, + 0, + 20448, + 0, + 20451, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20452, + 20453, + 0, + 0, + 20454, + 0, + 0, + 0, + 0, + 0, + 0, + 20457, + 0, + 20458, + 0, + 0, + 0, + 20465, + 0, + 0, + 0, + 0, + 0, + 20469, + 0, + 0, + 0, + 20473, + 0, + 20476, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20477, + 0, + 0, + 20485, + 0, + 0, + 20486, + 0, + 0, + 20487, + 0, + 20496, + 0, + 20497, + 0, + 0, + 20498, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20499, + 20500, + 0, + 20501, + 0, + 0, + 0, + 0, + 0, + 20520, + 20527, + 0, + 20529, + 0, + 0, + 0, + 0, + 20539, + 0, + 0, + 20540, + 0, + 0, + 0, + 20543, + 0, + 0, + 0, + 20546, + 0, + 0, + 0, + 0, + 0, + 20548, + 0, + 0, + 20563, + 0, + 0, + 20564, + 0, + 20566, + 0, + 0, + 0, + 0, + 0, + 20589, + 0, + 0, + 0, + 0, + 20590, + 0, + 0, + 20593, + 20594, + 0, + 0, + 0, + 0, + 20595, + 0, + 20597, + 20598, + 0, + 0, + 0, + 20618, + 20620, + 0, + 0, + 0, + 0, + 20621, + 0, + 0, + 0, + 0, + 20627, + 0, + 0, + 0, + 0, + 0, + 20628, + 0, + 0, + 0, + 20629, + 0, + 20630, + 0, + 0, + 20639, + 0, + 0, + 0, + 0, + 0, + 20707, + 0, + 0, + 20709, + 0, + 0, + 0, + 20713, + 20714, + 0, + 0, + 0, + 0, + 0, + 20724, + 20725, + 0, + 0, + 0, + 0, + 20726, + 20728, + 20729, + 0, + 20733, + 0, + 20734, + 0, + 20735, + 20736, + 0, + 20737, + 0, + 0, + 20744, + 0, + 20745, + 0, + 20748, + 0, + 0, + 20749, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20750, + 0, + 0, + 0, + 0, + 20754, + 0, + 0, + 0, + 20761, + 0, + 0, + 20763, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20766, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20767, + 0, + 0, + 0, + 0, + 20768, + 0, + 20769, + 20777, + 0, + 0, + 0, + 0, + 0, + 0, + 20785, + 0, + 0, + 0, + 20786, + 20795, + 20801, + 0, + 20802, + 0, + 20807, + 0, + 0, + 20808, + 0, + 0, + 20810, + 0, + 0, + 20811, + 0, + 20812, + 0, + 0, + 0, + 0, + 0, + 20813, + 0, + 0, + 20818, + 20820, + 20821, + 0, + 0, + 0, + 20822, + 0, + 20823, + 0, + 0, + 0, + 20826, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20829, + 20830, + 20831, + 0, + 20832, + 20836, + 0, + 0, + 20839, + 0, + 0, + 20840, + 20842, + 0, + 20843, + 0, + 20844, + 0, + 20854, + 0, + 0, + 0, + 20855, + 0, + 0, + 0, + 0, + 20856, + 0, + 0, + 0, + 20869, + 0, + 0, + 20871, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20873, + 0, + 0, + 0, + 0, + 0, + 20876, + 0, + 0, + 0, + 0, + 0, + 20880, + 0, + 0, + 20882, + 0, + 0, + 0, + 0, + 20883, + 20884, + 0, + 0, + 20890, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 20891, + 0, + 0, + 0, + 0, + 0, + 20905, + 0, + 20906, + 20910, + 0, + 0, + 20912, + 20915, + 0, + 0, + 0, + 0, + 0, + 20916, + 0, + 20917, + 0, + 20919, + 20920, + 20922, + 0, + 20927, + 0, + 20928, + 20929, + 20930, + 0, + 0, + 20935, + 0, + 0, + 20939, + 0, + 0, + 20941, + 0, + 0, + 0, + 20943, + 0, + 0, + 0, + 20946, + 20947, + 0, + 0, + 0, + 0, + 0, + 20950, + 0, + 20954, + 0, + 0, + 20955, + 20964, + 0, + 0, + 20967, + 0, + 0, + 0, + 0, + 0, + 20973, + 20975, + 0, + 0, + 0, + 20984, + 0, + 20987, + 20988, + 0, + 0, + 0, + 0, + 0, + 20989, + 0, + 0, + 0, + 20995, + 0, + 20998, + 0, + 20999, + 0, + 0, + 0, + 0, + 21000, + 21001, + 0, + 0, + 0, + 0, + 21008, + 0, + 21010, + 0, + 21016, + 0, + 0, + 0, + 21017, + 21018, + 0, + 0, + 0, + 0, + 0, + 21021, + 21026, + 21027, + 21028, + 0, + 0, + 21029, + 0, + 0, + 0, + 0, + 0, + 21030, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21031, + 21032, + 0, + 0, + 0, + 0, + 0, + 21037, + 0, + 0, + 21038, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21039, + 0, + 21041, + 0, + 21046, + 21047, + 0, + 0, + 0, + 21049, + 21053, + 0, + 0, + 21057, + 21064, + 21065, + 0, + 0, + 21066, + 21067, + 0, + 0, + 0, + 21069, + 0, + 0, + 0, + 21071, + 21072, + 0, + 0, + 21073, + 0, + 21074, + 0, + 0, + 21078, + 0, + 0, + 0, + 0, + 21079, + 0, + 0, + 21080, + 21081, + 0, + 0, + 21086, + 21087, + 0, + 21089, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21091, + 0, + 21093, + 0, + 21094, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21095, + 0, + 0, + 0, + 0, + 0, + 21096, + 0, + 21098, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21099, + 0, + 0, + 21100, + 21101, + 21102, + 0, + 0, + 0, + 0, + 0, + 21103, + 0, + 21104, + 0, + 0, + 0, + 0, + 0, + 21105, + 21108, + 21109, + 0, + 0, + 21112, + 21113, + 0, + 0, + 0, + 0, + 0, + 0, + 21115, + 21122, + 21123, + 0, + 0, + 0, + 0, + 0, + 21125, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21129, + 21131, + 0, + 0, + 21134, + 0, + 0, + 0, + 21137, + 21142, + 0, + 21143, + 0, + 0, + 21144, + 0, + 21145, + 21146, + 0, + 21152, + 21154, + 21155, + 21156, + 0, + 0, + 0, + 21160, + 0, + 0, + 0, + 0, + 0, + 0, + 21161, + 0, + 21164, + 0, + 21166, + 0, + 0, + 0, + 0, + 21170, + 0, + 0, + 0, + 0, + 21171, + 0, + 0, + 21172, + 0, + 21174, + 0, + 21175, + 0, + 0, + 0, + 0, + 0, + 21176, + 21179, + 21188, + 0, + 0, + 0, + 21189, + 0, + 0, + 21190, + 0, + 0, + 0, + 21192, + 0, + 0, + 21193, + 0, + 0, + 0, + 21198, + 0, + 21212, + 0, + 0, + 21213, + 0, + 0, + 0, + 0, + 0, + 0, + 21215, + 21216, + 0, + 0, + 21223, + 21225, + 0, + 21226, + 0, + 0, + 0, + 0, + 21227, + 21228, + 0, + 0, + 21229, + 0, + 0, + 0, + 0, + 21230, + 21236, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21237, + 0, + 0, + 21238, + 21239, + 0, + 0, + 0, + 0, + 21256, + 0, + 0, + 0, + 0, + 0, + 21257, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21259, + 0, + 0, + 0, + 21263, + 0, + 21272, + 0, + 21274, + 0, + 21282, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21283, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21294, + 0, + 0, + 21297, + 0, + 0, + 0, + 0, + 21298, + 0, + 0, + 0, + 21299, + 0, + 21300, + 21302, + 0, + 21316, + 0, + 21318, + 21322, + 21323, + 0, + 21324, + 0, + 21326, + 0, + 0, + 0, + 21327, + 21328, + 0, + 0, + 0, + 21352, + 0, + 0, + 21354, + 21361, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21362, + 0, + 0, + 0, + 21363, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21366, + 0, + 0, + 21367, + 21372, + 21374, + 0, + 0, + 0, + 21375, + 21377, + 0, + 21378, + 0, + 0, + 0, + 21380, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21381, + 0, + 0, + 0, + 0, + 0, + 0, + 21382, + 0, + 21383, + 0, + 0, + 21384, + 0, + 0, + 21385, + 0, + 0, + 0, + 0, + 21389, + 21390, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21397, + 21398, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21399, + 0, + 21400, + 0, + 0, + 0, + 0, + 21402, + 0, + 0, + 0, + 21403, + 21404, + 0, + 21405, + 21406, + 0, + 0, + 0, + 21407, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21408, + 0, + 0, + 0, + 0, + 21409, + 0, + 21421, + 0, + 21422, + 0, + 0, + 0, + 21425, + 21428, + 0, + 0, + 0, + 0, + 21429, + 0, + 0, + 0, + 0, + 0, + 21433, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21434, + 0, + 21443, + 0, + 21444, + 21449, + 0, + 21452, + 0, + 21453, + 21454, + 0, + 0, + 0, + 21457, + 0, + 0, + 21458, + 0, + 0, + 0, + 21460, + 21461, + 0, + 0, + 21464, + 0, + 0, + 0, + 21473, + 21478, + 0, + 0, + 21479, + 0, + 0, + 21481, + 21483, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21484, + 0, + 0, + 21485, + 21486, + 0, + 0, + 21488, + 0, + 0, + 0, + 0, + 0, + 0, + 21523, + 0, + 0, + 21525, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21526, + 0, + 0, + 0, + 0, + 0, + 0, + 21529, + 21530, + 0, + 0, + 21531, + 0, + 0, + 21533, + 0, + 0, + 21539, + 21564, + 0, + 21567, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21575, + 0, + 0, + 0, + 0, + 21577, + 0, + 0, + 0, + 0, + 0, + 21591, + 0, + 0, + 21604, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21605, + 0, + 21606, + 0, + 0, + 21617, + 21618, + 21619, + 21620, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21623, + 0, + 0, + 0, + 0, + 21631, + 0, + 21635, + 0, + 0, + 0, + 0, + 21639, + 21646, + 21653, + 21662, + 0, + 0, + 21663, + 21664, + 0, + 21666, + 0, + 0, + 21667, + 0, + 21670, + 21672, + 21673, + 0, + 21674, + 21683, + 0, + 0, + 0, + 0, + 0, + 21684, + 0, + 21694, + 0, + 0, + 0, + 0, + 21695, + 21700, + 0, + 21703, + 0, + 21704, + 0, + 0, + 21709, + 0, + 0, + 0, + 21710, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21711, + 0, + 0, + 0, + 21712, + 0, + 21717, + 0, + 21730, + 0, + 0, + 0, + 21731, + 21733, + 0, + 0, + 0, + 0, + 21737, + 21741, + 21742, + 0, + 21747, + 0, + 0, + 0, + 21749, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21750, + 0, + 0, + 0, + 0, + 0, + 21752, + 0, + 0, + 0, + 0, + 21753, + 0, + 0, + 0, + 0, + 0, + 0, + 21755, + 21756, + 0, + 21757, + 0, + 0, + 0, + 0, + 0, + 0, + 21760, + 0, + 0, + 21763, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21764, + 0, + 0, + 21766, + 0, + 0, + 21767, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21773, + 0, + 21774, + 0, + 0, + 21775, + 0, + 0, + 0, + 0, + 21776, + 0, + 0, + 21777, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21780, + 21787, + 21788, + 21791, + 0, + 0, + 0, + 21797, + 0, + 0, + 0, + 0, + 0, + 21805, + 0, + 0, + 0, + 0, + 21806, + 0, + 21807, + 21809, + 0, + 21810, + 21811, + 0, + 21817, + 21819, + 21820, + 0, + 21823, + 0, + 21824, + 0, + 0, + 21825, + 0, + 0, + 21826, + 21832, + 0, + 0, + 0, + 0, + 0, + 21833, + 21848, + 21849, + 0, + 0, + 21867, + 21870, + 21871, + 21873, + 0, + 0, + 0, + 21874, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21875, + 0, + 21878, + 0, + 0, + 0, + 21879, + 0, + 21881, + 21886, + 0, + 0, + 0, + 0, + 21887, + 0, + 0, + 21888, + 21894, + 21895, + 21897, + 0, + 21901, + 0, + 21904, + 0, + 0, + 21906, + 0, + 0, + 0, + 21909, + 21910, + 21911, + 0, + 0, + 21912, + 0, + 0, + 21913, + 21914, + 21915, + 0, + 21919, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21921, + 0, + 0, + 21922, + 21933, + 21939, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21944, + 0, + 0, + 0, + 0, + 0, + 21945, + 0, + 21947, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21949, + 0, + 0, + 0, + 21950, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21951, + 0, + 21952, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21954, + 21957, + 0, + 0, + 0, + 0, + 21958, + 0, + 21959, + 0, + 0, + 0, + 0, + 0, + 0, + 21962, + 21963, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 21964, + 21965, + 0, + 0, + 21969, + 21970, + 0, + 0, + 0, + 21974, + 0, + 0, + 21980, + 21981, + 0, + 21982, + 0, + 0, + 0, + 0, + 0, + 21985, + 0, + 21988, + 0, + 21992, + 0, + 21999, + 0, + 0, + 0, + 0, + 0, + 0, + 22001, + 0, + 22002, + 0, + 0, + 0, + 0, + 0, + 0, + 22003, + 0, + 0, + 0, + 0, + 0, + 22004, + 0, + 0, + 0, + 22008, + 0, + 22009, + 22015, + 0, + 0, + 22016, + 0, + 0, + 0, + 22017, + 22019, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22020, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22021, + 22037, + 0, + 22039, + 0, + 0, + 0, + 22040, + 0, + 0, + 0, + 22048, + 22049, + 0, + 0, + 22053, + 22055, + 22056, + 22059, + 0, + 0, + 22060, + 22061, + 0, + 0, + 22064, + 0, + 0, + 0, + 0, + 22066, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22073, + 0, + 0, + 0, + 22074, + 22075, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22076, + 0, + 0, + 0, + 0, + 22077, + 22084, + 22099, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22104, + 0, + 0, + 22107, + 0, + 22108, + 0, + 22109, + 0, + 22110, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22111, + 22119, + 0, + 22120, + 22122, + 0, + 0, + 0, + 0, + 22125, + 0, + 0, + 0, + 22128, + 22129, + 0, + 0, + 0, + 0, + 0, + 0, + 22141, + 0, + 0, + 0, + 22142, + 0, + 0, + 22144, + 22146, + 0, + 22148, + 22149, + 22151, + 22154, + 0, + 0, + 0, + 22162, + 0, + 0, + 0, + 0, + 22164, + 22177, + 0, + 0, + 0, + 0, + 22179, + 0, + 22182, + 22183, + 0, + 0, + 22184, + 22188, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22190, + 0, + 22194, + 22201, + 0, + 0, + 22208, + 0, + 22209, + 0, + 22212, + 0, + 0, + 22215, + 0, + 22223, + 22231, + 0, + 0, + 22232, + 0, + 22234, + 0, + 0, + 22235, + 22236, + 0, + 22237, + 0, + 22240, + 0, + 0, + 0, + 0, + 0, + 22241, + 0, + 0, + 0, + 22242, + 22246, + 22247, + 0, + 0, + 0, + 22259, + 22268, + 0, + 22269, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22270, + 0, + 0, + 0, + 0, + 22271, + 0, + 22272, + 0, + 22277, + 0, + 0, + 0, + 0, + 0, + 22278, + 22280, + 22283, + 22286, + 0, + 0, + 22287, + 22289, + 0, + 0, + 22290, + 0, + 22293, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22295, + 0, + 22301, + 22302, + 0, + 0, + 0, + 22305, + 0, + 22308, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22315, + 0, + 0, + 0, + 22317, + 0, + 22334, + 0, + 0, + 0, + 22335, + 0, + 0, + 0, + 0, + 0, + 22336, + 0, + 22338, + 22344, + 0, + 22347, + 22349, + 0, + 22350, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22357, + 0, + 0, + 0, + 0, + 0, + 22358, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22359, + 22360, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22361, + 22366, + 0, + 0, + 22369, + 0, + 22370, + 22373, + 0, + 0, + 0, + 0, + 0, + 22375, + 0, + 22377, + 0, + 0, + 0, + 0, + 0, + 22378, + 0, + 0, + 0, + 0, + 22381, + 0, + 0, + 0, + 0, + 22382, + 0, + 22383, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22391, + 0, + 0, + 22392, + 22395, + 22396, + 22402, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22405, + 0, + 0, + 22406, + 0, + 0, + 22408, + 0, + 0, + 22409, + 22410, + 0, + 0, + 0, + 0, + 0, + 0, + 22424, + 0, + 0, + 0, + 0, + 22426, + 0, + 0, + 0, + 22427, + 0, + 22428, + 0, + 22432, + 0, + 22435, + 22442, + 22443, + 0, + 0, + 0, + 0, + 22444, + 0, + 0, + 0, + 0, + 0, + 22446, + 0, + 22454, + 0, + 22455, + 0, + 0, + 0, + 22465, + 0, + 22470, + 0, + 22471, + 0, + 0, + 0, + 0, + 22472, + 22473, + 0, + 22487, + 0, + 0, + 0, + 22488, + 0, + 0, + 0, + 0, + 22489, + 0, + 0, + 22499, + 0, + 0, + 0, + 0, + 0, + 0, + 22514, + 0, + 0, + 22515, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22516, + 0, + 0, + 0, + 22517, + 22520, + 0, + 0, + 0, + 22534, + 0, + 0, + 22535, + 0, + 0, + 22536, + 0, + 22540, + 22553, + 0, + 22555, + 0, + 0, + 0, + 0, + 22561, + 0, + 0, + 22562, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22566, + 0, + 0, + 0, + 0, + 22567, + 22568, + 0, + 0, + 22575, + 0, + 22579, + 0, + 22582, + 22583, + 22585, + 0, + 0, + 0, + 0, + 0, + 22586, + 0, + 0, + 22587, + 0, + 0, + 22590, + 0, + 0, + 0, + 0, + 0, + 22591, + 0, + 22592, + 0, + 0, + 0, + 0, + 0, + 22593, + 0, + 22602, + 0, + 0, + 22604, + 0, + 0, + 22609, + 0, + 0, + 22618, + 0, + 0, + 0, + 0, + 0, + 0, + 22619, + 0, + 22624, + 22625, + 0, + 0, + 22638, + 0, + 0, + 0, + 0, + 0, + 22639, + 0, + 0, + 22640, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22644, + 0, + 22645, + 22647, + 0, + 0, + 0, + 0, + 22652, + 22653, + 0, + 0, + 0, + 22654, + 0, + 22655, + 0, + 0, + 0, + 22656, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22673, + 22675, + 22676, + 0, + 0, + 22678, + 22679, + 0, + 22691, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22693, + 0, + 0, + 22696, + 0, + 22699, + 22707, + 22708, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22718, + 0, + 22719, + 0, + 0, + 0, + 0, + 22723, + 0, + 0, + 0, + 22724, + 22725, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22726, + 22728, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22729, + 0, + 0, + 22731, + 0, + 0, + 0, + 0, + 22732, + 22735, + 22736, + 0, + 0, + 0, + 0, + 22739, + 0, + 22749, + 0, + 0, + 22751, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22758, + 0, + 0, + 0, + 0, + 0, + 22760, + 0, + 0, + 0, + 0, + 0, + 22764, + 22765, + 22766, + 0, + 22768, + 0, + 0, + 0, + 0, + 0, + 22769, + 22770, + 0, + 0, + 0, + 0, + 0, + 0, + 22771, + 0, + 0, + 22772, + 22775, + 0, + 22776, + 22777, + 22780, + 0, + 0, + 22782, + 22784, + 0, + 22787, + 0, + 22789, + 22796, + 0, + 0, + 0, + 0, + 0, + 22798, + 0, + 0, + 0, + 0, + 0, + 0, + 22802, + 0, + 22803, + 22804, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22805, + 0, + 0, + 22810, + 22811, + 22814, + 22816, + 0, + 22825, + 22826, + 0, + 22831, + 22833, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22834, + 0, + 22836, + 22838, + 0, + 22839, + 0, + 0, + 0, + 0, + 0, + 22840, + 0, + 22847, + 0, + 0, + 0, + 0, + 0, + 22856, + 22857, + 0, + 22858, + 22859, + 0, + 0, + 22862, + 0, + 0, + 22864, + 0, + 0, + 0, + 0, + 22865, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22866, + 0, + 22867, + 22868, + 0, + 0, + 0, + 0, + 22869, + 0, + 22871, + 0, + 22872, + 0, + 22873, + 22881, + 22882, + 22884, + 22885, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22886, + 22887, + 0, + 22894, + 0, + 22895, + 0, + 0, + 0, + 22900, + 0, + 22901, + 0, + 0, + 0, + 0, + 22904, + 0, + 0, + 0, + 0, + 22905, + 22907, + 0, + 0, + 0, + 22915, + 22917, + 0, + 0, + 22918, + 0, + 0, + 0, + 22920, + 0, + 0, + 0, + 22929, + 22930, + 0, + 0, + 0, + 22941, + 22942, + 0, + 0, + 0, + 22943, + 0, + 0, + 0, + 22944, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22946, + 0, + 22947, + 0, + 0, + 22954, + 0, + 22956, + 0, + 0, + 22962, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22963, + 0, + 0, + 22964, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 22965, + 0, + 22968, + 0, + 0, + 0, + 22969, + 0, + 0, + 0, + 0, + 0, + 22970, + 0, + 22971, + 0, + 0, + 0, + 0, + 0, + 22978, + 0, + 0, + 22979, + 0, + 22987, + 0, + 0, + 22989, + 0, + 0, + 0, + 0, + 0, + 0, + 22990, + 0, + 23005, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23006, + 23007, + 23008, + 0, + 0, + 23023, + 23024, + 23029, + 0, + 0, + 0, + 0, + 23030, + 0, + 0, + 0, + 0, + 0, + 23032, + 0, + 0, + 0, + 0, + 0, + 23035, + 0, + 0, + 0, + 0, + 23038, + 0, + 0, + 0, + 23048, + 0, + 23049, + 23052, + 23053, + 23060, + 23061, + 0, + 23063, + 0, + 0, + 0, + 0, + 23067, + 23068, + 0, + 0, + 0, + 23069, + 23073, + 0, + 0, + 0, + 23127, + 0, + 23128, + 0, + 0, + 0, + 0, + 0, + 23129, + 0, + 23138, + 23141, + 0, + 23149, + 0, + 0, + 23150, + 0, + 0, + 0, + 23152, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23154, + 0, + 0, + 0, + 0, + 23157, + 23159, + 23160, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23180, + 0, + 0, + 0, + 0, + 23181, + 0, + 0, + 23188, + 0, + 23189, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23195, + 0, + 0, + 23196, + 23199, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23202, + 0, + 23204, + 0, + 23207, + 0, + 23209, + 23210, + 0, + 0, + 0, + 0, + 0, + 0, + 23227, + 23229, + 0, + 0, + 23230, + 23234, + 23238, + 0, + 0, + 0, + 23245, + 23246, + 23248, + 0, + 0, + 0, + 0, + 23249, + 23254, + 0, + 0, + 0, + 23265, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23268, + 0, + 23276, + 0, + 0, + 0, + 0, + 23277, + 0, + 23297, + 0, + 23298, + 0, + 0, + 0, + 0, + 23299, + 0, + 23302, + 0, + 0, + 23303, + 23312, + 0, + 0, + 23314, + 0, + 23320, + 0, + 0, + 0, + 0, + 23324, + 0, + 23325, + 0, + 23328, + 0, + 23334, + 0, + 0, + 0, + 23337, + 0, + 0, + 0, + 0, + 23343, + 23344, + 23346, + 0, + 23348, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23353, + 0, + 0, + 0, + 0, + 23355, + 0, + 23356, + 23358, + 0, + 0, + 0, + 23359, + 23360, + 0, + 23361, + 0, + 23367, + 0, + 23369, + 0, + 0, + 23373, + 0, + 23378, + 23379, + 0, + 23382, + 23383, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23387, + 0, + 0, + 0, + 0, + 0, + 0, + 23388, + 23390, + 0, + 0, + 23393, + 23398, + 0, + 0, + 0, + 23399, + 0, + 0, + 0, + 23400, + 0, + 0, + 0, + 0, + 23401, + 0, + 0, + 0, + 23415, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23416, + 0, + 23422, + 0, + 23443, + 23444, + 0, + 0, + 0, + 0, + 23448, + 0, + 23454, + 0, + 0, + 0, + 0, + 0, + 0, + 23456, + 0, + 0, + 23458, + 23464, + 0, + 0, + 0, + 0, + 0, + 0, + 23465, + 0, + 0, + 0, + 23470, + 23471, + 0, + 0, + 23472, + 0, + 0, + 0, + 23473, + 23496, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23497, + 0, + 23499, + 0, + 0, + 23502, + 0, + 0, + 23503, + 0, + 0, + 23513, + 0, + 0, + 23515, + 0, + 0, + 0, + 23517, + 0, + 0, + 0, + 0, + 23518, + 23519, + 23521, + 23524, + 0, + 23525, + 23528, + 23539, + 0, + 0, + 0, + 0, + 0, + 23541, + 0, + 0, + 23544, + 0, + 0, + 23556, + 0, + 0, + 23557, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23559, + 0, + 23560, + 0, + 0, + 23561, + 0, + 0, + 23566, + 0, + 0, + 0, + 0, + 0, + 23568, + 23569, + 23570, + 0, + 0, + 0, + 0, + 23571, + 0, + 23574, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23575, + 0, + 23579, + 0, + 0, + 23581, + 0, + 0, + 0, + 0, + 0, + 0, + 23587, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23596, + 23598, + 0, + 0, + 0, + 0, + 23602, + 23606, + 0, + 0, + 23607, + 0, + 23608, + 0, + 0, + 0, + 23614, + 23616, + 0, + 0, + 0, + 0, + 0, + 23618, + 0, + 0, + 23619, + 0, + 0, + 0, + 0, + 23621, + 23626, + 0, + 23627, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23629, + 0, + 23630, + 0, + 0, + 0, + 0, + 23634, + 0, + 23636, + 0, + 0, + 0, + 0, + 0, + 0, + 23638, + 0, + 0, + 0, + 0, + 23640, + 23667, + 0, + 23669, + 0, + 0, + 0, + 23681, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23682, + 0, + 23683, + 0, + 0, + 0, + 0, + 0, + 23684, + 0, + 0, + 0, + 23685, + 23689, + 0, + 23693, + 23694, + 23700, + 0, + 23702, + 0, + 23709, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23712, + 0, + 0, + 0, + 0, + 0, + 23714, + 0, + 0, + 23715, + 0, + 0, + 0, + 0, + 23718, + 0, + 0, + 23720, + 0, + 0, + 0, + 0, + 23722, + 0, + 0, + 0, + 23726, + 23729, + 0, + 23741, + 23746, + 0, + 23748, + 0, + 0, + 0, + 0, + 23749, + 0, + 0, + 0, + 0, + 0, + 23750, + 0, + 0, + 0, + 0, + 23751, + 0, + 23753, + 0, + 0, + 0, + 0, + 23757, + 23765, + 0, + 0, + 0, + 23770, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23771, + 0, + 23772, + 23781, + 0, + 0, + 23796, + 0, + 0, + 0, + 0, + 23798, + 0, + 23799, + 0, + 0, + 0, + 23802, + 0, + 0, + 23806, + 0, + 23807, + 0, + 0, + 23808, + 0, + 23809, + 0, + 23819, + 0, + 0, + 0, + 23821, + 0, + 23827, + 0, + 0, + 0, + 23829, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23830, + 0, + 0, + 0, + 0, + 0, + 0, + 23832, + 23833, + 23834, + 23835, + 0, + 0, + 0, + 0, + 23837, + 23838, + 0, + 0, + 0, + 0, + 0, + 23846, + 0, + 0, + 0, + 0, + 0, + 0, + 23847, + 0, + 0, + 0, + 0, + 0, + 23879, + 23881, + 0, + 0, + 23882, + 23883, + 23895, + 0, + 23899, + 0, + 0, + 0, + 0, + 23901, + 0, + 0, + 0, + 0, + 0, + 0, + 23902, + 0, + 0, + 0, + 0, + 0, + 23903, + 23905, + 0, + 23906, + 0, + 23907, + 23918, + 23919, + 23920, + 0, + 23922, + 0, + 23924, + 0, + 23927, + 0, + 23934, + 0, + 23937, + 23941, + 0, + 23942, + 23946, + 0, + 0, + 0, + 0, + 0, + 23955, + 23956, + 23958, + 0, + 0, + 0, + 0, + 0, + 0, + 23959, + 0, + 23962, + 23965, + 0, + 23966, + 0, + 0, + 0, + 0, + 23967, + 23968, + 0, + 0, + 23973, + 0, + 0, + 23974, + 0, + 0, + 0, + 0, + 23975, + 0, + 23976, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23977, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23980, + 0, + 0, + 23984, + 0, + 23985, + 0, + 0, + 23987, + 0, + 0, + 23988, + 23990, + 23991, + 0, + 0, + 0, + 0, + 0, + 0, + 23992, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23994, + 0, + 0, + 0, + 23998, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 23999, + 0, + 0, + 24003, + 0, + 24004, + 0, + 24006, + 0, + 0, + 0, + 24007, + 0, + 0, + 24008, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24009, + 0, + 0, + 24010, + 0, + 0, + 24011, + 0, + 0, + 24013, + 24014, + 0, + 0, + 24015, + 24016, + 24027, + 0, + 24028, + 24029, + 0, + 24030, + 0, + 0, + 0, + 0, + 0, + 24033, + 24034, + 0, + 24035, + 0, + 0, + 24036, + 0, + 0, + 24044, + 0, + 24048, + 24049, + 24063, + 24067, + 0, + 24068, + 24070, + 0, + 0, + 24071, + 24078, + 24087, + 0, + 24090, + 0, + 0, + 0, + 24095, + 0, + 24098, + 24101, + 24104, + 24106, + 0, + 24107, + 0, + 0, + 0, + 24108, + 0, + 0, + 0, + 0, + 24110, + 24111, + 0, + 24113, + 0, + 0, + 24115, + 24120, + 0, + 0, + 0, + 0, + 0, + 0, + 24124, + 0, + 24125, + 0, + 24126, + 0, + 24127, + 0, + 0, + 0, + 0, + 0, + 24135, + 0, + 0, + 24136, + 0, + 24137, + 24142, + 0, + 0, + 0, + 24146, + 0, + 0, + 24147, + 24149, + 24154, + 0, + 24163, + 0, + 0, + 0, + 24165, + 24166, + 24167, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24169, + 24170, + 24175, + 0, + 0, + 0, + 24178, + 0, + 0, + 24179, + 0, + 0, + 24181, + 0, + 24184, + 24197, + 0, + 24201, + 24204, + 0, + 0, + 0, + 0, + 0, + 0, + 24206, + 24212, + 24220, + 0, + 0, + 0, + 24224, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24226, + 0, + 24234, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24235, + 0, + 24236, + 0, + 0, + 0, + 0, + 0, + 24239, + 24240, + 24241, + 0, + 0, + 24248, + 0, + 0, + 24249, + 0, + 24251, + 0, + 0, + 0, + 0, + 0, + 0, + 24253, + 0, + 24268, + 0, + 0, + 0, + 24269, + 0, + 24271, + 24272, + 0, + 0, + 0, + 0, + 24273, + 0, + 0, + 24274, + 0, + 0, + 24279, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24280, + 0, + 24293, + 24294, + 0, + 0, + 0, + 0, + 0, + 0, + 24296, + 0, + 0, + 24323, + 0, + 0, + 0, + 24329, + 24330, + 24331, + 24339, + 0, + 24351, + 0, + 0, + 24369, + 24370, + 0, + 0, + 0, + 24371, + 0, + 0, + 0, + 0, + 24372, + 24373, + 24374, + 0, + 0, + 0, + 0, + 0, + 24378, + 0, + 0, + 0, + 0, + 24379, + 0, + 24381, + 0, + 24383, + 24389, + 0, + 24390, + 0, + 0, + 24394, + 24395, + 24400, + 0, + 0, + 0, + 24401, + 24402, + 0, + 24406, + 0, + 0, + 0, + 24411, + 0, + 0, + 0, + 24415, + 0, + 24416, + 0, + 0, + 0, + 0, + 0, + 24417, + 0, + 24419, + 0, + 24422, + 0, + 24423, + 24428, + 0, + 24435, + 0, + 0, + 0, + 24439, + 0, + 0, + 0, + 24440, + 24442, + 24446, + 0, + 0, + 0, + 24447, + 24448, + 24449, + 24452, + 0, + 0, + 0, + 0, + 24453, + 24457, + 0, + 0, + 24458, + 24459, + 24460, + 0, + 24465, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24470, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24471, + 0, + 24473, + 24474, + 24475, + 24476, + 0, + 24478, + 0, + 0, + 0, + 0, + 24480, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24481, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24482, + 24485, + 0, + 0, + 0, + 0, + 24486, + 0, + 0, + 0, + 24488, + 0, + 0, + 0, + 24494, + 0, + 0, + 0, + 0, + 24497, + 0, + 0, + 24498, + 0, + 0, + 0, + 24499, + 24506, + 0, + 0, + 0, + 24507, + 0, + 0, + 24511, + 0, + 0, + 24513, + 24514, + 0, + 0, + 0, + 0, + 0, + 24517, + 0, + 24518, + 0, + 24520, + 0, + 24521, + 24524, + 24525, + 0, + 0, + 0, + 0, + 0, + 24527, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24528, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24537, + 24539, + 0, + 24540, + 0, + 0, + 0, + 24548, + 0, + 0, + 0, + 0, + 0, + 24549, + 24550, + 0, + 0, + 0, + 24553, + 24554, + 0, + 24555, + 0, + 24556, + 0, + 24558, + 0, + 0, + 0, + 0, + 0, + 24560, + 0, + 0, + 0, + 24561, + 0, + 0, + 0, + 0, + 0, + 24562, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24567, + 0, + 0, + 0, + 0, + 0, + 24569, + 0, + 0, + 0, + 24574, + 0, + 24575, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24577, + 24581, + 0, + 24584, + 0, + 0, + 0, + 0, + 0, + 24585, + 0, + 0, + 0, + 0, + 0, + 24586, + 0, + 0, + 24587, + 0, + 24588, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24590, + 24591, + 0, + 0, + 0, + 0, + 24592, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24594, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24596, + 24597, + 0, + 0, + 0, + 0, + 24602, + 24603, + 0, + 0, + 0, + 0, + 24604, + 0, + 0, + 24605, + 0, + 24610, + 0, + 0, + 24611, + 0, + 0, + 0, + 0, + 24612, + 24615, + 24616, + 24624, + 0, + 0, + 0, + 24627, + 0, + 24638, + 24639, + 0, + 0, + 0, + 0, + 24640, + 0, + 0, + 0, + 24655, + 24656, + 24657, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24662, + 0, + 24663, + 24664, + 0, + 0, + 0, + 0, + 0, + 24665, + 0, + 0, + 0, + 0, + 24667, + 0, + 0, + 0, + 0, + 0, + 0, + 24668, + 24669, + 0, + 24670, + 24674, + 0, + 0, + 0, + 24675, + 0, + 24678, + 0, + 0, + 24679, + 0, + 0, + 0, + 24681, + 0, + 24683, + 0, + 0, + 0, + 0, + 24684, + 0, + 24685, + 0, + 0, + 24686, + 0, + 0, + 24688, + 24689, + 0, + 0, + 0, + 0, + 24690, + 24691, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24697, + 0, + 24698, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24709, + 0, + 0, + 0, + 0, + 0, + 24710, + 0, + 24712, + 0, + 0, + 0, + 0, + 0, + 0, + 24713, + 24714, + 0, + 24715, + 0, + 24716, + 24718, + 0, + 24719, + 0, + 0, + 0, + 0, + 24720, + 0, + 0, + 24725, + 0, + 0, + 24738, + 0, + 24749, + 24750, + 0, + 0, + 0, + 24752, + 0, + 0, + 0, + 24753, + 0, + 0, + 0, + 24758, + 0, + 0, + 0, + 0, + 0, + 24762, + 0, + 24763, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24764, + 0, + 0, + 0, + 0, + 0, + 24765, + 24767, + 24768, + 0, + 24772, + 0, + 0, + 0, + 0, + 24773, + 0, + 0, + 0, + 0, + 24777, + 0, + 0, + 0, + 0, + 0, + 24785, + 0, + 24786, + 24788, + 0, + 0, + 0, + 24789, + 0, + 0, + 0, + 0, + 24794, + 24798, + 0, + 24799, + 24800, + 0, + 0, + 0, + 24803, + 0, + 24804, + 24806, + 0, + 24807, + 0, + 0, + 0, + 24810, + 0, + 0, + 0, + 0, + 0, + 0, + 24827, + 24828, + 0, + 24835, + 0, + 0, + 0, + 0, + 0, + 0, + 24836, + 0, + 0, + 0, + 0, + 0, + 24839, + 0, + 24843, + 24844, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24847, + 0, + 0, + 24848, + 0, + 0, + 0, + 0, + 0, + 0, + 24849, + 0, + 24850, + 24851, + 0, + 0, + 0, + 24852, + 0, + 24853, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24854, + 0, + 24855, + 0, + 0, + 24868, + 0, + 0, + 0, + 24883, + 0, + 0, + 0, + 24884, + 0, + 24895, + 24897, + 0, + 0, + 0, + 0, + 0, + 24899, + 0, + 0, + 0, + 0, + 0, + 24900, + 0, + 24913, + 0, + 0, + 0, + 0, + 0, + 0, + 24914, + 0, + 0, + 24917, + 24930, + 24931, + 0, + 0, + 0, + 24932, + 0, + 0, + 24939, + 0, + 0, + 24942, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 24945, + 24950, + 0, + 24951, + 0, + 0, + 24953, + 0, + 0, + 0, + 24954, + 0, + 24959, + 0, + 0, + 0, + 24961, + 0, + 0, + 24962, + 0, + 24964, + 24968, + 24970, + 24972, + 0, + 0, + 0, + 0, + 0, + 24976, + 0, + 0, + 0, + 24977, + 0, + 24982, + 0, + 0, + 24983, + 0, + 0, + 24984, + 0, + 0, + 0, + 24993, + 0, + 0, + 0, + 24994, + 0, + 0, + 25001, + 0, + 0, + 0, + 25003, + 0, + 0, + 25018, + 0, + 0, + 25023, + 0, + 0, + 0, + 25034, + 0, + 0, + 25035, + 25036, + 0, + 25037, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25039, + 0, + 0, + 0, + 0, + 0, + 25040, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25042, + 0, + 0, + 25043, + 25045, + 0, + 0, + 0, + 0, + 0, + 0, + 25049, + 0, + 0, + 25051, + 0, + 25052, + 25053, + 0, + 0, + 25054, + 0, + 0, + 0, + 25055, + 0, + 0, + 0, + 0, + 25057, + 25059, + 0, + 0, + 25060, + 25064, + 0, + 25065, + 25069, + 25070, + 0, + 0, + 0, + 0, + 25072, + 0, + 25073, + 0, + 25090, + 0, + 0, + 25092, + 25093, + 25101, + 0, + 0, + 0, + 0, + 0, + 0, + 25105, + 25108, + 0, + 0, + 25113, + 0, + 0, + 25115, + 25116, + 0, + 0, + 0, + 0, + 0, + 0, + 25117, + 0, + 0, + 0, + 25120, + 25121, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25125, + 0, + 0, + 0, + 25126, + 0, + 25130, + 25134, + 0, + 25139, + 0, + 25143, + 0, + 0, + 0, + 25151, + 0, + 25161, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25163, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25174, + 0, + 25175, + 0, + 25207, + 0, + 0, + 0, + 25209, + 0, + 0, + 0, + 0, + 25213, + 0, + 25219, + 0, + 25223, + 0, + 25225, + 0, + 0, + 0, + 25227, + 0, + 0, + 0, + 25228, + 0, + 0, + 0, + 25229, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25231, + 25233, + 0, + 0, + 0, + 0, + 25237, + 25239, + 0, + 0, + 0, + 25243, + 0, + 0, + 0, + 25252, + 0, + 25257, + 25258, + 0, + 0, + 0, + 0, + 25260, + 25265, + 0, + 25268, + 0, + 0, + 25273, + 25324, + 0, + 25325, + 0, + 25326, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25327, + 0, + 0, + 0, + 0, + 0, + 25328, + 0, + 0, + 0, + 0, + 0, + 0, + 25332, + 0, + 0, + 0, + 25333, + 0, + 0, + 0, + 25336, + 25337, + 25338, + 0, + 0, + 25343, + 0, + 25350, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25352, + 0, + 25354, + 0, + 25375, + 0, + 25379, + 0, + 0, + 0, + 0, + 25384, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25386, + 0, + 25388, + 0, + 25390, + 0, + 0, + 25399, + 0, + 0, + 25401, + 0, + 0, + 0, + 25402, + 0, + 0, + 0, + 25407, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25413, + 25415, + 0, + 0, + 25417, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25419, + 0, + 0, + 0, + 25421, + 0, + 0, + 0, + 25424, + 0, + 0, + 0, + 0, + 25433, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25435, + 0, + 0, + 0, + 0, + 0, + 0, + 25436, + 0, + 0, + 0, + 25437, + 0, + 0, + 25440, + 0, + 0, + 0, + 0, + 0, + 0, + 25442, + 0, + 0, + 25443, + 0, + 25446, + 0, + 0, + 25449, + 0, + 0, + 0, + 25450, + 0, + 0, + 0, + 0, + 25452, + 0, + 25453, + 25454, + 25455, + 0, + 0, + 0, + 25456, + 0, + 25457, + 0, + 0, + 0, + 25459, + 0, + 25461, + 0, + 25468, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25469, + 0, + 0, + 0, + 0, + 0, + 25471, + 0, + 0, + 0, + 0, + 0, + 25474, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25475, + 0, + 0, + 0, + 0, + 25477, + 0, + 0, + 0, + 0, + 25483, + 0, + 0, + 0, + 0, + 0, + 25484, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25485, + 0, + 25497, + 0, + 0, + 25498, + 0, + 25504, + 0, + 25510, + 0, + 25512, + 0, + 0, + 25513, + 25514, + 0, + 0, + 0, + 0, + 0, + 0, + 25517, + 25518, + 25519, + 0, + 25520, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25521, + 0, + 25522, + 25527, + 25534, + 0, + 25536, + 0, + 25537, + 0, + 0, + 25548, + 25550, + 0, + 0, + 25551, + 0, + 25552, + 0, + 0, + 0, + 0, + 0, + 25554, + 0, + 25555, + 0, + 25556, + 25557, + 25568, + 0, + 0, + 0, + 25570, + 25571, + 0, + 0, + 0, + 0, + 0, + 0, + 25574, + 0, + 0, + 0, + 0, + 25579, + 0, + 0, + 0, + 25581, + 0, + 0, + 0, + 25582, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25588, + 0, + 0, + 0, + 0, + 25589, + 0, + 0, + 0, + 0, + 25590, + 0, + 25591, + 25592, + 25593, + 0, + 25594, + 0, + 0, + 0, + 25596, + 0, + 25597, + 25615, + 0, + 0, + 0, + 0, + 0, + 25618, + 0, + 0, + 0, + 0, + 25619, + 25623, + 0, + 0, + 25629, + 0, + 0, + 25631, + 0, + 0, + 0, + 25635, + 25636, + 0, + 0, + 25649, + 0, + 0, + 0, + 0, + 25654, + 0, + 0, + 0, + 25661, + 25663, + 0, + 0, + 25671, + 0, + 0, + 25678, + 25698, + 0, + 25699, + 25702, + 25703, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25704, + 0, + 0, + 0, + 0, + 0, + 25706, + 0, + 0, + 25710, + 0, + 25711, + 0, + 25712, + 0, + 25715, + 25716, + 25717, + 0, + 0, + 25718, + 25728, + 25732, + 0, + 0, + 0, + 25734, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25737, + 0, + 0, + 25739, + 0, + 0, + 0, + 25740, + 0, + 25741, + 25745, + 0, + 25746, + 0, + 25748, + 25772, + 25778, + 0, + 0, + 0, + 0, + 0, + 25780, + 0, + 0, + 0, + 0, + 25781, + 0, + 25782, + 25784, + 25785, + 0, + 0, + 0, + 25789, + 0, + 0, + 0, + 0, + 0, + 0, + 25797, + 25801, + 0, + 0, + 0, + 25808, + 25809, + 0, + 0, + 25811, + 25814, + 25815, + 0, + 0, + 25817, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25820, + 0, + 0, + 0, + 0, + 25832, + 25833, + 0, + 0, + 0, + 25846, + 0, + 0, + 0, + 25847, + 25848, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25849, + 25850, + 0, + 0, + 25851, + 0, + 0, + 25852, + 0, + 25862, + 0, + 0, + 0, + 25863, + 25865, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25867, + 25868, + 0, + 25869, + 25874, + 0, + 25875, + 0, + 25876, + 25877, + 0, + 0, + 0, + 0, + 25878, + 25902, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25903, + 25904, + 25905, + 0, + 0, + 0, + 25908, + 25909, + 0, + 0, + 0, + 0, + 25910, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25912, + 0, + 25913, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25914, + 0, + 0, + 25916, + 0, + 0, + 0, + 0, + 0, + 25917, + 25927, + 0, + 0, + 0, + 0, + 25928, + 0, + 0, + 25930, + 0, + 0, + 0, + 25933, + 0, + 0, + 25938, + 25942, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25945, + 0, + 25950, + 0, + 25956, + 0, + 0, + 25961, + 25962, + 0, + 0, + 25963, + 0, + 25964, + 25965, + 25966, + 0, + 0, + 0, + 0, + 0, + 25967, + 0, + 0, + 0, + 0, + 25968, + 0, + 0, + 0, + 25969, + 25971, + 0, + 0, + 0, + 0, + 0, + 25973, + 25975, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25978, + 0, + 25981, + 0, + 0, + 0, + 25982, + 0, + 0, + 0, + 25984, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 25993, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26002, + 0, + 0, + 0, + 26005, + 0, + 0, + 0, + 26006, + 26007, + 0, + 0, + 26014, + 26015, + 26016, + 0, + 0, + 0, + 0, + 0, + 0, + 26017, + 26018, + 26020, + 0, + 26022, + 26023, + 0, + 0, + 0, + 26024, + 26028, + 0, + 26029, + 26033, + 26034, + 26044, + 0, + 0, + 0, + 0, + 0, + 26046, + 0, + 0, + 26047, + 0, + 0, + 26049, + 0, + 26050, + 0, + 26051, + 0, + 0, + 0, + 0, + 0, + 26053, + 0, + 0, + 0, + 0, + 26054, + 26059, + 0, + 0, + 0, + 0, + 0, + 0, + 26060, + 0, + 26066, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26067, + 0, + 26069, + 0, + 0, + 26071, + 0, + 0, + 0, + 26073, + 0, + 26074, + 26077, + 0, + 0, + 0, + 0, + 26078, + 0, + 0, + 0, + 26079, + 0, + 26090, + 0, + 0, + 26094, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26095, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26096, + 26101, + 0, + 26107, + 26122, + 0, + 26124, + 0, + 0, + 26125, + 0, + 0, + 0, + 0, + 0, + 0, + 26136, + 26141, + 26155, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26164, + 26166, + 0, + 0, + 0, + 26167, + 0, + 26170, + 26171, + 0, + 0, + 26172, + 0, + 0, + 26174, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26175, + 0, + 0, + 0, + 26176, + 26177, + 0, + 26321, + 26322, + 0, + 26323, + 0, + 0, + 26324, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26325, + 0, + 26331, + 0, + 0, + 0, + 0, + 0, + 0, + 26335, + 0, + 0, + 0, + 26350, + 0, + 0, + 0, + 26379, + 0, + 0, + 26382, + 26383, + 26385, + 0, + 0, + 26392, + 26406, + 0, + 0, + 0, + 0, + 26411, + 0, + 0, + 0, + 0, + 0, + 26412, + 0, + 0, + 26420, + 0, + 0, + 26423, + 0, + 26424, + 26426, + 26432, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26435, + 0, + 26436, + 0, + 0, + 0, + 0, + 0, + 26441, + 0, + 26444, + 0, + 0, + 0, + 26446, + 0, + 0, + 0, + 0, + 26447, + 0, + 0, + 0, + 0, + 26449, + 0, + 26450, + 26452, + 0, + 26453, + 26454, + 0, + 0, + 0, + 26455, + 0, + 0, + 0, + 26456, + 0, + 0, + 26458, + 0, + 0, + 26460, + 0, + 26463, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26464, + 26470, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26473, + 0, + 0, + 26474, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26475, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26477, + 0, + 26485, + 0, + 0, + 26486, + 0, + 26487, + 0, + 0, + 26488, + 26493, + 26494, + 0, + 0, + 26495, + 0, + 26497, + 26504, + 26506, + 0, + 0, + 0, + 0, + 0, + 26507, + 0, + 0, + 0, + 0, + 0, + 26509, + 0, + 0, + 26510, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26512, + 0, + 26513, + 26515, + 0, + 0, + 0, + 26518, + 0, + 0, + 0, + 26519, + 0, + 26524, + 26526, + 0, + 0, + 0, + 26527, + 0, + 26532, + 0, + 26533, + 26537, + 26558, + 0, + 0, + 0, + 26559, + 0, + 0, + 0, + 26571, + 0, + 0, + 26573, + 0, + 26588, + 0, + 26593, + 0, + 0, + 0, + 0, + 0, + 0, + 26603, + 0, + 26604, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26606, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26607, + 26609, + 26611, + 26614, + 0, + 0, + 0, + 26616, + 26620, + 0, + 26621, + 0, + 0, + 0, + 0, + 0, + 26627, + 0, + 26629, + 0, + 0, + 26630, + 0, + 0, + 26632, + 26643, + 0, + 0, + 0, + 26644, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26646, + 26647, + 0, + 0, + 0, + 26650, + 0, + 0, + 26656, + 0, + 0, + 0, + 0, + 26663, + 26670, + 26671, + 0, + 0, + 0, + 26685, + 26686, + 26687, + 0, + 26689, + 0, + 0, + 0, + 0, + 26744, + 0, + 26745, + 0, + 26747, + 26748, + 0, + 26749, + 26750, + 26751, + 0, + 0, + 0, + 0, + 26752, + 26755, + 0, + 0, + 0, + 26756, + 26769, + 0, + 0, + 0, + 26774, + 0, + 0, + 0, + 0, + 0, + 26775, + 0, + 26777, + 26778, + 0, + 26786, + 0, + 0, + 0, + 26787, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26788, + 0, + 0, + 26789, + 0, + 0, + 0, + 0, + 0, + 26791, + 0, + 26792, + 26793, + 0, + 0, + 0, + 26794, + 0, + 26797, + 26798, + 0, + 0, + 0, + 26800, + 0, + 0, + 26803, + 0, + 26804, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26805, + 0, + 0, + 26808, + 0, + 0, + 26809, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26812, + 0, + 26825, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26826, + 0, + 0, + 26827, + 26829, + 26834, + 0, + 0, + 0, + 0, + 26835, + 0, + 0, + 26849, + 0, + 26851, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26852, + 0, + 26853, + 26857, + 0, + 26858, + 0, + 26859, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26876, + 0, + 26878, + 26882, + 26883, + 0, + 0, + 0, + 0, + 26890, + 26894, + 0, + 0, + 0, + 0, + 26895, + 26896, + 0, + 0, + 0, + 0, + 0, + 26900, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26911, + 26913, + 26914, + 26915, + 26916, + 26919, + 0, + 0, + 0, + 26921, + 26922, + 0, + 0, + 26925, + 0, + 0, + 0, + 26928, + 0, + 0, + 26929, + 26930, + 0, + 0, + 0, + 26931, + 0, + 26932, + 0, + 0, + 0, + 0, + 0, + 26933, + 0, + 0, + 0, + 0, + 0, + 0, + 26937, + 0, + 0, + 26943, + 0, + 0, + 26944, + 0, + 0, + 0, + 26946, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26956, + 0, + 26958, + 0, + 0, + 26963, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 26965, + 0, + 26969, + 26970, + 26972, + 0, + 0, + 0, + 0, + 0, + 26973, + 0, + 26974, + 0, + 26978, + 0, + 26980, + 0, + 0, + 0, + 0, + 0, + 0, + 26982, + 0, + 26986, + 26987, + 0, + 26990, + 0, + 0, + 0, + 0, + 27003, + 27006, + 0, + 0, + 27007, + 27010, + 27012, + 27013, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27014, + 27015, + 27018, + 0, + 27019, + 0, + 0, + 0, + 0, + 0, + 27025, + 0, + 0, + 0, + 27026, + 0, + 0, + 0, + 0, + 27029, + 27030, + 27031, + 27034, + 0, + 0, + 27036, + 27037, + 0, + 0, + 0, + 27038, + 27042, + 0, + 0, + 0, + 27044, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27045, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27046, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27047, + 27049, + 0, + 27050, + 0, + 0, + 0, + 27051, + 27052, + 0, + 27055, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27056, + 27058, + 27059, + 0, + 27061, + 0, + 27064, + 0, + 0, + 0, + 0, + 0, + 27069, + 0, + 0, + 27070, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27072, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27076, + 0, + 0, + 0, + 0, + 0, + 27078, + 0, + 27079, + 0, + 0, + 0, + 27081, + 0, + 0, + 0, + 0, + 0, + 0, + 27082, + 0, + 27083, + 27086, + 0, + 0, + 0, + 0, + 27087, + 0, + 0, + 0, + 0, + 0, + 27088, + 27090, + 0, + 27094, + 0, + 0, + 27095, + 0, + 27099, + 27102, + 0, + 0, + 0, + 27103, + 0, + 0, + 0, + 0, + 27105, + 0, + 0, + 0, + 27106, + 0, + 0, + 0, + 0, + 0, + 0, + 27107, + 0, + 0, + 0, + 0, + 27108, + 27117, + 0, + 0, + 0, + 0, + 27118, + 0, + 0, + 27124, + 0, + 27126, + 0, + 0, + 27130, + 27131, + 0, + 0, + 0, + 0, + 0, + 0, + 27147, + 0, + 0, + 0, + 0, + 27148, + 27149, + 0, + 0, + 0, + 0, + 27150, + 27151, + 0, + 27152, + 0, + 27159, + 0, + 0, + 0, + 27164, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27175, + 0, + 27189, + 0, + 0, + 27191, + 0, + 27193, + 0, + 27195, + 0, + 27198, + 0, + 0, + 0, + 0, + 0, + 27200, + 0, + 0, + 0, + 0, + 27202, + 0, + 0, + 0, + 0, + 27203, + 0, + 0, + 27204, + 0, + 0, + 27206, + 0, + 27207, + 0, + 0, + 0, + 0, + 27209, + 0, + 0, + 0, + 27213, + 0, + 0, + 27216, + 27219, + 27220, + 27222, + 27223, + 0, + 27224, + 0, + 27225, + 27226, + 0, + 0, + 27233, + 0, + 0, + 0, + 0, + 27235, + 0, + 27237, + 0, + 27238, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27239, + 0, + 27242, + 27243, + 0, + 27250, + 0, + 0, + 0, + 27251, + 0, + 27253, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27254, + 27255, + 27258, + 0, + 0, + 0, + 27259, + 0, + 0, + 0, + 0, + 0, + 0, + 27267, + 0, + 27276, + 27278, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27296, + 27297, + 27301, + 0, + 0, + 0, + 0, + 0, + 0, + 27302, + 0, + 0, + 0, + 0, + 0, + 0, + 27312, + 27313, + 0, + 0, + 0, + 0, + 0, + 27318, + 0, + 27320, + 0, + 27329, + 0, + 27330, + 27331, + 0, + 27332, + 0, + 0, + 0, + 0, + 27340, + 0, + 0, + 0, + 27348, + 0, + 0, + 0, + 0, + 0, + 0, + 27350, + 0, + 27351, + 0, + 0, + 0, + 0, + 27355, + 0, + 0, + 27358, + 27359, + 27361, + 0, + 0, + 0, + 27365, + 0, + 27367, + 0, + 27376, + 27378, + 0, + 0, + 27379, + 0, + 0, + 0, + 0, + 0, + 0, + 27396, + 0, + 27397, + 27404, + 0, + 0, + 0, + 0, + 0, + 27408, + 0, + 0, + 0, + 0, + 27453, + 0, + 0, + 0, + 27456, + 0, + 0, + 0, + 27458, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27459, + 0, + 0, + 0, + 27460, + 0, + 0, + 27461, + 0, + 27465, + 27467, + 0, + 0, + 27469, + 0, + 27470, + 0, + 27471, + 0, + 27477, + 27482, + 0, + 0, + 0, + 0, + 0, + 0, + 27484, + 0, + 0, + 0, + 0, + 0, + 0, + 27485, + 0, + 0, + 0, + 0, + 0, + 27493, + 0, + 27494, + 27502, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27511, + 27532, + 0, + 0, + 0, + 27533, + 27545, + 0, + 0, + 0, + 27546, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27547, + 0, + 0, + 27549, + 27550, + 0, + 27551, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27555, + 0, + 0, + 27571, + 0, + 27573, + 27574, + 27575, + 27577, + 0, + 27578, + 0, + 0, + 27579, + 27585, + 0, + 0, + 0, + 0, + 0, + 27586, + 0, + 0, + 27588, + 27589, + 0, + 0, + 0, + 0, + 27596, + 0, + 0, + 27600, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27608, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27610, + 0, + 0, + 0, + 27618, + 0, + 0, + 27620, + 0, + 0, + 0, + 27631, + 0, + 0, + 27632, + 27634, + 0, + 27636, + 27638, + 0, + 0, + 0, + 27643, + 0, + 27644, + 27649, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27651, + 27660, + 0, + 27661, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27662, + 0, + 0, + 27664, + 0, + 27665, + 0, + 0, + 0, + 27669, + 0, + 27671, + 0, + 0, + 0, + 27673, + 27674, + 0, + 0, + 0, + 27682, + 0, + 0, + 0, + 27711, + 0, + 27712, + 27713, + 27719, + 27720, + 0, + 0, + 27728, + 0, + 27729, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27731, + 0, + 0, + 27732, + 0, + 27733, + 0, + 27738, + 0, + 0, + 0, + 27742, + 0, + 0, + 0, + 27743, + 27744, + 0, + 0, + 0, + 0, + 0, + 0, + 27745, + 27746, + 0, + 0, + 0, + 27747, + 27748, + 27751, + 27752, + 0, + 0, + 0, + 27768, + 27770, + 0, + 0, + 0, + 27774, + 27775, + 0, + 27776, + 27777, + 0, + 0, + 27781, + 0, + 27784, + 0, + 27786, + 0, + 0, + 27791, + 0, + 27792, + 27793, + 27804, + 0, + 27812, + 27813, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27814, + 0, + 27825, + 0, + 27827, + 0, + 0, + 0, + 0, + 27828, + 27861, + 27862, + 0, + 0, + 0, + 27864, + 0, + 0, + 0, + 27865, + 27884, + 0, + 27889, + 0, + 0, + 0, + 0, + 0, + 27890, + 0, + 27891, + 0, + 0, + 0, + 27892, + 0, + 0, + 0, + 0, + 0, + 27897, + 27898, + 0, + 0, + 27899, + 0, + 0, + 0, + 27901, + 27905, + 0, + 0, + 27920, + 0, + 0, + 27921, + 0, + 27922, + 0, + 0, + 0, + 27931, + 27934, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27941, + 0, + 27942, + 0, + 27945, + 0, + 27947, + 27954, + 0, + 0, + 0, + 0, + 27960, + 27963, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 27964, + 27965, + 0, + 0, + 0, + 27967, + 0, + 27969, + 27975, + 0, + 27976, + 27977, + 0, + 27981, + 0, + 27983, + 28051, + 28052, + 0, + 0, + 0, + 0, + 0, + 28056, + 0, + 0, + 0, + 0, + 0, + 0, + 28058, + 28059, + 0, + 0, + 28061, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28063, + 0, + 0, + 0, + 0, + 0, + 0, + 28066, + 0, + 0, + 0, + 0, + 0, + 0, + 28069, + 28070, + 28072, + 0, + 28073, + 0, + 0, + 28074, + 0, + 0, + 0, + 0, + 28075, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28078, + 0, + 0, + 0, + 0, + 28085, + 0, + 0, + 0, + 0, + 28086, + 0, + 0, + 0, + 0, + 0, + 0, + 28088, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28090, + 0, + 28097, + 28114, + 28115, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28116, + 0, + 0, + 0, + 0, + 0, + 28118, + 0, + 28129, + 0, + 28131, + 0, + 0, + 28135, + 0, + 0, + 0, + 28140, + 28141, + 0, + 0, + 0, + 28146, + 0, + 0, + 0, + 0, + 28152, + 0, + 0, + 0, + 0, + 28155, + 28157, + 28161, + 0, + 0, + 0, + 0, + 28166, + 0, + 28167, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28172, + 0, + 0, + 0, + 0, + 0, + 0, + 28173, + 0, + 0, + 28175, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28178, + 28188, + 0, + 28190, + 0, + 0, + 0, + 0, + 0, + 28191, + 0, + 28193, + 28206, + 0, + 0, + 28207, + 28209, + 0, + 28211, + 0, + 28213, + 0, + 0, + 0, + 28215, + 28216, + 28217, + 0, + 28222, + 0, + 28223, + 28225, + 0, + 0, + 0, + 28226, + 0, + 28227, + 28229, + 28232, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28235, + 0, + 28241, + 0, + 0, + 28242, + 0, + 0, + 0, + 0, + 28243, + 0, + 0, + 0, + 28245, + 0, + 0, + 0, + 28248, + 28250, + 0, + 28251, + 28252, + 0, + 0, + 0, + 0, + 0, + 0, + 28253, + 0, + 0, + 28254, + 28255, + 0, + 0, + 28256, + 0, + 0, + 28258, + 0, + 0, + 0, + 0, + 0, + 28259, + 0, + 0, + 28260, + 0, + 0, + 28261, + 0, + 0, + 0, + 0, + 28262, + 28263, + 0, + 0, + 28264, + 0, + 0, + 0, + 28266, + 0, + 28268, + 28269, + 0, + 28270, + 28272, + 28274, + 0, + 28277, + 28278, + 0, + 0, + 0, + 28279, + 0, + 28280, + 28281, + 28283, + 0, + 28292, + 0, + 28294, + 0, + 28297, + 0, + 0, + 0, + 0, + 28299, + 0, + 0, + 0, + 0, + 0, + 28300, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28301, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28302, + 28303, + 0, + 0, + 0, + 0, + 28304, + 0, + 0, + 28305, + 0, + 28312, + 0, + 28313, + 28314, + 0, + 0, + 0, + 0, + 0, + 0, + 28315, + 0, + 0, + 0, + 28320, + 28321, + 0, + 0, + 28328, + 0, + 0, + 0, + 28329, + 28338, + 0, + 28339, + 0, + 0, + 28344, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28347, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28348, + 0, + 0, + 0, + 0, + 0, + 28411, + 0, + 28412, + 28413, + 0, + 28416, + 0, + 0, + 0, + 28420, + 0, + 0, + 0, + 0, + 0, + 28421, + 0, + 0, + 0, + 0, + 28423, + 0, + 0, + 0, + 28424, + 0, + 0, + 28428, + 0, + 0, + 0, + 0, + 0, + 28429, + 0, + 0, + 0, + 28431, + 28434, + 0, + 28458, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28464, + 0, + 0, + 0, + 0, + 28465, + 0, + 28467, + 0, + 0, + 0, + 0, + 0, + 0, + 28471, + 0, + 0, + 0, + 0, + 28474, + 0, + 28480, + 0, + 28481, + 0, + 0, + 28485, + 0, + 0, + 0, + 0, + 28486, + 28488, + 0, + 0, + 28489, + 0, + 0, + 0, + 0, + 28492, + 0, + 0, + 0, + 28495, + 0, + 28497, + 0, + 28499, + 0, + 0, + 0, + 0, + 28500, + 0, + 0, + 28502, + 28503, + 0, + 0, + 0, + 28508, + 0, + 0, + 0, + 28510, + 0, + 0, + 28512, + 28513, + 28514, + 28521, + 0, + 28526, + 0, + 28527, + 28528, + 0, + 0, + 0, + 0, + 28529, + 0, + 0, + 28532, + 0, + 0, + 28537, + 28538, + 0, + 0, + 0, + 28539, + 0, + 28548, + 0, + 28553, + 28554, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28560, + 28563, + 0, + 0, + 28564, + 0, + 0, + 0, + 0, + 28565, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28566, + 28568, + 0, + 0, + 0, + 0, + 0, + 0, + 28569, + 0, + 0, + 0, + 28570, + 0, + 28572, + 28573, + 0, + 0, + 0, + 0, + 28575, + 0, + 0, + 0, + 0, + 28576, + 28581, + 28588, + 0, + 0, + 28589, + 0, + 0, + 0, + 28590, + 28595, + 0, + 28598, + 0, + 0, + 28601, + 0, + 0, + 28605, + 0, + 0, + 0, + 0, + 28614, + 28615, + 28619, + 0, + 0, + 0, + 0, + 0, + 0, + 28620, + 0, + 28626, + 0, + 0, + 28628, + 0, + 28631, + 0, + 28632, + 0, + 0, + 0, + 0, + 0, + 0, + 28635, + 0, + 0, + 0, + 28637, + 28638, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28639, + 0, + 28643, + 0, + 0, + 28652, + 0, + 0, + 0, + 28662, + 0, + 28670, + 28671, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28672, + 28673, + 28675, + 28676, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28691, + 0, + 0, + 0, + 28695, + 0, + 0, + 0, + 28696, + 0, + 28697, + 28698, + 0, + 28705, + 0, + 28707, + 28708, + 28710, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28711, + 28728, + 0, + 0, + 0, + 28736, + 0, + 0, + 0, + 28737, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28738, + 0, + 28739, + 0, + 28741, + 0, + 0, + 28742, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28745, + 0, + 0, + 0, + 0, + 0, + 0, + 28749, + 28750, + 28752, + 28754, + 28756, + 0, + 28757, + 0, + 0, + 0, + 0, + 28759, + 28760, + 0, + 0, + 0, + 0, + 0, + 0, + 28762, + 0, + 0, + 0, + 28764, + 0, + 0, + 0, + 0, + 0, + 0, + 28766, + 0, + 28767, + 28768, + 0, + 0, + 0, + 0, + 28769, + 28770, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28771, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28772, + 0, + 28773, + 0, + 28782, + 0, + 0, + 0, + 0, + 0, + 0, + 28784, + 0, + 28785, + 0, + 28786, + 0, + 0, + 0, + 28787, + 0, + 0, + 0, + 28797, + 0, + 0, + 0, + 0, + 0, + 0, + 28799, + 0, + 0, + 28801, + 0, + 0, + 0, + 0, + 28802, + 0, + 28805, + 0, + 0, + 28806, + 0, + 0, + 28807, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28808, + 0, + 0, + 0, + 0, + 0, + 28810, + 28812, + 0, + 0, + 28816, + 28819, + 0, + 0, + 28821, + 0, + 28826, + 0, + 0, + 0, + 28842, + 28852, + 0, + 0, + 28853, + 0, + 28854, + 28855, + 0, + 0, + 0, + 28857, + 0, + 0, + 0, + 28858, + 0, + 28867, + 28868, + 28869, + 0, + 0, + 0, + 28874, + 28880, + 28882, + 28890, + 28892, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28895, + 0, + 0, + 0, + 28898, + 28899, + 0, + 0, + 0, + 28900, + 0, + 0, + 28904, + 0, + 28906, + 0, + 0, + 0, + 0, + 28907, + 0, + 0, + 0, + 0, + 0, + 0, + 28908, + 0, + 0, + 0, + 28910, + 0, + 28914, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28915, + 28916, + 28919, + 0, + 0, + 28920, + 0, + 28921, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28924, + 0, + 0, + 0, + 0, + 28926, + 28929, + 0, + 0, + 0, + 28930, + 0, + 28936, + 0, + 28939, + 0, + 0, + 0, + 0, + 28942, + 0, + 0, + 0, + 0, + 0, + 0, + 28956, + 0, + 0, + 0, + 28966, + 0, + 0, + 0, + 0, + 28967, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 28968, + 0, + 28971, + 0, + 28975, + 28976, + 0, + 28982, + 28983, + 0, + 0, + 28984, + 28989, + 28996, + 28997, + 28998, + 0, + 0, + 0, + 0, + 0, + 0, + 28999, + 0, + 0, + 0, + 0, + 0, + 29000, + 0, + 29001, + 0, + 0, + 0, + 29009, + 0, + 0, + 29011, + 0, + 0, + 29021, + 0, + 0, + 0, + 0, + 29024, + 0, + 29025, + 0, + 0, + 0, + 0, + 0, + 29026, + 0, + 0, + 0, + 29036, + 0, + 0, + 0, + 29037, + 0, + 0, + 0, + 0, + 29038, + 0, + 29045, + 0, + 29047, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29051, + 0, + 0, + 0, + 29054, + 29056, + 29062, + 0, + 29070, + 29082, + 0, + 0, + 0, + 29083, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29084, + 0, + 0, + 0, + 0, + 29085, + 29088, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29090, + 29097, + 0, + 0, + 0, + 29103, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29105, + 0, + 0, + 0, + 0, + 0, + 29107, + 0, + 29109, + 0, + 0, + 0, + 29115, + 0, + 0, + 29120, + 0, + 0, + 29138, + 29140, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29152, + 0, + 29160, + 29174, + 0, + 29176, + 0, + 0, + 29180, + 0, + 29181, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29228, + 0, + 0, + 29229, + 0, + 0, + 29230, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29234, + 0, + 0, + 0, + 29241, + 0, + 29245, + 0, + 29248, + 0, + 29250, + 29256, + 29280, + 0, + 29282, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29285, + 0, + 0, + 29286, + 29291, + 29292, + 0, + 0, + 0, + 0, + 29294, + 0, + 29295, + 0, + 0, + 0, + 0, + 0, + 29296, + 29297, + 29298, + 29300, + 0, + 29302, + 0, + 0, + 29304, + 29307, + 0, + 29312, + 0, + 0, + 0, + 29322, + 0, + 0, + 29323, + 0, + 0, + 29324, + 29326, + 29328, + 0, + 29335, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29338, + 29339, + 0, + 0, + 0, + 0, + 0, + 29341, + 29343, + 0, + 0, + 0, + 0, + 29344, + 0, + 0, + 0, + 0, + 0, + 29345, + 0, + 0, + 0, + 0, + 29346, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29347, + 29348, + 29349, + 0, + 0, + 29354, + 0, + 0, + 29355, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29357, + 0, + 0, + 0, + 0, + 29364, + 0, + 29365, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29366, + 0, + 0, + 29368, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29378, + 0, + 29381, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29386, + 0, + 0, + 0, + 0, + 0, + 0, + 29389, + 0, + 0, + 0, + 29390, + 0, + 0, + 29391, + 29397, + 0, + 29398, + 29412, + 29414, + 29418, + 29419, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29420, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29423, + 0, + 0, + 0, + 29435, + 0, + 0, + 0, + 29437, + 0, + 0, + 29439, + 0, + 29441, + 0, + 0, + 0, + 0, + 29443, + 0, + 29446, + 29450, + 29452, + 0, + 0, + 0, + 0, + 0, + 29456, + 0, + 0, + 0, + 0, + 0, + 29461, + 0, + 0, + 0, + 29464, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29468, + 0, + 29473, + 0, + 0, + 0, + 29486, + 0, + 0, + 0, + 29490, + 0, + 0, + 0, + 29491, + 29492, + 0, + 0, + 29497, + 0, + 0, + 0, + 29498, + 0, + 29499, + 0, + 29502, + 29505, + 0, + 29509, + 0, + 0, + 0, + 29510, + 0, + 0, + 0, + 29512, + 0, + 0, + 0, + 29516, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29518, + 0, + 29519, + 0, + 0, + 0, + 0, + 0, + 29520, + 29521, + 29529, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29530, + 0, + 0, + 29531, + 29538, + 0, + 29540, + 0, + 0, + 0, + 29542, + 0, + 29543, + 29544, + 29547, + 0, + 0, + 29548, + 0, + 0, + 0, + 29549, + 0, + 0, + 0, + 29550, + 0, + 0, + 29552, + 0, + 0, + 0, + 0, + 29558, + 29561, + 0, + 29562, + 29564, + 0, + 0, + 29565, + 0, + 0, + 29566, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29578, + 29584, + 29586, + 29591, + 0, + 0, + 0, + 0, + 29593, + 29594, + 0, + 0, + 29597, + 0, + 0, + 29613, + 0, + 29614, + 0, + 29615, + 0, + 0, + 0, + 0, + 29616, + 29617, + 0, + 0, + 29625, + 0, + 0, + 0, + 29632, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29633, + 0, + 0, + 0, + 0, + 0, + 29634, + 29635, + 29637, + 0, + 29638, + 0, + 29641, + 29643, + 0, + 0, + 0, + 0, + 0, + 0, + 29644, + 0, + 29645, + 0, + 29649, + 0, + 0, + 0, + 29650, + 0, + 29653, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29656, + 29659, + 0, + 0, + 29660, + 0, + 0, + 0, + 29661, + 0, + 0, + 0, + 0, + 0, + 29664, + 0, + 0, + 0, + 29671, + 29673, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29675, + 0, + 29677, + 29679, + 0, + 0, + 29684, + 0, + 0, + 0, + 0, + 0, + 29685, + 0, + 0, + 0, + 29687, + 0, + 0, + 0, + 29688, + 0, + 29689, + 29690, + 29700, + 0, + 29701, + 0, + 0, + 0, + 29702, + 0, + 29706, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29720, + 0, + 29721, + 0, + 29727, + 0, + 29733, + 29734, + 0, + 29750, + 29761, + 0, + 29763, + 0, + 0, + 0, + 0, + 0, + 29764, + 0, + 0, + 29765, + 0, + 0, + 0, + 29771, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29772, + 0, + 0, + 0, + 29773, + 29774, + 29775, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29822, + 0, + 0, + 0, + 29824, + 0, + 29825, + 0, + 0, + 0, + 0, + 0, + 29827, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29829, + 0, + 29832, + 29834, + 0, + 0, + 29835, + 0, + 0, + 29837, + 29838, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29843, + 0, + 0, + 0, + 0, + 29844, + 29845, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29849, + 0, + 0, + 29869, + 29872, + 29890, + 29905, + 0, + 0, + 0, + 0, + 0, + 29907, + 29921, + 0, + 29922, + 0, + 0, + 29923, + 29926, + 29944, + 29946, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 29947, + 29948, + 0, + 0, + 0, + 29951, + 0, + 0, + 0, + 0, + 0, + 29953, + 0, + 0, + 29956, + 0, + 29957, + 0, + 0, + 29962, + 0, + 0, + 0, + 0, + 29971, + 0, + 0, + 0, + 29972, + 0, + 0, + 0, + 0, + 0, + 29978, + 0, + 29979, + 29992, + 30007, + 30008, + 30010, + 0, + 0, + 0, + 30013, + 0, + 0, + 0, + 0, + 30014, + 30016, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30017, + 0, + 0, + 0, + 0, + 0, + 30023, + 30031, + 0, + 0, + 30033, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30034, + 0, + 30038, + 0, + 30039, + 0, + 30040, + 0, + 0, + 0, + 0, + 0, + 0, + 30067, + 30068, + 0, + 0, + 0, + 30069, + 0, + 30072, + 0, + 0, + 0, + 30073, + 0, + 0, + 0, + 0, + 30075, + 0, + 0, + 0, + 0, + 0, + 0, + 30079, + 0, + 0, + 30080, + 0, + 0, + 0, + 0, + 0, + 30082, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30084, + 30090, + 0, + 0, + 30091, + 0, + 0, + 0, + 0, + 30098, + 30118, + 0, + 30119, + 0, + 30121, + 30130, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30131, + 30132, + 30133, + 0, + 0, + 0, + 0, + 0, + 0, + 30135, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30136, + 0, + 0, + 30137, + 30138, + 0, + 0, + 0, + 30139, + 30146, + 0, + 0, + 0, + 0, + 0, + 30147, + 0, + 0, + 30148, + 30151, + 0, + 0, + 0, + 30168, + 0, + 30172, + 30173, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30180, + 30181, + 0, + 30192, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30194, + 30196, + 0, + 0, + 30199, + 0, + 0, + 30202, + 0, + 0, + 0, + 0, + 30203, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30213, + 0, + 0, + 0, + 30216, + 0, + 0, + 30217, + 0, + 0, + 0, + 30218, + 0, + 0, + 0, + 0, + 30219, + 0, + 30220, + 0, + 30222, + 30227, + 0, + 0, + 0, + 0, + 0, + 30231, + 0, + 0, + 30233, + 30235, + 0, + 0, + 0, + 0, + 30238, + 0, + 30240, + 30243, + 30245, + 0, + 30250, + 30252, + 0, + 0, + 0, + 30269, + 0, + 0, + 30271, + 30272, + 0, + 0, + 0, + 30278, + 30280, + 0, + 0, + 30282, + 0, + 30284, + 0, + 30294, + 0, + 0, + 0, + 0, + 30295, + 30296, + 0, + 0, + 0, + 0, + 0, + 30298, + 30299, + 30302, + 30304, + 30306, + 0, + 0, + 0, + 0, + 0, + 0, + 30316, + 30317, + 0, + 0, + 0, + 30318, + 0, + 0, + 0, + 30319, + 0, + 30320, + 30322, + 30326, + 0, + 0, + 0, + 0, + 0, + 30327, + 0, + 30332, + 30348, + 30349, + 0, + 0, + 30356, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30357, + 0, + 30358, + 0, + 30359, + 30360, + 0, + 0, + 30365, + 30366, + 30378, + 0, + 0, + 0, + 0, + 30379, + 0, + 0, + 30381, + 0, + 30385, + 0, + 30388, + 30397, + 0, + 0, + 0, + 30401, + 0, + 0, + 0, + 0, + 30403, + 0, + 0, + 0, + 0, + 0, + 30404, + 0, + 0, + 30405, + 0, + 30406, + 30408, + 0, + 30409, + 0, + 30410, + 0, + 0, + 0, + 30417, + 0, + 0, + 30418, + 30419, + 0, + 30420, + 0, + 30424, + 0, + 0, + 0, + 30427, + 30430, + 30432, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30433, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30436, + 0, + 30437, + 30438, + 0, + 30441, + 30442, + 0, + 0, + 0, + 30445, + 0, + 0, + 0, + 0, + 30452, + 30456, + 30457, + 0, + 0, + 0, + 30458, + 0, + 30464, + 0, + 0, + 0, + 0, + 0, + 0, + 30467, + 0, + 30469, + 0, + 0, + 0, + 0, + 0, + 30477, + 0, + 0, + 30484, + 0, + 0, + 0, + 0, + 0, + 30485, + 0, + 0, + 0, + 0, + 0, + 30486, + 30487, + 30497, + 30498, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30505, + 0, + 30508, + 0, + 0, + 0, + 30509, + 30510, + 0, + 30514, + 30516, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30523, + 0, + 30524, + 0, + 30525, + 0, + 0, + 0, + 0, + 30537, + 0, + 0, + 30538, + 0, + 0, + 0, + 0, + 0, + 30553, + 0, + 0, + 30555, + 30556, + 30558, + 30559, + 30560, + 0, + 0, + 30561, + 0, + 30562, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30563, + 30570, + 30571, + 0, + 30586, + 30587, + 0, + 0, + 30590, + 0, + 0, + 30594, + 0, + 0, + 0, + 0, + 30611, + 30612, + 30623, + 30634, + 0, + 0, + 30636, + 30640, + 30655, + 30656, + 0, + 30657, + 0, + 0, + 30658, + 30669, + 0, + 30670, + 0, + 30676, + 30678, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30679, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30695, + 0, + 0, + 30698, + 0, + 0, + 0, + 0, + 30700, + 0, + 0, + 0, + 0, + 30701, + 0, + 30702, + 30703, + 0, + 0, + 0, + 0, + 30707, + 0, + 0, + 0, + 30709, + 0, + 0, + 30710, + 30719, + 30729, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30731, + 0, + 0, + 30733, + 0, + 0, + 0, + 30734, + 0, + 0, + 0, + 0, + 0, + 30736, + 30737, + 0, + 0, + 0, + 30740, + 0, + 0, + 0, + 30743, + 0, + 30746, + 0, + 30747, + 30748, + 0, + 0, + 30751, + 30752, + 30753, + 0, + 0, + 0, + 30754, + 0, + 0, + 30760, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30763, + 0, + 30764, + 0, + 0, + 30766, + 0, + 30769, + 30770, + 30771, + 30774, + 30777, + 0, + 0, + 30779, + 30780, + 30781, + 0, + 0, + 0, + 0, + 30790, + 0, + 0, + 0, + 30792, + 0, + 0, + 0, + 0, + 30810, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30812, + 30819, + 0, + 0, + 30823, + 30824, + 0, + 30825, + 0, + 30827, + 0, + 0, + 0, + 0, + 0, + 0, + 30828, + 0, + 0, + 30830, + 0, + 0, + 0, + 30834, + 0, + 30835, + 0, + 30837, + 30838, + 0, + 30845, + 0, + 0, + 0, + 0, + 0, + 30846, + 30847, + 0, + 0, + 30849, + 0, + 30851, + 0, + 0, + 0, + 0, + 0, + 30852, + 30858, + 0, + 0, + 30859, + 0, + 30865, + 0, + 0, + 30866, + 0, + 0, + 30868, + 0, + 0, + 30869, + 0, + 0, + 0, + 30881, + 30883, + 0, + 0, + 0, + 0, + 0, + 30889, + 0, + 30891, + 0, + 0, + 0, + 0, + 30894, + 0, + 30895, + 0, + 30897, + 0, + 30898, + 0, + 0, + 0, + 30904, + 30906, + 0, + 30909, + 0, + 0, + 0, + 0, + 0, + 0, + 30910, + 0, + 0, + 0, + 30915, + 30933, + 30942, + 0, + 0, + 0, + 0, + 30943, + 0, + 0, + 30945, + 0, + 0, + 0, + 0, + 0, + 0, + 30946, + 0, + 0, + 30947, + 0, + 0, + 30955, + 30956, + 0, + 0, + 30960, + 0, + 0, + 30961, + 30962, + 30966, + 0, + 0, + 30969, + 30974, + 0, + 0, + 0, + 30976, + 0, + 0, + 30977, + 0, + 30978, + 30982, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 30994, + 30995, + 30998, + 0, + 31000, + 0, + 0, + 31001, + 0, + 0, + 31003, + 31005, + 0, + 0, + 31006, + 31011, + 0, + 0, + 31014, + 0, + 31016, + 0, + 0, + 0, + 0, + 31018, + 0, + 0, + 31020, + 31023, + 31024, + 31025, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31027, + 31028, + 31029, + 0, + 0, + 0, + 0, + 0, + 0, + 31032, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31036, + 31037, + 31038, + 0, + 0, + 0, + 31041, + 31043, + 31045, + 0, + 31047, + 0, + 0, + 0, + 31048, + 0, + 31049, + 0, + 0, + 0, + 31053, + 31054, + 31055, + 0, + 0, + 31063, + 0, + 0, + 0, + 0, + 0, + 31066, + 0, + 31068, + 31071, + 0, + 0, + 0, + 31072, + 31073, + 0, + 0, + 0, + 0, + 31075, + 0, + 0, + 31076, + 0, + 0, + 0, + 31077, + 31079, + 0, + 31080, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31087, + 0, + 31142, + 0, + 31144, + 0, + 0, + 31145, + 31146, + 31147, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31149, + 0, + 31151, + 31152, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31162, + 31171, + 31174, + 31175, + 0, + 0, + 0, + 31176, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31179, + 0, + 0, + 0, + 31186, + 0, + 0, + 0, + 31192, + 31195, + 0, + 0, + 31196, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31198, + 0, + 0, + 0, + 0, + 0, + 31199, + 0, + 0, + 0, + 31205, + 0, + 0, + 0, + 0, + 31211, + 31215, + 0, + 0, + 0, + 0, + 31231, + 0, + 31232, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31233, + 31236, + 31253, + 0, + 31254, + 0, + 0, + 0, + 0, + 0, + 0, + 31255, + 0, + 0, + 31257, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31258, + 31259, + 0, + 0, + 31260, + 0, + 31261, + 0, + 0, + 0, + 0, + 0, + 31262, + 31263, + 0, + 0, + 31264, + 0, + 31266, + 0, + 31267, + 0, + 0, + 0, + 0, + 0, + 31281, + 0, + 31282, + 0, + 31284, + 0, + 0, + 31285, + 31287, + 31288, + 0, + 0, + 31290, + 0, + 0, + 0, + 31292, + 31295, + 0, + 31299, + 0, + 31300, + 0, + 0, + 0, + 0, + 0, + 31302, + 0, + 0, + 0, + 0, + 31303, + 0, + 0, + 0, + 0, + 0, + 0, + 31304, + 0, + 0, + 0, + 0, + 0, + 31305, + 31308, + 31309, + 31315, + 0, + 31317, + 0, + 0, + 0, + 0, + 0, + 31323, + 0, + 31324, + 0, + 0, + 0, + 0, + 0, + 31325, + 31327, + 0, + 0, + 31331, + 0, + 0, + 0, + 0, + 0, + 31333, + 0, + 0, + 0, + 0, + 0, + 31336, + 0, + 0, + 31337, + 0, + 0, + 0, + 0, + 0, + 0, + 31338, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31339, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31342, + 0, + 0, + 0, + 0, + 31345, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31347, + 0, + 0, + 0, + 0, + 0, + 0, + 31348, + 0, + 0, + 31350, + 31351, + 0, + 31352, + 0, + 0, + 31354, + 0, + 0, + 0, + 0, + 31355, + 0, + 0, + 31356, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31363, + 0, + 31372, + 0, + 0, + 31373, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31376, + 0, + 31388, + 0, + 31389, + 0, + 31392, + 0, + 31401, + 0, + 31405, + 31407, + 31408, + 0, + 31409, + 0, + 0, + 0, + 0, + 0, + 0, + 31413, + 31415, + 0, + 0, + 0, + 31416, + 31418, + 0, + 0, + 0, + 0, + 0, + 0, + 31422, + 31423, + 0, + 0, + 31424, + 0, + 31425, + 31432, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31433, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31434, + 0, + 0, + 0, + 0, + 0, + 0, + 31435, + 0, + 0, + 0, + 0, + 31438, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31442, + 0, + 31444, + 0, + 31448, + 0, + 0, + 31451, + 0, + 0, + 0, + 0, + 31452, + 0, + 31461, + 31465, + 0, + 0, + 31466, + 0, + 0, + 31467, + 0, + 0, + 31468, + 0, + 0, + 0, + 31469, + 31473, + 0, + 31476, + 0, + 0, + 0, + 0, + 31489, + 31490, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31492, + 31493, + 31494, + 0, + 0, + 0, + 0, + 31501, + 31504, + 31505, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31509, + 0, + 0, + 0, + 0, + 31510, + 0, + 0, + 31511, + 0, + 0, + 31513, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31514, + 0, + 31522, + 31536, + 31539, + 31540, + 0, + 31541, + 0, + 0, + 0, + 0, + 0, + 0, + 31546, + 31553, + 31559, + 0, + 0, + 0, + 31560, + 31561, + 31562, + 0, + 0, + 31564, + 31567, + 0, + 31569, + 0, + 0, + 0, + 31570, + 0, + 0, + 0, + 0, + 31571, + 0, + 0, + 0, + 0, + 0, + 0, + 31572, + 31574, + 31580, + 31581, + 0, + 0, + 31582, + 31584, + 31585, + 31586, + 31595, + 0, + 31596, + 0, + 0, + 0, + 0, + 31597, + 0, + 31599, + 0, + 31600, + 31601, + 0, + 0, + 31603, + 31604, + 0, + 0, + 31608, + 31610, + 0, + 0, + 0, + 31611, + 0, + 31615, + 0, + 0, + 0, + 0, + 31616, + 0, + 0, + 0, + 0, + 0, + 0, + 31617, + 0, + 0, + 0, + 0, + 0, + 31618, + 0, + 0, + 0, + 0, + 0, + 0, + 31621, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31622, + 31625, + 0, + 0, + 0, + 0, + 31627, + 0, + 31641, + 0, + 0, + 31642, + 0, + 0, + 31643, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31644, + 0, + 31646, + 0, + 0, + 0, + 0, + 31648, + 0, + 0, + 0, + 31652, + 0, + 0, + 0, + 31657, + 0, + 0, + 31676, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 31689, + 31691, + 31692, + 0, + 31694, + 0, + 0, + 0, + 31696, + 0, + 31702, + 0, + 31703, + 0, +} + +var kStaticDictionaryWords = [31705]dictWord{ + dictWord{0, 0, 0}, + dictWord{8, 0, 1002}, + dictWord{136, 0, 1015}, + dictWord{4, 0, 683}, + dictWord{4, 10, 325}, + dictWord{138, 10, 125}, + dictWord{7, 11, 572}, + dictWord{ + 9, + 11, + 592, + }, + dictWord{11, 11, 680}, + dictWord{11, 11, 842}, + dictWord{11, 11, 924}, + dictWord{12, 11, 356}, + dictWord{12, 11, 550}, + dictWord{13, 11, 317}, + dictWord{13, 11, 370}, + dictWord{13, 11, 469}, + dictWord{13, 11, 471}, + dictWord{14, 11, 397}, + dictWord{18, 11, 69}, + dictWord{146, 11, 145}, + dictWord{ + 134, + 0, + 1265, + }, + dictWord{136, 11, 534}, + dictWord{134, 0, 1431}, + dictWord{11, 0, 138}, + dictWord{140, 0, 40}, + dictWord{4, 0, 155}, + dictWord{7, 0, 1689}, + dictWord{ + 4, + 10, + 718, + }, + dictWord{135, 10, 1216}, + dictWord{4, 0, 245}, + dictWord{5, 0, 151}, + dictWord{5, 0, 741}, + dictWord{6, 0, 1147}, + dictWord{7, 0, 498}, + dictWord{7, 0, 870}, + dictWord{7, 0, 1542}, + dictWord{12, 0, 213}, + dictWord{14, 0, 36}, + dictWord{14, 0, 391}, + dictWord{17, 0, 111}, + dictWord{18, 0, 6}, + dictWord{18, 0, 46}, + dictWord{ + 18, + 0, + 151, + }, + dictWord{19, 0, 36}, + dictWord{20, 0, 32}, + dictWord{20, 0, 56}, + dictWord{20, 0, 69}, + dictWord{20, 0, 102}, + dictWord{21, 0, 4}, + dictWord{22, 0, 8}, + dictWord{ + 22, + 0, + 10, + }, + dictWord{22, 0, 14}, + dictWord{150, 0, 31}, + dictWord{4, 0, 624}, + dictWord{135, 0, 1752}, + dictWord{5, 10, 124}, + dictWord{5, 10, 144}, + dictWord{6, 10, 548}, + dictWord{7, 10, 15}, + dictWord{7, 10, 153}, + dictWord{137, 10, 629}, + dictWord{6, 0, 503}, + dictWord{9, 0, 586}, + dictWord{13, 0, 468}, + dictWord{14, 0, 66}, + dictWord{ + 16, + 0, + 58, + }, + dictWord{7, 10, 1531}, + dictWord{8, 10, 416}, + dictWord{9, 10, 275}, + dictWord{10, 10, 100}, + dictWord{11, 10, 658}, + dictWord{11, 10, 979}, + dictWord{ + 12, + 10, + 86, + }, + dictWord{14, 10, 207}, + dictWord{15, 10, 20}, + dictWord{143, 10, 25}, + dictWord{5, 0, 603}, + dictWord{7, 0, 1212}, + dictWord{9, 0, 565}, + dictWord{ + 14, + 0, + 301, + }, + dictWord{5, 10, 915}, + dictWord{6, 10, 1783}, + dictWord{7, 10, 211}, + dictWord{7, 10, 1353}, + dictWord{9, 10, 83}, + dictWord{10, 10, 376}, + dictWord{ + 10, + 10, + 431, + }, + dictWord{11, 10, 543}, + dictWord{12, 10, 664}, + dictWord{13, 10, 280}, + dictWord{13, 10, 428}, + dictWord{14, 10, 128}, + dictWord{17, 10, 52}, + dictWord{ + 145, + 10, + 81, + }, + dictWord{4, 0, 492}, + dictWord{133, 0, 451}, + dictWord{135, 0, 835}, + dictWord{141, 0, 70}, + dictWord{132, 0, 539}, + dictWord{7, 11, 748}, + dictWord{ + 139, + 11, + 700, + }, + dictWord{7, 11, 1517}, + dictWord{11, 11, 597}, + dictWord{14, 11, 76}, + dictWord{14, 11, 335}, + dictWord{148, 11, 33}, + dictWord{6, 0, 113}, + dictWord{135, 0, 436}, + dictWord{4, 10, 338}, + dictWord{133, 10, 400}, + dictWord{136, 0, 718}, + dictWord{133, 11, 127}, + dictWord{133, 11, 418}, + dictWord{ + 6, + 0, + 1505, + }, + dictWord{7, 0, 520}, + dictWord{6, 11, 198}, + dictWord{11, 10, 892}, + dictWord{140, 11, 83}, + dictWord{4, 10, 221}, + dictWord{5, 10, 659}, + dictWord{ + 5, + 10, + 989, + }, + dictWord{7, 10, 697}, + dictWord{7, 10, 1211}, + dictWord{138, 10, 284}, + dictWord{135, 0, 1070}, + dictWord{5, 11, 276}, + dictWord{6, 11, 55}, + dictWord{ + 135, + 11, + 1369, + }, + dictWord{134, 0, 1515}, + dictWord{6, 11, 1752}, + dictWord{136, 11, 726}, + dictWord{138, 10, 507}, + dictWord{15, 0, 78}, + dictWord{4, 10, 188}, + dictWord{135, 10, 805}, + dictWord{5, 10, 884}, + dictWord{139, 10, 991}, + dictWord{133, 11, 764}, + dictWord{134, 10, 1653}, + dictWord{6, 11, 309}, + dictWord{ + 7, + 11, + 331, + }, + dictWord{138, 11, 550}, + dictWord{135, 11, 1861}, + dictWord{132, 11, 348}, + dictWord{135, 11, 986}, + dictWord{135, 11, 1573}, + dictWord{ + 12, + 0, + 610, + }, + dictWord{13, 0, 431}, + dictWord{144, 0, 59}, + dictWord{9, 11, 799}, + dictWord{140, 10, 166}, + dictWord{134, 0, 1530}, + dictWord{132, 0, 750}, + dictWord{132, 0, 307}, + dictWord{133, 0, 964}, + dictWord{6, 11, 194}, + dictWord{7, 11, 133}, + dictWord{10, 11, 493}, + dictWord{10, 11, 570}, + dictWord{139, 11, 664}, + dictWord{5, 11, 24}, + dictWord{5, 11, 569}, + dictWord{6, 11, 3}, + dictWord{6, 11, 119}, + dictWord{6, 11, 143}, + dictWord{6, 11, 440}, + dictWord{7, 11, 295}, + dictWord{ + 7, + 11, + 599, + }, + dictWord{7, 11, 1686}, + dictWord{7, 11, 1854}, + dictWord{8, 11, 424}, + dictWord{9, 11, 43}, + dictWord{9, 11, 584}, + dictWord{9, 11, 760}, + dictWord{ + 10, + 11, + 148, + }, + dictWord{10, 11, 328}, + dictWord{11, 11, 159}, + dictWord{11, 11, 253}, + dictWord{11, 11, 506}, + dictWord{12, 11, 487}, + dictWord{12, 11, 531}, + dictWord{144, 11, 33}, + dictWord{136, 10, 760}, + dictWord{5, 11, 14}, + dictWord{5, 11, 892}, + dictWord{6, 11, 283}, + dictWord{7, 11, 234}, + dictWord{136, 11, 537}, + dictWord{135, 11, 1251}, + dictWord{4, 11, 126}, + dictWord{8, 11, 635}, + dictWord{147, 11, 34}, + dictWord{4, 11, 316}, + dictWord{135, 11, 1561}, + dictWord{ + 6, + 0, + 999, + }, + dictWord{6, 0, 1310}, + dictWord{137, 11, 861}, + dictWord{4, 11, 64}, + dictWord{5, 11, 352}, + dictWord{5, 11, 720}, + dictWord{6, 11, 368}, + dictWord{ + 139, + 11, + 359, + }, + dictWord{4, 0, 75}, + dictWord{5, 0, 180}, + dictWord{6, 0, 500}, + dictWord{7, 0, 58}, + dictWord{7, 0, 710}, + dictWord{10, 0, 645}, + dictWord{136, 10, 770}, + dictWord{133, 0, 649}, + dictWord{6, 0, 276}, + dictWord{7, 0, 282}, + dictWord{7, 0, 879}, + dictWord{7, 0, 924}, + dictWord{8, 0, 459}, + dictWord{9, 0, 599}, + dictWord{9, 0, 754}, + dictWord{11, 0, 574}, + dictWord{12, 0, 128}, + dictWord{12, 0, 494}, + dictWord{13, 0, 52}, + dictWord{13, 0, 301}, + dictWord{15, 0, 30}, + dictWord{143, 0, 132}, + dictWord{132, 0, 200}, + dictWord{4, 10, 89}, + dictWord{5, 10, 489}, + dictWord{6, 10, 315}, + dictWord{7, 10, 553}, + dictWord{7, 10, 1745}, + dictWord{138, 10, 243}, + dictWord{135, 11, 1050}, + dictWord{7, 0, 1621}, + dictWord{6, 10, 1658}, + dictWord{9, 10, 3}, + dictWord{10, 10, 154}, + dictWord{11, 10, 641}, + dictWord{13, 10, 85}, + dictWord{13, 10, 201}, + dictWord{141, 10, 346}, + dictWord{6, 11, 175}, + dictWord{137, 11, 289}, + dictWord{5, 11, 432}, + dictWord{133, 11, 913}, + dictWord{ + 6, + 0, + 225, + }, + dictWord{137, 0, 211}, + dictWord{7, 0, 718}, + dictWord{8, 0, 687}, + dictWord{139, 0, 374}, + dictWord{4, 10, 166}, + dictWord{133, 10, 505}, + dictWord{ + 9, + 0, + 110, + }, + dictWord{134, 10, 1670}, + dictWord{8, 0, 58}, + dictWord{9, 0, 724}, + dictWord{11, 0, 809}, + dictWord{13, 0, 113}, + dictWord{145, 0, 72}, + dictWord{6, 0, 345}, + dictWord{7, 0, 1247}, + dictWord{144, 11, 82}, + dictWord{5, 11, 931}, + dictWord{134, 11, 1698}, + dictWord{8, 0, 767}, + dictWord{8, 0, 803}, + dictWord{9, 0, 301}, + dictWord{137, 0, 903}, + dictWord{139, 0, 203}, + dictWord{134, 0, 1154}, + dictWord{7, 0, 1949}, + dictWord{136, 0, 674}, + dictWord{134, 0, 259}, + dictWord{ + 135, + 0, + 1275, + }, + dictWord{5, 11, 774}, + dictWord{6, 11, 1637}, + dictWord{6, 11, 1686}, + dictWord{134, 11, 1751}, + dictWord{134, 0, 1231}, + dictWord{7, 10, 445}, + dictWord{8, 10, 307}, + dictWord{8, 10, 704}, + dictWord{10, 10, 41}, + dictWord{10, 10, 439}, + dictWord{11, 10, 237}, + dictWord{11, 10, 622}, + dictWord{140, 10, 201}, + dictWord{136, 0, 254}, + dictWord{6, 11, 260}, + dictWord{135, 11, 1484}, + dictWord{139, 0, 277}, + dictWord{135, 10, 1977}, + dictWord{4, 10, 189}, + dictWord{ + 5, + 10, + 713, + }, + dictWord{6, 11, 573}, + dictWord{136, 10, 57}, + dictWord{138, 10, 371}, + dictWord{132, 10, 552}, + dictWord{134, 11, 344}, + dictWord{133, 0, 248}, + dictWord{9, 0, 800}, + dictWord{10, 0, 693}, + dictWord{11, 0, 482}, + dictWord{11, 0, 734}, + dictWord{11, 0, 789}, + dictWord{134, 11, 240}, + dictWord{4, 0, 116}, + dictWord{ + 5, + 0, + 95, + }, + dictWord{5, 0, 445}, + dictWord{7, 0, 1688}, + dictWord{8, 0, 29}, + dictWord{9, 0, 272}, + dictWord{11, 0, 509}, + dictWord{11, 0, 915}, + dictWord{4, 11, 292}, + dictWord{4, 11, 736}, + dictWord{5, 11, 871}, + dictWord{6, 11, 171}, + dictWord{6, 11, 1689}, + dictWord{7, 11, 1324}, + dictWord{7, 11, 1944}, + dictWord{9, 11, 415}, + dictWord{9, 11, 580}, + dictWord{14, 11, 230}, + dictWord{146, 11, 68}, + dictWord{7, 0, 490}, + dictWord{13, 0, 100}, + dictWord{143, 0, 75}, + dictWord{135, 0, 1641}, + dictWord{133, 0, 543}, + dictWord{7, 11, 209}, + dictWord{8, 11, 661}, + dictWord{10, 11, 42}, + dictWord{11, 11, 58}, + dictWord{12, 11, 58}, + dictWord{12, 11, 118}, + dictWord{141, 11, 32}, + dictWord{5, 0, 181}, + dictWord{8, 0, 41}, + dictWord{6, 11, 63}, + dictWord{135, 11, 920}, + dictWord{133, 0, 657}, + dictWord{133, 11, 793}, + dictWord{138, 0, 709}, + dictWord{7, 0, 25}, + dictWord{8, 0, 202}, + dictWord{138, 0, 536}, + dictWord{5, 11, 665}, + dictWord{135, 10, 1788}, + dictWord{145, 10, 49}, + dictWord{9, 0, 423}, + dictWord{140, 0, 89}, + dictWord{5, 11, 67}, + dictWord{6, 11, 62}, + dictWord{6, 11, 374}, + dictWord{135, 11, 1391}, + dictWord{8, 0, 113}, + dictWord{ + 9, + 0, + 877, + }, + dictWord{10, 0, 554}, + dictWord{11, 0, 83}, + dictWord{12, 0, 136}, + dictWord{19, 0, 109}, + dictWord{9, 11, 790}, + dictWord{140, 11, 47}, + dictWord{ + 138, + 10, + 661, + }, + dictWord{4, 0, 963}, + dictWord{10, 0, 927}, + dictWord{14, 0, 442}, + dictWord{135, 10, 1945}, + dictWord{133, 0, 976}, + dictWord{132, 0, 206}, + dictWord{ + 4, + 11, + 391, + }, + dictWord{135, 11, 1169}, + dictWord{134, 0, 2002}, + dictWord{6, 0, 696}, + dictWord{134, 0, 1008}, + dictWord{134, 0, 1170}, + dictWord{132, 11, 271}, + dictWord{7, 0, 13}, + dictWord{8, 0, 226}, + dictWord{10, 0, 537}, + dictWord{11, 0, 570}, + dictWord{11, 0, 605}, + dictWord{11, 0, 799}, + dictWord{11, 0, 804}, + dictWord{ + 12, + 0, + 85, + }, + dictWord{12, 0, 516}, + dictWord{12, 0, 623}, + dictWord{13, 0, 112}, + dictWord{13, 0, 361}, + dictWord{14, 0, 77}, + dictWord{14, 0, 78}, + dictWord{17, 0, 28}, + dictWord{19, 0, 110}, + dictWord{140, 11, 314}, + dictWord{132, 0, 769}, + dictWord{134, 0, 1544}, + dictWord{4, 0, 551}, + dictWord{137, 0, 678}, + dictWord{5, 10, 84}, + dictWord{134, 10, 163}, + dictWord{9, 0, 57}, + dictWord{9, 0, 459}, + dictWord{10, 0, 425}, + dictWord{11, 0, 119}, + dictWord{12, 0, 184}, + dictWord{12, 0, 371}, + dictWord{ + 13, + 0, + 358, + }, + dictWord{145, 0, 51}, + dictWord{5, 0, 188}, + dictWord{5, 0, 814}, + dictWord{8, 0, 10}, + dictWord{9, 0, 421}, + dictWord{9, 0, 729}, + dictWord{10, 0, 609}, + dictWord{11, 0, 689}, + dictWord{4, 11, 253}, + dictWord{5, 10, 410}, + dictWord{5, 11, 544}, + dictWord{7, 11, 300}, + dictWord{137, 11, 340}, + dictWord{134, 0, 624}, + dictWord{138, 11, 321}, + dictWord{135, 0, 1941}, + dictWord{18, 0, 130}, + dictWord{5, 10, 322}, + dictWord{8, 10, 186}, + dictWord{9, 10, 262}, + dictWord{10, 10, 187}, + dictWord{142, 10, 208}, + dictWord{5, 11, 53}, + dictWord{5, 11, 541}, + dictWord{6, 11, 94}, + dictWord{6, 11, 499}, + dictWord{7, 11, 230}, + dictWord{139, 11, 321}, + dictWord{133, 10, 227}, + dictWord{4, 0, 378}, + dictWord{4, 11, 920}, + dictWord{5, 11, 25}, + dictWord{5, 11, 790}, + dictWord{6, 11, 457}, + dictWord{135, 11, 853}, + dictWord{137, 0, 269}, + dictWord{132, 0, 528}, + dictWord{134, 0, 1146}, + dictWord{7, 10, 1395}, + dictWord{8, 10, 486}, + dictWord{9, 10, 236}, + dictWord{9, 10, 878}, + dictWord{10, 10, 218}, + dictWord{11, 10, 95}, + dictWord{19, 10, 17}, + dictWord{147, 10, 31}, + dictWord{7, 10, 2043}, + dictWord{8, 10, 672}, + dictWord{ + 141, + 10, + 448, + }, + dictWord{134, 0, 1105}, + dictWord{134, 0, 1616}, + dictWord{134, 11, 1765}, + dictWord{140, 11, 163}, + dictWord{5, 10, 412}, + dictWord{133, 11, 822}, + dictWord{132, 11, 634}, + dictWord{6, 0, 656}, + dictWord{134, 11, 1730}, + dictWord{134, 0, 1940}, + dictWord{5, 0, 104}, + dictWord{6, 0, 173}, + dictWord{ + 135, + 0, + 1631, + }, + dictWord{136, 10, 562}, + dictWord{6, 11, 36}, + dictWord{7, 11, 658}, + dictWord{8, 11, 454}, + dictWord{147, 11, 86}, + dictWord{5, 0, 457}, + dictWord{ + 134, + 10, + 1771, + }, + dictWord{7, 0, 810}, + dictWord{8, 0, 138}, + dictWord{8, 0, 342}, + dictWord{9, 0, 84}, + dictWord{10, 0, 193}, + dictWord{11, 0, 883}, + dictWord{140, 0, 359}, + dictWord{9, 0, 620}, + dictWord{135, 10, 1190}, + dictWord{137, 10, 132}, + dictWord{7, 11, 975}, + dictWord{137, 11, 789}, + dictWord{6, 0, 95}, + dictWord{6, 0, 1934}, + dictWord{136, 0, 967}, + dictWord{141, 11, 335}, + dictWord{6, 0, 406}, + dictWord{10, 0, 409}, + dictWord{10, 0, 447}, + dictWord{11, 0, 44}, + dictWord{140, 0, 100}, + dictWord{4, 10, 317}, + dictWord{135, 10, 1279}, + dictWord{132, 0, 477}, + dictWord{134, 0, 1268}, + dictWord{6, 0, 1941}, + dictWord{8, 0, 944}, + dictWord{5, 10, 63}, + dictWord{133, 10, 509}, + dictWord{132, 0, 629}, + dictWord{132, 11, 104}, + dictWord{4, 0, 246}, + dictWord{133, 0, 375}, + dictWord{6, 0, 1636}, + dictWord{ + 132, + 10, + 288, + }, + dictWord{135, 11, 1614}, + dictWord{9, 0, 49}, + dictWord{10, 0, 774}, + dictWord{8, 10, 89}, + dictWord{8, 10, 620}, + dictWord{11, 10, 628}, + dictWord{ + 12, + 10, + 322, + }, + dictWord{143, 10, 124}, + dictWord{4, 0, 282}, + dictWord{7, 0, 1034}, + dictWord{11, 0, 398}, + dictWord{11, 0, 634}, + dictWord{12, 0, 1}, + dictWord{12, 0, 79}, + dictWord{12, 0, 544}, + dictWord{14, 0, 237}, + dictWord{17, 0, 10}, + dictWord{146, 0, 20}, + dictWord{132, 0, 824}, + dictWord{7, 11, 45}, + dictWord{9, 11, 542}, + dictWord{ + 9, + 11, + 566, + }, + dictWord{138, 11, 728}, + dictWord{5, 0, 118}, + dictWord{5, 0, 499}, + dictWord{6, 0, 476}, + dictWord{6, 0, 665}, + dictWord{6, 0, 1176}, + dictWord{ + 6, + 0, + 1196, + }, + dictWord{7, 0, 600}, + dictWord{7, 0, 888}, + dictWord{135, 0, 1096}, + dictWord{7, 0, 296}, + dictWord{7, 0, 596}, + dictWord{8, 0, 560}, + dictWord{8, 0, 586}, + dictWord{9, 0, 612}, + dictWord{11, 0, 304}, + dictWord{12, 0, 46}, + dictWord{13, 0, 89}, + dictWord{14, 0, 112}, + dictWord{145, 0, 122}, + dictWord{5, 0, 894}, + dictWord{ + 6, + 0, + 1772, + }, + dictWord{9, 0, 1009}, + dictWord{138, 10, 120}, + dictWord{5, 11, 533}, + dictWord{7, 11, 755}, + dictWord{138, 11, 780}, + dictWord{151, 10, 1}, + dictWord{ + 6, + 0, + 1474, + }, + dictWord{7, 11, 87}, + dictWord{142, 11, 288}, + dictWord{139, 0, 366}, + dictWord{137, 10, 461}, + dictWord{7, 11, 988}, + dictWord{7, 11, 1939}, + dictWord{ + 9, + 11, + 64, + }, + dictWord{9, 11, 502}, + dictWord{12, 11, 7}, + dictWord{12, 11, 34}, + dictWord{13, 11, 12}, + dictWord{13, 11, 234}, + dictWord{147, 11, 77}, + dictWord{ + 7, + 0, + 1599, + }, + dictWord{7, 0, 1723}, + dictWord{8, 0, 79}, + dictWord{8, 0, 106}, + dictWord{8, 0, 190}, + dictWord{8, 0, 302}, + dictWord{8, 0, 383}, + dictWord{8, 0, 713}, + dictWord{ + 9, + 0, + 119, + }, + dictWord{9, 0, 233}, + dictWord{9, 0, 419}, + dictWord{9, 0, 471}, + dictWord{10, 0, 181}, + dictWord{10, 0, 406}, + dictWord{11, 0, 57}, + dictWord{11, 0, 85}, + dictWord{11, 0, 120}, + dictWord{11, 0, 177}, + dictWord{11, 0, 296}, + dictWord{11, 0, 382}, + dictWord{11, 0, 454}, + dictWord{11, 0, 758}, + dictWord{11, 0, 999}, + dictWord{ + 12, + 0, + 27, + }, + dictWord{12, 0, 98}, + dictWord{12, 0, 131}, + dictWord{12, 0, 245}, + dictWord{12, 0, 312}, + dictWord{12, 0, 446}, + dictWord{12, 0, 454}, + dictWord{13, 0, 25}, + dictWord{13, 0, 98}, + dictWord{13, 0, 426}, + dictWord{13, 0, 508}, + dictWord{14, 0, 70}, + dictWord{14, 0, 163}, + dictWord{14, 0, 272}, + dictWord{14, 0, 277}, + dictWord{ + 14, + 0, + 370, + }, + dictWord{15, 0, 95}, + dictWord{15, 0, 138}, + dictWord{15, 0, 167}, + dictWord{17, 0, 38}, + dictWord{148, 0, 96}, + dictWord{135, 10, 1346}, + dictWord{ + 10, + 0, + 200, + }, + dictWord{19, 0, 2}, + dictWord{151, 0, 22}, + dictWord{135, 11, 141}, + dictWord{134, 10, 85}, + dictWord{134, 0, 1759}, + dictWord{138, 0, 372}, + dictWord{ + 145, + 0, + 16, + }, + dictWord{8, 0, 943}, + dictWord{132, 11, 619}, + dictWord{139, 11, 88}, + dictWord{5, 11, 246}, + dictWord{8, 11, 189}, + dictWord{9, 11, 355}, + dictWord{ + 9, + 11, + 512, + }, + dictWord{10, 11, 124}, + dictWord{10, 11, 453}, + dictWord{11, 11, 143}, + dictWord{11, 11, 416}, + dictWord{11, 11, 859}, + dictWord{141, 11, 341}, + dictWord{ + 5, + 0, + 258, + }, + dictWord{134, 0, 719}, + dictWord{6, 0, 1798}, + dictWord{6, 0, 1839}, + dictWord{8, 0, 900}, + dictWord{10, 0, 874}, + dictWord{10, 0, 886}, + dictWord{ + 12, + 0, + 698, + }, + dictWord{12, 0, 732}, + dictWord{12, 0, 770}, + dictWord{16, 0, 106}, + dictWord{18, 0, 163}, + dictWord{18, 0, 170}, + dictWord{18, 0, 171}, + dictWord{152, 0, 20}, + dictWord{9, 0, 707}, + dictWord{11, 0, 326}, + dictWord{11, 0, 339}, + dictWord{12, 0, 423}, + dictWord{12, 0, 502}, + dictWord{20, 0, 62}, + dictWord{9, 11, 707}, + dictWord{ + 11, + 11, + 326, + }, + dictWord{11, 11, 339}, + dictWord{12, 11, 423}, + dictWord{12, 11, 502}, + dictWord{148, 11, 62}, + dictWord{5, 0, 30}, + dictWord{7, 0, 495}, + dictWord{ + 8, + 0, + 134, + }, + dictWord{9, 0, 788}, + dictWord{140, 0, 438}, + dictWord{133, 11, 678}, + dictWord{5, 10, 279}, + dictWord{6, 10, 235}, + dictWord{7, 10, 468}, + dictWord{ + 8, + 10, + 446, + }, + dictWord{9, 10, 637}, + dictWord{10, 10, 717}, + dictWord{11, 10, 738}, + dictWord{140, 10, 514}, + dictWord{5, 11, 35}, + dictWord{6, 11, 287}, + dictWord{ + 7, + 11, + 862, + }, + dictWord{7, 11, 1886}, + dictWord{138, 11, 179}, + dictWord{7, 0, 1948}, + dictWord{7, 0, 2004}, + dictWord{132, 11, 517}, + dictWord{5, 10, 17}, + dictWord{ + 6, + 10, + 371, + }, + dictWord{137, 10, 528}, + dictWord{4, 0, 115}, + dictWord{5, 0, 669}, + dictWord{6, 0, 407}, + dictWord{8, 0, 311}, + dictWord{11, 0, 10}, + dictWord{141, 0, 5}, + dictWord{137, 0, 381}, + dictWord{5, 0, 50}, + dictWord{6, 0, 439}, + dictWord{7, 0, 780}, + dictWord{135, 0, 1040}, + dictWord{136, 11, 667}, + dictWord{11, 11, 403}, + dictWord{146, 11, 83}, + dictWord{5, 0, 1}, + dictWord{6, 0, 81}, + dictWord{138, 0, 520}, + dictWord{134, 0, 738}, + dictWord{5, 0, 482}, + dictWord{8, 0, 98}, + dictWord{9, 0, 172}, + dictWord{10, 0, 360}, + dictWord{10, 0, 700}, + dictWord{10, 0, 822}, + dictWord{11, 0, 302}, + dictWord{11, 0, 778}, + dictWord{12, 0, 50}, + dictWord{12, 0, 127}, + dictWord{ + 12, + 0, + 396, + }, + dictWord{13, 0, 62}, + dictWord{13, 0, 328}, + dictWord{14, 0, 122}, + dictWord{147, 0, 72}, + dictWord{9, 11, 157}, + dictWord{10, 11, 131}, + dictWord{ + 140, + 11, + 72, + }, + dictWord{135, 11, 714}, + dictWord{135, 11, 539}, + dictWord{5, 0, 2}, + dictWord{6, 0, 512}, + dictWord{7, 0, 797}, + dictWord{7, 0, 1494}, + dictWord{8, 0, 253}, + dictWord{8, 0, 589}, + dictWord{9, 0, 77}, + dictWord{10, 0, 1}, + dictWord{10, 0, 129}, + dictWord{10, 0, 225}, + dictWord{11, 0, 118}, + dictWord{11, 0, 226}, + dictWord{ + 11, + 0, + 251, + }, + dictWord{11, 0, 430}, + dictWord{11, 0, 701}, + dictWord{11, 0, 974}, + dictWord{11, 0, 982}, + dictWord{12, 0, 64}, + dictWord{12, 0, 260}, + dictWord{12, 0, 488}, + dictWord{140, 0, 690}, + dictWord{5, 11, 394}, + dictWord{7, 11, 367}, + dictWord{7, 11, 487}, + dictWord{7, 11, 857}, + dictWord{7, 11, 1713}, + dictWord{8, 11, 246}, + dictWord{9, 11, 537}, + dictWord{10, 11, 165}, + dictWord{12, 11, 219}, + dictWord{140, 11, 561}, + dictWord{136, 0, 557}, + dictWord{5, 10, 779}, + dictWord{5, 10, 807}, + dictWord{6, 10, 1655}, + dictWord{134, 10, 1676}, + dictWord{4, 10, 196}, + dictWord{5, 10, 558}, + dictWord{133, 10, 949}, + dictWord{11, 11, 827}, + dictWord{ + 12, + 11, + 56, + }, + dictWord{14, 11, 34}, + dictWord{143, 11, 148}, + dictWord{137, 0, 347}, + dictWord{133, 0, 572}, + dictWord{134, 0, 832}, + dictWord{4, 0, 12}, + dictWord{ + 7, + 0, + 504, + }, + dictWord{7, 0, 522}, + dictWord{7, 0, 809}, + dictWord{8, 0, 797}, + dictWord{141, 0, 88}, + dictWord{4, 10, 752}, + dictWord{133, 11, 449}, + dictWord{7, 11, 86}, + dictWord{8, 11, 103}, + dictWord{145, 11, 69}, + dictWord{7, 11, 2028}, + dictWord{138, 11, 641}, + dictWord{5, 0, 528}, + dictWord{6, 11, 1}, + dictWord{142, 11, 2}, + dictWord{134, 0, 861}, + dictWord{10, 0, 294}, + dictWord{4, 10, 227}, + dictWord{5, 10, 159}, + dictWord{5, 10, 409}, + dictWord{7, 10, 80}, + dictWord{10, 10, 479}, + dictWord{ + 12, + 10, + 418, + }, + dictWord{14, 10, 50}, + dictWord{14, 10, 249}, + dictWord{142, 10, 295}, + dictWord{7, 10, 1470}, + dictWord{8, 10, 66}, + dictWord{8, 10, 137}, + dictWord{ + 8, + 10, + 761, + }, + dictWord{9, 10, 638}, + dictWord{11, 10, 80}, + dictWord{11, 10, 212}, + dictWord{11, 10, 368}, + dictWord{11, 10, 418}, + dictWord{12, 10, 8}, + dictWord{ + 13, + 10, + 15, + }, + dictWord{16, 10, 61}, + dictWord{17, 10, 59}, + dictWord{19, 10, 28}, + dictWord{148, 10, 84}, + dictWord{20, 0, 109}, + dictWord{135, 11, 1148}, + dictWord{ + 6, + 11, + 277, + }, + dictWord{7, 11, 1274}, + dictWord{7, 11, 1386}, + dictWord{7, 11, 1392}, + dictWord{12, 11, 129}, + dictWord{146, 11, 87}, + dictWord{6, 11, 187}, + dictWord{7, 11, 39}, + dictWord{7, 11, 1203}, + dictWord{8, 11, 380}, + dictWord{8, 11, 542}, + dictWord{14, 11, 117}, + dictWord{149, 11, 28}, + dictWord{134, 0, 1187}, + dictWord{5, 0, 266}, + dictWord{9, 0, 290}, + dictWord{9, 0, 364}, + dictWord{10, 0, 293}, + dictWord{11, 0, 606}, + dictWord{142, 0, 45}, + dictWord{6, 11, 297}, + dictWord{ + 7, + 11, + 793, + }, + dictWord{139, 11, 938}, + dictWord{4, 0, 50}, + dictWord{6, 0, 594}, + dictWord{9, 0, 121}, + dictWord{10, 0, 49}, + dictWord{10, 0, 412}, + dictWord{139, 0, 834}, + dictWord{136, 0, 748}, + dictWord{7, 11, 464}, + dictWord{8, 11, 438}, + dictWord{11, 11, 105}, + dictWord{11, 11, 363}, + dictWord{12, 11, 231}, + dictWord{ + 14, + 11, + 386, + }, + dictWord{15, 11, 102}, + dictWord{148, 11, 75}, + dictWord{132, 0, 466}, + dictWord{13, 0, 399}, + dictWord{14, 0, 337}, + dictWord{6, 10, 38}, + dictWord{ + 7, + 10, + 1220, + }, + dictWord{8, 10, 185}, + dictWord{8, 10, 256}, + dictWord{9, 10, 22}, + dictWord{9, 10, 331}, + dictWord{10, 10, 738}, + dictWord{11, 10, 205}, + dictWord{ + 11, + 10, + 540, + }, + dictWord{11, 10, 746}, + dictWord{13, 10, 465}, + dictWord{142, 10, 194}, + dictWord{9, 0, 378}, + dictWord{141, 0, 162}, + dictWord{137, 0, 519}, + dictWord{ + 4, + 10, + 159, + }, + dictWord{6, 10, 115}, + dictWord{7, 10, 252}, + dictWord{7, 10, 257}, + dictWord{7, 10, 1928}, + dictWord{8, 10, 69}, + dictWord{9, 10, 384}, + dictWord{ + 10, + 10, + 91, + }, + dictWord{10, 10, 615}, + dictWord{12, 10, 375}, + dictWord{14, 10, 235}, + dictWord{18, 10, 117}, + dictWord{147, 10, 123}, + dictWord{5, 11, 604}, + dictWord{ + 5, + 10, + 911, + }, + dictWord{136, 10, 278}, + dictWord{132, 0, 667}, + dictWord{8, 0, 351}, + dictWord{9, 0, 322}, + dictWord{4, 10, 151}, + dictWord{135, 10, 1567}, + dictWord{134, 0, 902}, + dictWord{133, 10, 990}, + dictWord{12, 0, 180}, + dictWord{5, 10, 194}, + dictWord{7, 10, 1662}, + dictWord{137, 10, 90}, + dictWord{4, 0, 869}, + dictWord{134, 0, 1996}, + dictWord{134, 0, 813}, + dictWord{133, 10, 425}, + dictWord{137, 11, 761}, + dictWord{132, 0, 260}, + dictWord{133, 10, 971}, + dictWord{ + 5, + 11, + 20, + }, + dictWord{6, 11, 298}, + dictWord{7, 11, 659}, + dictWord{7, 11, 1366}, + dictWord{137, 11, 219}, + dictWord{4, 0, 39}, + dictWord{5, 0, 36}, + dictWord{ + 7, + 0, + 1843, + }, + dictWord{8, 0, 407}, + dictWord{11, 0, 144}, + dictWord{140, 0, 523}, + dictWord{4, 0, 510}, + dictWord{10, 0, 587}, + dictWord{139, 10, 752}, + dictWord{7, 0, 29}, + dictWord{7, 0, 66}, + dictWord{7, 0, 1980}, + dictWord{10, 0, 487}, + dictWord{138, 0, 809}, + dictWord{13, 0, 260}, + dictWord{14, 0, 82}, + dictWord{18, 0, 63}, + dictWord{ + 137, + 10, + 662, + }, + dictWord{5, 10, 72}, + dictWord{6, 10, 264}, + dictWord{7, 10, 21}, + dictWord{7, 10, 46}, + dictWord{7, 10, 2013}, + dictWord{8, 10, 215}, + dictWord{ + 8, + 10, + 513, + }, + dictWord{10, 10, 266}, + dictWord{139, 10, 22}, + dictWord{134, 0, 570}, + dictWord{6, 0, 565}, + dictWord{7, 0, 1667}, + dictWord{4, 11, 439}, + dictWord{ + 10, + 10, + 95, + }, + dictWord{11, 10, 603}, + dictWord{12, 11, 242}, + dictWord{13, 10, 443}, + dictWord{14, 10, 160}, + dictWord{143, 10, 4}, + dictWord{134, 0, 1464}, + dictWord{ + 134, + 10, + 431, + }, + dictWord{9, 0, 372}, + dictWord{15, 0, 2}, + dictWord{19, 0, 10}, + dictWord{19, 0, 18}, + dictWord{5, 10, 874}, + dictWord{6, 10, 1677}, + dictWord{143, 10, 0}, + dictWord{132, 0, 787}, + dictWord{6, 0, 380}, + dictWord{12, 0, 399}, + dictWord{21, 0, 19}, + dictWord{7, 10, 939}, + dictWord{7, 10, 1172}, + dictWord{7, 10, 1671}, + dictWord{9, 10, 540}, + dictWord{10, 10, 696}, + dictWord{11, 10, 265}, + dictWord{11, 10, 732}, + dictWord{11, 10, 928}, + dictWord{11, 10, 937}, + dictWord{ + 141, + 10, + 438, + }, + dictWord{137, 0, 200}, + dictWord{132, 11, 233}, + dictWord{132, 0, 516}, + dictWord{134, 11, 577}, + dictWord{132, 0, 844}, + dictWord{11, 0, 887}, + dictWord{14, 0, 365}, + dictWord{142, 0, 375}, + dictWord{132, 11, 482}, + dictWord{8, 0, 821}, + dictWord{140, 0, 44}, + dictWord{7, 0, 1655}, + dictWord{136, 0, 305}, + dictWord{5, 10, 682}, + dictWord{135, 10, 1887}, + dictWord{135, 11, 346}, + dictWord{132, 10, 696}, + dictWord{4, 0, 10}, + dictWord{7, 0, 917}, + dictWord{139, 0, 786}, + dictWord{5, 11, 795}, + dictWord{6, 11, 1741}, + dictWord{8, 11, 417}, + dictWord{137, 11, 782}, + dictWord{4, 0, 1016}, + dictWord{134, 0, 2031}, + dictWord{5, 0, 684}, + dictWord{4, 10, 726}, + dictWord{133, 10, 630}, + dictWord{6, 0, 1021}, + dictWord{134, 0, 1480}, + dictWord{8, 10, 802}, + dictWord{136, 10, 838}, + dictWord{ + 134, + 0, + 27, + }, + dictWord{134, 0, 395}, + dictWord{135, 11, 622}, + dictWord{7, 11, 625}, + dictWord{135, 11, 1750}, + dictWord{4, 11, 203}, + dictWord{135, 11, 1936}, + dictWord{6, 10, 118}, + dictWord{7, 10, 215}, + dictWord{7, 10, 1521}, + dictWord{140, 10, 11}, + dictWord{132, 0, 813}, + dictWord{136, 0, 511}, + dictWord{7, 10, 615}, + dictWord{138, 10, 251}, + dictWord{135, 10, 1044}, + dictWord{145, 0, 56}, + dictWord{133, 10, 225}, + dictWord{6, 0, 342}, + dictWord{6, 0, 496}, + dictWord{8, 0, 275}, + dictWord{137, 0, 206}, + dictWord{4, 0, 909}, + dictWord{133, 0, 940}, + dictWord{132, 0, 891}, + dictWord{7, 11, 311}, + dictWord{9, 11, 308}, + dictWord{ + 140, + 11, + 255, + }, + dictWord{4, 10, 370}, + dictWord{5, 10, 756}, + dictWord{135, 10, 1326}, + dictWord{4, 0, 687}, + dictWord{134, 0, 1596}, + dictWord{134, 0, 1342}, + dictWord{ + 6, + 10, + 1662, + }, + dictWord{7, 10, 48}, + dictWord{8, 10, 771}, + dictWord{10, 10, 116}, + dictWord{13, 10, 104}, + dictWord{14, 10, 105}, + dictWord{14, 10, 184}, + dictWord{15, 10, 168}, + dictWord{19, 10, 92}, + dictWord{148, 10, 68}, + dictWord{138, 10, 209}, + dictWord{4, 11, 400}, + dictWord{5, 11, 267}, + dictWord{135, 11, 232}, + dictWord{151, 11, 12}, + dictWord{6, 0, 41}, + dictWord{141, 0, 160}, + dictWord{141, 11, 314}, + dictWord{134, 0, 1718}, + dictWord{136, 0, 778}, + dictWord{ + 142, + 11, + 261, + }, + dictWord{134, 0, 1610}, + dictWord{133, 0, 115}, + dictWord{132, 0, 294}, + dictWord{14, 0, 314}, + dictWord{132, 10, 120}, + dictWord{132, 0, 983}, + dictWord{5, 0, 193}, + dictWord{140, 0, 178}, + dictWord{138, 10, 429}, + dictWord{5, 10, 820}, + dictWord{135, 10, 931}, + dictWord{6, 0, 994}, + dictWord{6, 0, 1051}, + dictWord{6, 0, 1439}, + dictWord{7, 0, 174}, + dictWord{133, 11, 732}, + dictWord{4, 11, 100}, + dictWord{7, 11, 679}, + dictWord{8, 11, 313}, + dictWord{138, 10, 199}, + dictWord{6, 10, 151}, + dictWord{6, 10, 1675}, + dictWord{7, 10, 383}, + dictWord{151, 10, 10}, + dictWord{6, 0, 1796}, + dictWord{8, 0, 848}, + dictWord{8, 0, 867}, + dictWord{ + 8, + 0, + 907, + }, + dictWord{10, 0, 855}, + dictWord{140, 0, 703}, + dictWord{140, 0, 221}, + dictWord{4, 0, 122}, + dictWord{5, 0, 796}, + dictWord{5, 0, 952}, + dictWord{6, 0, 1660}, + dictWord{6, 0, 1671}, + dictWord{8, 0, 567}, + dictWord{9, 0, 687}, + dictWord{9, 0, 742}, + dictWord{10, 0, 686}, + dictWord{11, 0, 682}, + dictWord{11, 0, 909}, + dictWord{ + 140, + 0, + 281, + }, + dictWord{5, 11, 362}, + dictWord{5, 11, 443}, + dictWord{6, 11, 318}, + dictWord{7, 11, 1019}, + dictWord{139, 11, 623}, + dictWord{5, 11, 463}, + dictWord{136, 11, 296}, + dictWord{11, 0, 583}, + dictWord{13, 0, 262}, + dictWord{6, 10, 1624}, + dictWord{12, 10, 422}, + dictWord{142, 10, 360}, + dictWord{5, 0, 179}, + dictWord{7, 0, 1095}, + dictWord{135, 0, 1213}, + dictWord{4, 10, 43}, + dictWord{4, 11, 454}, + dictWord{5, 10, 344}, + dictWord{133, 10, 357}, + dictWord{4, 0, 66}, + dictWord{7, 0, 722}, + dictWord{135, 0, 904}, + dictWord{134, 0, 773}, + dictWord{7, 0, 352}, + dictWord{133, 10, 888}, + dictWord{5, 11, 48}, + dictWord{5, 11, 404}, + dictWord{ + 6, + 11, + 557, + }, + dictWord{7, 11, 458}, + dictWord{8, 11, 597}, + dictWord{10, 11, 455}, + dictWord{10, 11, 606}, + dictWord{11, 11, 49}, + dictWord{11, 11, 548}, + dictWord{ + 12, + 11, + 476, + }, + dictWord{13, 11, 18}, + dictWord{141, 11, 450}, + dictWord{134, 11, 418}, + dictWord{132, 10, 711}, + dictWord{5, 11, 442}, + dictWord{ + 135, + 11, + 1984, + }, + dictWord{141, 0, 35}, + dictWord{137, 0, 152}, + dictWord{134, 0, 1197}, + dictWord{135, 11, 1093}, + dictWord{137, 11, 203}, + dictWord{137, 10, 440}, + dictWord{10, 0, 592}, + dictWord{10, 0, 753}, + dictWord{12, 0, 317}, + dictWord{12, 0, 355}, + dictWord{12, 0, 465}, + dictWord{12, 0, 469}, + dictWord{12, 0, 560}, + dictWord{12, 0, 578}, + dictWord{141, 0, 243}, + dictWord{133, 0, 564}, + dictWord{134, 0, 797}, + dictWord{5, 10, 958}, + dictWord{133, 10, 987}, + dictWord{5, 11, 55}, + dictWord{7, 11, 376}, + dictWord{140, 11, 161}, + dictWord{133, 11, 450}, + dictWord{134, 0, 556}, + dictWord{134, 0, 819}, + dictWord{11, 10, 276}, + dictWord{ + 142, + 10, + 293, + }, + dictWord{7, 0, 544}, + dictWord{138, 0, 61}, + dictWord{8, 0, 719}, + dictWord{4, 10, 65}, + dictWord{5, 10, 479}, + dictWord{5, 10, 1004}, + dictWord{7, 10, 1913}, + dictWord{8, 10, 317}, + dictWord{9, 10, 302}, + dictWord{10, 10, 612}, + dictWord{141, 10, 22}, + dictWord{4, 0, 5}, + dictWord{5, 0, 498}, + dictWord{8, 0, 637}, + dictWord{ + 9, + 0, + 521, + }, + dictWord{4, 11, 213}, + dictWord{4, 10, 261}, + dictWord{7, 11, 223}, + dictWord{7, 10, 510}, + dictWord{136, 11, 80}, + dictWord{5, 0, 927}, + dictWord{7, 0, 101}, + dictWord{4, 10, 291}, + dictWord{7, 11, 381}, + dictWord{7, 11, 806}, + dictWord{7, 11, 820}, + dictWord{8, 11, 354}, + dictWord{8, 11, 437}, + dictWord{8, 11, 787}, + dictWord{9, 10, 515}, + dictWord{9, 11, 657}, + dictWord{10, 11, 58}, + dictWord{10, 11, 339}, + dictWord{10, 11, 749}, + dictWord{11, 11, 914}, + dictWord{12, 10, 152}, + dictWord{12, 11, 162}, + dictWord{12, 10, 443}, + dictWord{13, 11, 75}, + dictWord{13, 10, 392}, + dictWord{14, 11, 106}, + dictWord{14, 11, 198}, + dictWord{ + 14, + 11, + 320, + }, + dictWord{14, 10, 357}, + dictWord{14, 11, 413}, + dictWord{146, 11, 43}, + dictWord{6, 0, 1153}, + dictWord{7, 0, 1441}, + dictWord{136, 11, 747}, + dictWord{ + 4, + 0, + 893, + }, + dictWord{5, 0, 780}, + dictWord{133, 0, 893}, + dictWord{138, 11, 654}, + dictWord{133, 11, 692}, + dictWord{133, 0, 238}, + dictWord{134, 11, 191}, + dictWord{4, 10, 130}, + dictWord{135, 10, 843}, + dictWord{6, 0, 1296}, + dictWord{5, 10, 42}, + dictWord{5, 10, 879}, + dictWord{7, 10, 245}, + dictWord{7, 10, 324}, + dictWord{ + 7, + 10, + 1532, + }, + dictWord{11, 10, 463}, + dictWord{11, 10, 472}, + dictWord{13, 10, 363}, + dictWord{144, 10, 52}, + dictWord{134, 0, 1729}, + dictWord{6, 0, 1999}, + dictWord{136, 0, 969}, + dictWord{4, 10, 134}, + dictWord{133, 10, 372}, + dictWord{4, 0, 60}, + dictWord{7, 0, 941}, + dictWord{7, 0, 1800}, + dictWord{8, 0, 314}, + dictWord{ + 9, + 0, + 700, + }, + dictWord{139, 0, 487}, + dictWord{134, 0, 1144}, + dictWord{6, 11, 162}, + dictWord{7, 11, 1960}, + dictWord{136, 11, 831}, + dictWord{132, 11, 706}, + dictWord{135, 0, 1147}, + dictWord{138, 11, 426}, + dictWord{138, 11, 89}, + dictWord{7, 0, 1853}, + dictWord{138, 0, 437}, + dictWord{136, 0, 419}, + dictWord{ + 135, + 10, + 1634, + }, + dictWord{133, 0, 828}, + dictWord{5, 0, 806}, + dictWord{7, 0, 176}, + dictWord{7, 0, 178}, + dictWord{7, 0, 1240}, + dictWord{7, 0, 1976}, + dictWord{ + 132, + 10, + 644, + }, + dictWord{135, 11, 1877}, + dictWord{5, 11, 420}, + dictWord{135, 11, 1449}, + dictWord{4, 0, 51}, + dictWord{5, 0, 39}, + dictWord{6, 0, 4}, + dictWord{7, 0, 591}, + dictWord{7, 0, 849}, + dictWord{7, 0, 951}, + dictWord{7, 0, 1613}, + dictWord{7, 0, 1760}, + dictWord{7, 0, 1988}, + dictWord{9, 0, 434}, + dictWord{10, 0, 754}, + dictWord{ + 11, + 0, + 25, + }, + dictWord{139, 0, 37}, + dictWord{10, 11, 57}, + dictWord{138, 11, 277}, + dictWord{135, 10, 540}, + dictWord{132, 11, 204}, + dictWord{135, 0, 159}, + dictWord{139, 11, 231}, + dictWord{133, 0, 902}, + dictWord{7, 0, 928}, + dictWord{7, 11, 366}, + dictWord{9, 11, 287}, + dictWord{12, 11, 199}, + dictWord{12, 11, 556}, + dictWord{140, 11, 577}, + dictWord{6, 10, 623}, + dictWord{136, 10, 789}, + dictWord{4, 10, 908}, + dictWord{5, 10, 359}, + dictWord{5, 10, 508}, + dictWord{6, 10, 1723}, + dictWord{7, 10, 343}, + dictWord{7, 10, 1996}, + dictWord{135, 10, 2026}, + dictWord{134, 0, 270}, + dictWord{4, 10, 341}, + dictWord{135, 10, 480}, + dictWord{ + 5, + 11, + 356, + }, + dictWord{135, 11, 224}, + dictWord{11, 11, 588}, + dictWord{11, 11, 864}, + dictWord{11, 11, 968}, + dictWord{143, 11, 160}, + dictWord{132, 0, 556}, + dictWord{137, 0, 801}, + dictWord{132, 0, 416}, + dictWord{142, 0, 372}, + dictWord{5, 0, 152}, + dictWord{5, 0, 197}, + dictWord{7, 0, 340}, + dictWord{7, 0, 867}, + dictWord{ + 10, + 0, + 548, + }, + dictWord{10, 0, 581}, + dictWord{11, 0, 6}, + dictWord{12, 0, 3}, + dictWord{12, 0, 19}, + dictWord{14, 0, 110}, + dictWord{142, 0, 289}, + dictWord{139, 0, 369}, + dictWord{7, 11, 630}, + dictWord{9, 11, 567}, + dictWord{11, 11, 150}, + dictWord{11, 11, 444}, + dictWord{141, 11, 119}, + dictWord{134, 11, 539}, + dictWord{ + 7, + 10, + 1995, + }, + dictWord{8, 10, 299}, + dictWord{11, 10, 890}, + dictWord{140, 10, 674}, + dictWord{7, 0, 34}, + dictWord{7, 0, 190}, + dictWord{8, 0, 28}, + dictWord{8, 0, 141}, + dictWord{8, 0, 444}, + dictWord{8, 0, 811}, + dictWord{9, 0, 468}, + dictWord{11, 0, 334}, + dictWord{12, 0, 24}, + dictWord{12, 0, 386}, + dictWord{140, 0, 576}, + dictWord{ + 133, + 0, + 757, + }, + dictWord{7, 0, 1553}, + dictWord{136, 0, 898}, + dictWord{133, 0, 721}, + dictWord{136, 0, 1012}, + dictWord{4, 0, 789}, + dictWord{5, 0, 647}, + dictWord{ + 135, + 0, + 1102, + }, + dictWord{132, 0, 898}, + dictWord{10, 0, 183}, + dictWord{4, 10, 238}, + dictWord{5, 10, 503}, + dictWord{6, 10, 179}, + dictWord{7, 10, 2003}, + dictWord{ + 8, + 10, + 381, + }, + dictWord{8, 10, 473}, + dictWord{9, 10, 149}, + dictWord{10, 10, 788}, + dictWord{15, 10, 45}, + dictWord{15, 10, 86}, + dictWord{20, 10, 110}, + dictWord{ + 150, + 10, + 57, + }, + dictWord{9, 0, 136}, + dictWord{19, 0, 107}, + dictWord{4, 10, 121}, + dictWord{5, 10, 156}, + dictWord{5, 10, 349}, + dictWord{10, 10, 605}, + dictWord{ + 142, + 10, + 342, + }, + dictWord{4, 11, 235}, + dictWord{135, 11, 255}, + dictWord{4, 11, 194}, + dictWord{5, 11, 584}, + dictWord{6, 11, 384}, + dictWord{7, 11, 583}, + dictWord{ + 10, + 11, + 761, + }, + dictWord{11, 11, 760}, + dictWord{139, 11, 851}, + dictWord{6, 10, 80}, + dictWord{6, 10, 1694}, + dictWord{7, 10, 173}, + dictWord{7, 10, 1974}, + dictWord{ + 9, + 10, + 547, + }, + dictWord{10, 10, 730}, + dictWord{14, 10, 18}, + dictWord{150, 10, 39}, + dictWord{4, 10, 923}, + dictWord{134, 10, 1711}, + dictWord{5, 0, 277}, + dictWord{141, 0, 247}, + dictWord{132, 0, 435}, + dictWord{133, 11, 562}, + dictWord{134, 0, 1311}, + dictWord{5, 11, 191}, + dictWord{137, 11, 271}, + dictWord{ + 132, + 10, + 595, + }, + dictWord{7, 11, 1537}, + dictWord{14, 11, 96}, + dictWord{143, 11, 73}, + dictWord{5, 0, 437}, + dictWord{7, 0, 502}, + dictWord{7, 0, 519}, + dictWord{7, 0, 1122}, + dictWord{7, 0, 1751}, + dictWord{14, 0, 211}, + dictWord{6, 10, 459}, + dictWord{7, 10, 1753}, + dictWord{7, 10, 1805}, + dictWord{8, 10, 658}, + dictWord{9, 10, 1}, + dictWord{11, 10, 959}, + dictWord{141, 10, 446}, + dictWord{6, 0, 814}, + dictWord{4, 11, 470}, + dictWord{5, 11, 473}, + dictWord{6, 11, 153}, + dictWord{7, 11, 1503}, + dictWord{7, 11, 1923}, + dictWord{10, 11, 701}, + dictWord{11, 11, 132}, + dictWord{11, 11, 168}, + dictWord{11, 11, 227}, + dictWord{11, 11, 320}, + dictWord{ + 11, + 11, + 436, + }, + dictWord{11, 11, 525}, + dictWord{11, 11, 855}, + dictWord{12, 11, 41}, + dictWord{12, 11, 286}, + dictWord{13, 11, 103}, + dictWord{13, 11, 284}, + dictWord{ + 14, + 11, + 255, + }, + dictWord{14, 11, 262}, + dictWord{15, 11, 117}, + dictWord{143, 11, 127}, + dictWord{5, 0, 265}, + dictWord{6, 0, 212}, + dictWord{135, 0, 28}, + dictWord{ + 138, + 0, + 750, + }, + dictWord{133, 11, 327}, + dictWord{6, 11, 552}, + dictWord{7, 11, 1754}, + dictWord{137, 11, 604}, + dictWord{134, 0, 2012}, + dictWord{132, 0, 702}, + dictWord{5, 11, 80}, + dictWord{6, 11, 405}, + dictWord{7, 11, 403}, + dictWord{7, 11, 1502}, + dictWord{7, 11, 1626}, + dictWord{8, 11, 456}, + dictWord{9, 11, 487}, + dictWord{9, 11, 853}, + dictWord{9, 11, 889}, + dictWord{10, 11, 309}, + dictWord{11, 11, 721}, + dictWord{11, 11, 994}, + dictWord{12, 11, 430}, + dictWord{ + 141, + 11, + 165, + }, + dictWord{5, 0, 808}, + dictWord{135, 0, 2045}, + dictWord{5, 0, 166}, + dictWord{8, 0, 739}, + dictWord{140, 0, 511}, + dictWord{134, 10, 490}, + dictWord{ + 4, + 11, + 453, + }, + dictWord{5, 11, 887}, + dictWord{6, 11, 535}, + dictWord{8, 11, 6}, + dictWord{136, 11, 543}, + dictWord{4, 0, 119}, + dictWord{5, 0, 170}, + dictWord{5, 0, 447}, + dictWord{7, 0, 1708}, + dictWord{7, 0, 1889}, + dictWord{9, 0, 357}, + dictWord{9, 0, 719}, + dictWord{12, 0, 486}, + dictWord{140, 0, 596}, + dictWord{137, 0, 500}, + dictWord{ + 7, + 10, + 250, + }, + dictWord{136, 10, 507}, + dictWord{132, 10, 158}, + dictWord{6, 0, 809}, + dictWord{134, 0, 1500}, + dictWord{9, 0, 327}, + dictWord{11, 0, 350}, + dictWord{11, 0, 831}, + dictWord{13, 0, 352}, + dictWord{4, 10, 140}, + dictWord{7, 10, 362}, + dictWord{8, 10, 209}, + dictWord{9, 10, 10}, + dictWord{9, 10, 503}, + dictWord{ + 9, + 10, + 614, + }, + dictWord{10, 10, 689}, + dictWord{11, 10, 327}, + dictWord{11, 10, 725}, + dictWord{12, 10, 252}, + dictWord{12, 10, 583}, + dictWord{13, 10, 192}, + dictWord{14, 10, 269}, + dictWord{14, 10, 356}, + dictWord{148, 10, 50}, + dictWord{135, 11, 741}, + dictWord{4, 0, 450}, + dictWord{7, 0, 1158}, + dictWord{19, 10, 1}, + dictWord{19, 10, 26}, + dictWord{150, 10, 9}, + dictWord{6, 0, 597}, + dictWord{135, 0, 1318}, + dictWord{134, 0, 1602}, + dictWord{6, 10, 228}, + dictWord{7, 10, 1341}, + dictWord{9, 10, 408}, + dictWord{138, 10, 343}, + dictWord{7, 0, 1375}, + dictWord{7, 0, 1466}, + dictWord{138, 0, 331}, + dictWord{132, 0, 754}, + dictWord{ + 132, + 10, + 557, + }, + dictWord{5, 11, 101}, + dictWord{6, 11, 88}, + dictWord{6, 11, 543}, + dictWord{7, 11, 1677}, + dictWord{9, 11, 100}, + dictWord{10, 11, 677}, + dictWord{ + 14, + 11, + 169, + }, + dictWord{14, 11, 302}, + dictWord{14, 11, 313}, + dictWord{15, 11, 48}, + dictWord{143, 11, 84}, + dictWord{134, 0, 1368}, + dictWord{4, 11, 310}, + dictWord{ + 9, + 11, + 795, + }, + dictWord{10, 11, 733}, + dictWord{11, 11, 451}, + dictWord{12, 11, 249}, + dictWord{14, 11, 115}, + dictWord{14, 11, 286}, + dictWord{143, 11, 100}, + dictWord{132, 10, 548}, + dictWord{10, 0, 557}, + dictWord{7, 10, 197}, + dictWord{8, 10, 142}, + dictWord{8, 10, 325}, + dictWord{9, 10, 150}, + dictWord{9, 10, 596}, + dictWord{10, 10, 353}, + dictWord{11, 10, 74}, + dictWord{11, 10, 315}, + dictWord{12, 10, 662}, + dictWord{12, 10, 681}, + dictWord{14, 10, 423}, + dictWord{ + 143, + 10, + 141, + }, + dictWord{133, 11, 587}, + dictWord{5, 0, 850}, + dictWord{136, 0, 799}, + dictWord{10, 0, 908}, + dictWord{12, 0, 701}, + dictWord{12, 0, 757}, + dictWord{ + 142, + 0, + 466, + }, + dictWord{4, 0, 62}, + dictWord{5, 0, 275}, + dictWord{18, 0, 19}, + dictWord{6, 10, 399}, + dictWord{6, 10, 579}, + dictWord{7, 10, 692}, + dictWord{7, 10, 846}, + dictWord{ + 7, + 10, + 1015, + }, + dictWord{7, 10, 1799}, + dictWord{8, 10, 403}, + dictWord{9, 10, 394}, + dictWord{10, 10, 133}, + dictWord{12, 10, 4}, + dictWord{12, 10, 297}, + dictWord{12, 10, 452}, + dictWord{16, 10, 81}, + dictWord{18, 10, 25}, + dictWord{21, 10, 14}, + dictWord{22, 10, 12}, + dictWord{151, 10, 18}, + dictWord{12, 0, 459}, + dictWord{ + 7, + 10, + 1546, + }, + dictWord{11, 10, 299}, + dictWord{142, 10, 407}, + dictWord{132, 10, 177}, + dictWord{132, 11, 498}, + dictWord{7, 11, 217}, + dictWord{ + 8, + 11, + 140, + }, + dictWord{138, 11, 610}, + dictWord{5, 10, 411}, + dictWord{135, 10, 653}, + dictWord{134, 0, 1802}, + dictWord{7, 10, 439}, + dictWord{10, 10, 727}, + dictWord{11, 10, 260}, + dictWord{139, 10, 684}, + dictWord{133, 11, 905}, + dictWord{11, 11, 580}, + dictWord{142, 11, 201}, + dictWord{134, 0, 1397}, + dictWord{ + 5, + 10, + 208, + }, + dictWord{7, 10, 753}, + dictWord{135, 10, 1528}, + dictWord{7, 0, 238}, + dictWord{7, 0, 2033}, + dictWord{8, 0, 120}, + dictWord{8, 0, 188}, + dictWord{8, 0, 659}, + dictWord{9, 0, 598}, + dictWord{10, 0, 466}, + dictWord{12, 0, 342}, + dictWord{12, 0, 588}, + dictWord{13, 0, 503}, + dictWord{14, 0, 246}, + dictWord{143, 0, 92}, + dictWord{135, 11, 1041}, + dictWord{4, 11, 456}, + dictWord{7, 11, 105}, + dictWord{7, 11, 358}, + dictWord{7, 11, 1637}, + dictWord{8, 11, 643}, + dictWord{139, 11, 483}, + dictWord{6, 0, 1318}, + dictWord{134, 0, 1324}, + dictWord{4, 0, 201}, + dictWord{7, 0, 1744}, + dictWord{8, 0, 602}, + dictWord{11, 0, 247}, + dictWord{11, 0, 826}, + dictWord{17, 0, 65}, + dictWord{133, 10, 242}, + dictWord{8, 0, 164}, + dictWord{146, 0, 62}, + dictWord{133, 10, 953}, + dictWord{139, 10, 802}, + dictWord{133, 0, 615}, + dictWord{7, 11, 1566}, + dictWord{8, 11, 269}, + dictWord{9, 11, 212}, + dictWord{9, 11, 718}, + dictWord{14, 11, 15}, + dictWord{14, 11, 132}, + dictWord{142, 11, 227}, + dictWord{133, 10, 290}, + dictWord{132, 10, 380}, + dictWord{5, 10, 52}, + dictWord{7, 10, 277}, + dictWord{9, 10, 368}, + dictWord{139, 10, 791}, + dictWord{ + 135, + 0, + 1243, + }, + dictWord{133, 11, 539}, + dictWord{11, 11, 919}, + dictWord{141, 11, 409}, + dictWord{136, 0, 968}, + dictWord{133, 11, 470}, + dictWord{134, 0, 882}, + dictWord{132, 0, 907}, + dictWord{5, 0, 100}, + dictWord{10, 0, 329}, + dictWord{12, 0, 416}, + dictWord{149, 0, 29}, + dictWord{10, 10, 138}, + dictWord{139, 10, 476}, + dictWord{5, 10, 725}, + dictWord{5, 10, 727}, + dictWord{6, 11, 91}, + dictWord{7, 11, 435}, + dictWord{135, 10, 1811}, + dictWord{4, 11, 16}, + dictWord{5, 11, 316}, + dictWord{5, 11, 842}, + dictWord{6, 11, 370}, + dictWord{6, 11, 1778}, + dictWord{8, 11, 166}, + dictWord{11, 11, 812}, + dictWord{12, 11, 206}, + dictWord{12, 11, 351}, + dictWord{14, 11, 418}, + dictWord{16, 11, 15}, + dictWord{16, 11, 34}, + dictWord{18, 11, 3}, + dictWord{19, 11, 3}, + dictWord{19, 11, 7}, + dictWord{20, 11, 4}, + dictWord{ + 149, + 11, + 21, + }, + dictWord{132, 0, 176}, + dictWord{5, 0, 636}, + dictWord{5, 0, 998}, + dictWord{7, 0, 9}, + dictWord{7, 0, 1508}, + dictWord{8, 0, 26}, + dictWord{9, 0, 317}, + dictWord{ + 9, + 0, + 358, + }, + dictWord{10, 0, 210}, + dictWord{10, 0, 292}, + dictWord{10, 0, 533}, + dictWord{11, 0, 555}, + dictWord{12, 0, 526}, + dictWord{12, 0, 607}, + dictWord{ + 13, + 0, + 263, + }, + dictWord{13, 0, 459}, + dictWord{142, 0, 271}, + dictWord{6, 0, 256}, + dictWord{8, 0, 265}, + dictWord{4, 10, 38}, + dictWord{7, 10, 307}, + dictWord{7, 10, 999}, + dictWord{7, 10, 1481}, + dictWord{7, 10, 1732}, + dictWord{7, 10, 1738}, + dictWord{9, 10, 414}, + dictWord{11, 10, 316}, + dictWord{12, 10, 52}, + dictWord{13, 10, 420}, + dictWord{147, 10, 100}, + dictWord{135, 10, 1296}, + dictWord{4, 11, 611}, + dictWord{133, 11, 606}, + dictWord{4, 0, 643}, + dictWord{142, 11, 21}, + dictWord{ + 133, + 11, + 715, + }, + dictWord{133, 10, 723}, + dictWord{6, 0, 610}, + dictWord{135, 11, 597}, + dictWord{10, 0, 127}, + dictWord{141, 0, 27}, + dictWord{6, 0, 1995}, + dictWord{ + 6, + 0, + 2001, + }, + dictWord{8, 0, 119}, + dictWord{136, 0, 973}, + dictWord{4, 11, 149}, + dictWord{138, 11, 368}, + dictWord{12, 0, 522}, + dictWord{4, 11, 154}, + dictWord{ + 5, + 10, + 109, + }, + dictWord{6, 10, 1784}, + dictWord{7, 11, 1134}, + dictWord{7, 10, 1895}, + dictWord{8, 11, 105}, + dictWord{12, 10, 296}, + dictWord{140, 10, 302}, + dictWord{4, 11, 31}, + dictWord{6, 11, 429}, + dictWord{7, 11, 962}, + dictWord{9, 11, 458}, + dictWord{139, 11, 691}, + dictWord{10, 0, 553}, + dictWord{11, 0, 876}, + dictWord{13, 0, 193}, + dictWord{13, 0, 423}, + dictWord{14, 0, 166}, + dictWord{19, 0, 84}, + dictWord{4, 11, 312}, + dictWord{5, 10, 216}, + dictWord{7, 10, 1879}, + dictWord{ + 9, + 10, + 141, + }, + dictWord{9, 10, 270}, + dictWord{9, 10, 679}, + dictWord{10, 10, 159}, + dictWord{11, 10, 197}, + dictWord{12, 10, 538}, + dictWord{12, 10, 559}, + dictWord{14, 10, 144}, + dictWord{14, 10, 167}, + dictWord{143, 10, 67}, + dictWord{134, 0, 1582}, + dictWord{7, 0, 1578}, + dictWord{135, 11, 1578}, + dictWord{ + 137, + 10, + 81, + }, + dictWord{132, 11, 236}, + dictWord{134, 10, 391}, + dictWord{134, 0, 795}, + dictWord{7, 10, 322}, + dictWord{136, 10, 249}, + dictWord{5, 11, 836}, + dictWord{ + 5, + 11, + 857, + }, + dictWord{6, 11, 1680}, + dictWord{7, 11, 59}, + dictWord{147, 11, 53}, + dictWord{135, 0, 432}, + dictWord{10, 11, 68}, + dictWord{139, 11, 494}, + dictWord{4, 11, 81}, + dictWord{139, 11, 867}, + dictWord{7, 0, 126}, + dictWord{136, 0, 84}, + dictWord{142, 11, 280}, + dictWord{5, 11, 282}, + dictWord{8, 11, 650}, + dictWord{ + 9, + 11, + 295, + }, + dictWord{9, 11, 907}, + dictWord{138, 11, 443}, + dictWord{136, 0, 790}, + dictWord{5, 10, 632}, + dictWord{138, 10, 526}, + dictWord{6, 0, 64}, + dictWord{12, 0, 377}, + dictWord{13, 0, 309}, + dictWord{14, 0, 141}, + dictWord{14, 0, 429}, + dictWord{14, 11, 141}, + dictWord{142, 11, 429}, + dictWord{134, 0, 1529}, + dictWord{6, 0, 321}, + dictWord{7, 0, 1857}, + dictWord{9, 0, 530}, + dictWord{19, 0, 99}, + dictWord{7, 10, 948}, + dictWord{7, 10, 1042}, + dictWord{8, 10, 235}, + dictWord{ + 8, + 10, + 461, + }, + dictWord{9, 10, 453}, + dictWord{10, 10, 354}, + dictWord{145, 10, 77}, + dictWord{7, 0, 1104}, + dictWord{11, 0, 269}, + dictWord{11, 0, 539}, + dictWord{ + 11, + 0, + 627, + }, + dictWord{11, 0, 706}, + dictWord{11, 0, 975}, + dictWord{12, 0, 248}, + dictWord{12, 0, 434}, + dictWord{12, 0, 600}, + dictWord{12, 0, 622}, + dictWord{ + 13, + 0, + 297, + }, + dictWord{13, 0, 485}, + dictWord{14, 0, 69}, + dictWord{14, 0, 409}, + dictWord{143, 0, 108}, + dictWord{4, 10, 362}, + dictWord{7, 10, 52}, + dictWord{7, 10, 303}, + dictWord{10, 11, 70}, + dictWord{12, 11, 26}, + dictWord{14, 11, 17}, + dictWord{14, 11, 178}, + dictWord{15, 11, 34}, + dictWord{149, 11, 12}, + dictWord{11, 0, 977}, + dictWord{141, 0, 507}, + dictWord{9, 0, 34}, + dictWord{139, 0, 484}, + dictWord{5, 10, 196}, + dictWord{6, 10, 486}, + dictWord{7, 10, 212}, + dictWord{8, 10, 309}, + dictWord{136, 10, 346}, + dictWord{6, 0, 1700}, + dictWord{7, 0, 26}, + dictWord{7, 0, 293}, + dictWord{7, 0, 382}, + dictWord{7, 0, 1026}, + dictWord{7, 0, 1087}, + dictWord{ + 7, + 0, + 2027, + }, + dictWord{8, 0, 24}, + dictWord{8, 0, 114}, + dictWord{8, 0, 252}, + dictWord{8, 0, 727}, + dictWord{8, 0, 729}, + dictWord{9, 0, 30}, + dictWord{9, 0, 199}, + dictWord{ + 9, + 0, + 231, + }, + dictWord{9, 0, 251}, + dictWord{9, 0, 334}, + dictWord{9, 0, 361}, + dictWord{9, 0, 712}, + dictWord{10, 0, 55}, + dictWord{10, 0, 60}, + dictWord{10, 0, 232}, + dictWord{ + 10, + 0, + 332, + }, + dictWord{10, 0, 384}, + dictWord{10, 0, 396}, + dictWord{10, 0, 504}, + dictWord{10, 0, 542}, + dictWord{10, 0, 652}, + dictWord{11, 0, 20}, + dictWord{11, 0, 48}, + dictWord{11, 0, 207}, + dictWord{11, 0, 291}, + dictWord{11, 0, 298}, + dictWord{11, 0, 342}, + dictWord{11, 0, 365}, + dictWord{11, 0, 394}, + dictWord{11, 0, 620}, + dictWord{11, 0, 705}, + dictWord{11, 0, 1017}, + dictWord{12, 0, 123}, + dictWord{12, 0, 340}, + dictWord{12, 0, 406}, + dictWord{12, 0, 643}, + dictWord{13, 0, 61}, + dictWord{ + 13, + 0, + 269, + }, + dictWord{13, 0, 311}, + dictWord{13, 0, 319}, + dictWord{13, 0, 486}, + dictWord{14, 0, 234}, + dictWord{15, 0, 62}, + dictWord{15, 0, 85}, + dictWord{16, 0, 71}, + dictWord{18, 0, 119}, + dictWord{20, 0, 105}, + dictWord{135, 10, 1912}, + dictWord{4, 11, 71}, + dictWord{5, 11, 376}, + dictWord{7, 11, 119}, + dictWord{138, 11, 665}, + dictWord{10, 0, 918}, + dictWord{10, 0, 926}, + dictWord{4, 10, 686}, + dictWord{136, 11, 55}, + dictWord{138, 10, 625}, + dictWord{136, 10, 706}, + dictWord{ + 132, + 11, + 479, + }, + dictWord{4, 10, 30}, + dictWord{133, 10, 43}, + dictWord{6, 0, 379}, + dictWord{7, 0, 270}, + dictWord{8, 0, 176}, + dictWord{8, 0, 183}, + dictWord{9, 0, 432}, + dictWord{ + 9, + 0, + 661, + }, + dictWord{12, 0, 247}, + dictWord{12, 0, 617}, + dictWord{18, 0, 125}, + dictWord{7, 11, 607}, + dictWord{8, 11, 99}, + dictWord{152, 11, 4}, + dictWord{ + 5, + 0, + 792, + }, + dictWord{133, 0, 900}, + dictWord{4, 11, 612}, + dictWord{133, 11, 561}, + dictWord{4, 11, 41}, + dictWord{4, 10, 220}, + dictWord{5, 11, 74}, + dictWord{ + 7, + 10, + 1535, + }, + dictWord{7, 11, 1627}, + dictWord{11, 11, 871}, + dictWord{140, 11, 619}, + dictWord{135, 0, 1920}, + dictWord{7, 11, 94}, + dictWord{11, 11, 329}, + dictWord{11, 11, 965}, + dictWord{12, 11, 241}, + dictWord{14, 11, 354}, + dictWord{15, 11, 22}, + dictWord{148, 11, 63}, + dictWord{9, 11, 209}, + dictWord{137, 11, 300}, + dictWord{134, 0, 771}, + dictWord{135, 0, 1979}, + dictWord{4, 0, 901}, + dictWord{133, 0, 776}, + dictWord{142, 0, 254}, + dictWord{133, 11, 98}, + dictWord{ + 9, + 11, + 16, + }, + dictWord{141, 11, 386}, + dictWord{133, 11, 984}, + dictWord{4, 11, 182}, + dictWord{6, 11, 205}, + dictWord{135, 11, 220}, + dictWord{7, 10, 1725}, + dictWord{ + 7, + 10, + 1774, + }, + dictWord{138, 10, 393}, + dictWord{5, 10, 263}, + dictWord{134, 10, 414}, + dictWord{4, 11, 42}, + dictWord{9, 11, 205}, + dictWord{9, 11, 786}, + dictWord{138, 11, 659}, + dictWord{14, 0, 140}, + dictWord{148, 0, 41}, + dictWord{8, 0, 440}, + dictWord{10, 0, 359}, + dictWord{6, 10, 178}, + dictWord{6, 11, 289}, + dictWord{ + 6, + 10, + 1750, + }, + dictWord{7, 11, 1670}, + dictWord{9, 10, 690}, + dictWord{10, 10, 155}, + dictWord{10, 10, 373}, + dictWord{11, 10, 698}, + dictWord{12, 11, 57}, + dictWord{13, 10, 155}, + dictWord{20, 10, 93}, + dictWord{151, 11, 4}, + dictWord{4, 0, 37}, + dictWord{5, 0, 334}, + dictWord{7, 0, 1253}, + dictWord{151, 11, 25}, + dictWord{ + 4, + 0, + 508, + }, + dictWord{4, 11, 635}, + dictWord{5, 10, 97}, + dictWord{137, 10, 393}, + dictWord{139, 11, 533}, + dictWord{4, 0, 640}, + dictWord{133, 0, 513}, + dictWord{ + 134, + 10, + 1639, + }, + dictWord{132, 11, 371}, + dictWord{4, 11, 272}, + dictWord{7, 11, 836}, + dictWord{7, 11, 1651}, + dictWord{145, 11, 89}, + dictWord{5, 11, 825}, + dictWord{6, 11, 444}, + dictWord{6, 11, 1640}, + dictWord{136, 11, 308}, + dictWord{4, 10, 191}, + dictWord{7, 10, 934}, + dictWord{8, 10, 647}, + dictWord{145, 10, 97}, + dictWord{12, 0, 246}, + dictWord{15, 0, 162}, + dictWord{19, 0, 64}, + dictWord{20, 0, 8}, + dictWord{20, 0, 95}, + dictWord{22, 0, 24}, + dictWord{152, 0, 17}, + dictWord{4, 0, 533}, + dictWord{5, 10, 165}, + dictWord{9, 10, 346}, + dictWord{138, 10, 655}, + dictWord{5, 11, 737}, + dictWord{139, 10, 885}, + dictWord{133, 10, 877}, + dictWord{ + 8, + 10, + 128, + }, + dictWord{139, 10, 179}, + dictWord{137, 11, 307}, + dictWord{140, 0, 752}, + dictWord{133, 0, 920}, + dictWord{135, 0, 1048}, + dictWord{5, 0, 153}, + dictWord{ + 6, + 0, + 580, + }, + dictWord{6, 10, 1663}, + dictWord{7, 10, 132}, + dictWord{7, 10, 1154}, + dictWord{7, 10, 1415}, + dictWord{7, 10, 1507}, + dictWord{12, 10, 493}, + dictWord{15, 10, 105}, + dictWord{151, 10, 15}, + dictWord{5, 10, 459}, + dictWord{7, 10, 1073}, + dictWord{8, 10, 241}, + dictWord{136, 10, 334}, + dictWord{138, 0, 391}, + dictWord{135, 0, 1952}, + dictWord{133, 11, 525}, + dictWord{8, 11, 641}, + dictWord{11, 11, 388}, + dictWord{140, 11, 580}, + dictWord{142, 0, 126}, + dictWord{ + 134, + 0, + 640, + }, + dictWord{132, 0, 483}, + dictWord{7, 0, 1616}, + dictWord{9, 0, 69}, + dictWord{6, 10, 324}, + dictWord{6, 10, 520}, + dictWord{7, 10, 338}, + dictWord{ + 7, + 10, + 1729, + }, + dictWord{8, 10, 228}, + dictWord{139, 10, 750}, + dictWord{5, 11, 493}, + dictWord{134, 11, 528}, + dictWord{135, 0, 734}, + dictWord{4, 11, 174}, + dictWord{135, 11, 911}, + dictWord{138, 0, 480}, + dictWord{9, 0, 495}, + dictWord{146, 0, 104}, + dictWord{135, 10, 705}, + dictWord{9, 0, 472}, + dictWord{4, 10, 73}, + dictWord{6, 10, 612}, + dictWord{7, 10, 927}, + dictWord{7, 10, 1330}, + dictWord{7, 10, 1822}, + dictWord{8, 10, 217}, + dictWord{9, 10, 765}, + dictWord{9, 10, 766}, + dictWord{10, 10, 408}, + dictWord{11, 10, 51}, + dictWord{11, 10, 793}, + dictWord{12, 10, 266}, + dictWord{15, 10, 158}, + dictWord{20, 10, 89}, + dictWord{150, 10, 32}, + dictWord{7, 11, 548}, + dictWord{137, 11, 58}, + dictWord{4, 11, 32}, + dictWord{5, 11, 215}, + dictWord{6, 11, 269}, + dictWord{7, 11, 1782}, + dictWord{7, 11, 1892}, + dictWord{10, 11, 16}, + dictWord{11, 11, 822}, + dictWord{11, 11, 954}, + dictWord{141, 11, 481}, + dictWord{132, 0, 874}, + dictWord{9, 0, 229}, + dictWord{5, 10, 389}, + dictWord{136, 10, 636}, + dictWord{7, 11, 1749}, + dictWord{136, 11, 477}, + dictWord{134, 0, 948}, + dictWord{5, 11, 308}, + dictWord{135, 11, 1088}, + dictWord{ + 4, + 0, + 748, + }, + dictWord{139, 0, 1009}, + dictWord{136, 10, 21}, + dictWord{6, 0, 555}, + dictWord{135, 0, 485}, + dictWord{5, 11, 126}, + dictWord{8, 11, 297}, + dictWord{ + 9, + 11, + 366, + }, + dictWord{9, 11, 445}, + dictWord{12, 11, 53}, + dictWord{12, 11, 374}, + dictWord{141, 11, 492}, + dictWord{7, 11, 1551}, + dictWord{139, 11, 361}, + dictWord{136, 0, 193}, + dictWord{136, 0, 472}, + dictWord{8, 0, 653}, + dictWord{13, 0, 93}, + dictWord{147, 0, 14}, + dictWord{132, 0, 984}, + dictWord{132, 11, 175}, + dictWord{5, 0, 172}, + dictWord{6, 0, 1971}, + dictWord{132, 11, 685}, + dictWord{149, 11, 8}, + dictWord{133, 11, 797}, + dictWord{13, 0, 83}, + dictWord{5, 10, 189}, + dictWord{ + 7, + 10, + 442, + }, + dictWord{7, 10, 443}, + dictWord{8, 10, 281}, + dictWord{12, 10, 174}, + dictWord{141, 10, 261}, + dictWord{134, 0, 1568}, + dictWord{133, 11, 565}, + dictWord{139, 0, 384}, + dictWord{133, 0, 260}, + dictWord{7, 0, 758}, + dictWord{7, 0, 880}, + dictWord{7, 0, 1359}, + dictWord{9, 0, 164}, + dictWord{9, 0, 167}, + dictWord{ + 10, + 0, + 156, + }, + dictWord{10, 0, 588}, + dictWord{12, 0, 101}, + dictWord{14, 0, 48}, + dictWord{15, 0, 70}, + dictWord{6, 10, 2}, + dictWord{7, 10, 1262}, + dictWord{ + 7, + 10, + 1737, + }, + dictWord{8, 10, 22}, + dictWord{8, 10, 270}, + dictWord{8, 10, 612}, + dictWord{9, 10, 312}, + dictWord{9, 10, 436}, + dictWord{10, 10, 311}, + dictWord{ + 10, + 10, + 623, + }, + dictWord{11, 10, 72}, + dictWord{11, 10, 330}, + dictWord{11, 10, 455}, + dictWord{12, 10, 321}, + dictWord{12, 10, 504}, + dictWord{12, 10, 530}, + dictWord{ + 12, + 10, + 543, + }, + dictWord{13, 10, 17}, + dictWord{13, 10, 156}, + dictWord{13, 10, 334}, + dictWord{17, 10, 60}, + dictWord{148, 10, 64}, + dictWord{4, 11, 252}, + dictWord{ + 7, + 11, + 1068, + }, + dictWord{10, 11, 434}, + dictWord{11, 11, 228}, + dictWord{11, 11, 426}, + dictWord{13, 11, 231}, + dictWord{18, 11, 106}, + dictWord{148, 11, 87}, + dictWord{7, 10, 354}, + dictWord{10, 10, 410}, + dictWord{139, 10, 815}, + dictWord{6, 0, 367}, + dictWord{7, 10, 670}, + dictWord{7, 10, 1327}, + dictWord{8, 10, 411}, + dictWord{8, 10, 435}, + dictWord{9, 10, 653}, + dictWord{9, 10, 740}, + dictWord{10, 10, 385}, + dictWord{11, 10, 222}, + dictWord{11, 10, 324}, + dictWord{11, 10, 829}, + dictWord{140, 10, 611}, + dictWord{7, 0, 1174}, + dictWord{6, 10, 166}, + dictWord{135, 10, 374}, + dictWord{146, 0, 121}, + dictWord{132, 0, 828}, + dictWord{ + 5, + 11, + 231, + }, + dictWord{138, 11, 509}, + dictWord{7, 11, 601}, + dictWord{9, 11, 277}, + dictWord{9, 11, 674}, + dictWord{10, 11, 178}, + dictWord{10, 11, 257}, + dictWord{ + 10, + 11, + 418, + }, + dictWord{11, 11, 531}, + dictWord{11, 11, 544}, + dictWord{11, 11, 585}, + dictWord{12, 11, 113}, + dictWord{12, 11, 475}, + dictWord{13, 11, 99}, + dictWord{142, 11, 428}, + dictWord{134, 0, 1541}, + dictWord{135, 11, 1779}, + dictWord{5, 0, 343}, + dictWord{134, 10, 398}, + dictWord{135, 10, 50}, + dictWord{ + 135, + 11, + 1683, + }, + dictWord{4, 0, 440}, + dictWord{7, 0, 57}, + dictWord{8, 0, 167}, + dictWord{8, 0, 375}, + dictWord{9, 0, 82}, + dictWord{9, 0, 561}, + dictWord{9, 0, 744}, + dictWord{ + 10, + 0, + 620, + }, + dictWord{137, 11, 744}, + dictWord{134, 0, 926}, + dictWord{6, 10, 517}, + dictWord{7, 10, 1159}, + dictWord{10, 10, 621}, + dictWord{139, 10, 192}, + dictWord{137, 0, 827}, + dictWord{8, 0, 194}, + dictWord{136, 0, 756}, + dictWord{10, 10, 223}, + dictWord{139, 10, 645}, + dictWord{7, 10, 64}, + dictWord{ + 136, + 10, + 245, + }, + dictWord{4, 11, 399}, + dictWord{5, 11, 119}, + dictWord{5, 11, 494}, + dictWord{7, 11, 751}, + dictWord{137, 11, 556}, + dictWord{132, 0, 808}, + dictWord{ + 135, + 0, + 22, + }, + dictWord{7, 10, 1763}, + dictWord{140, 10, 310}, + dictWord{5, 0, 639}, + dictWord{7, 0, 1249}, + dictWord{11, 0, 896}, + dictWord{134, 11, 584}, + dictWord{ + 134, + 0, + 1614, + }, + dictWord{135, 0, 860}, + dictWord{135, 11, 1121}, + dictWord{5, 10, 129}, + dictWord{6, 10, 61}, + dictWord{135, 10, 947}, + dictWord{4, 0, 102}, + dictWord{ + 7, + 0, + 815, + }, + dictWord{7, 0, 1699}, + dictWord{139, 0, 964}, + dictWord{13, 10, 505}, + dictWord{141, 10, 506}, + dictWord{139, 10, 1000}, + dictWord{ + 132, + 11, + 679, + }, + dictWord{132, 0, 899}, + dictWord{132, 0, 569}, + dictWord{5, 11, 694}, + dictWord{137, 11, 714}, + dictWord{136, 0, 795}, + dictWord{6, 0, 2045}, + dictWord{ + 139, + 11, + 7, + }, + dictWord{6, 0, 52}, + dictWord{9, 0, 104}, + dictWord{9, 0, 559}, + dictWord{12, 0, 308}, + dictWord{147, 0, 87}, + dictWord{4, 0, 301}, + dictWord{132, 0, 604}, + dictWord{133, 10, 637}, + dictWord{136, 0, 779}, + dictWord{5, 11, 143}, + dictWord{5, 11, 769}, + dictWord{6, 11, 1760}, + dictWord{7, 11, 682}, + dictWord{7, 11, 1992}, + dictWord{136, 11, 736}, + dictWord{137, 10, 590}, + dictWord{147, 0, 32}, + dictWord{137, 11, 527}, + dictWord{5, 10, 280}, + dictWord{135, 10, 1226}, + dictWord{134, 0, 494}, + dictWord{6, 0, 677}, + dictWord{6, 0, 682}, + dictWord{134, 0, 1044}, + dictWord{133, 10, 281}, + dictWord{135, 10, 1064}, + dictWord{7, 0, 508}, + dictWord{133, 11, 860}, + dictWord{6, 11, 422}, + dictWord{7, 11, 0}, + dictWord{7, 11, 1544}, + dictWord{9, 11, 577}, + dictWord{11, 11, 990}, + dictWord{12, 11, 141}, + dictWord{12, 11, 453}, + dictWord{13, 11, 47}, + dictWord{141, 11, 266}, + dictWord{134, 0, 1014}, + dictWord{5, 11, 515}, + dictWord{137, 11, 131}, + dictWord{ + 134, + 0, + 957, + }, + dictWord{132, 11, 646}, + dictWord{6, 0, 310}, + dictWord{7, 0, 1849}, + dictWord{8, 0, 72}, + dictWord{8, 0, 272}, + dictWord{8, 0, 431}, + dictWord{9, 0, 12}, + dictWord{ + 9, + 0, + 376, + }, + dictWord{10, 0, 563}, + dictWord{10, 0, 630}, + dictWord{10, 0, 796}, + dictWord{10, 0, 810}, + dictWord{11, 0, 367}, + dictWord{11, 0, 599}, + dictWord{ + 11, + 0, + 686, + }, + dictWord{140, 0, 672}, + dictWord{7, 0, 570}, + dictWord{4, 11, 396}, + dictWord{7, 10, 120}, + dictWord{7, 11, 728}, + dictWord{8, 10, 489}, + dictWord{9, 11, 117}, + dictWord{9, 10, 319}, + dictWord{10, 10, 820}, + dictWord{11, 10, 1004}, + dictWord{12, 10, 379}, + dictWord{12, 10, 679}, + dictWord{13, 10, 117}, + dictWord{ + 13, + 11, + 202, + }, + dictWord{13, 10, 412}, + dictWord{14, 10, 25}, + dictWord{15, 10, 52}, + dictWord{15, 10, 161}, + dictWord{16, 10, 47}, + dictWord{20, 11, 51}, + dictWord{ + 149, + 10, + 2, + }, + dictWord{6, 11, 121}, + dictWord{6, 11, 124}, + dictWord{6, 11, 357}, + dictWord{7, 11, 1138}, + dictWord{7, 11, 1295}, + dictWord{8, 11, 162}, + dictWord{ + 139, + 11, + 655, + }, + dictWord{8, 0, 449}, + dictWord{4, 10, 937}, + dictWord{5, 10, 801}, + dictWord{136, 11, 449}, + dictWord{139, 11, 958}, + dictWord{6, 0, 181}, + dictWord{ + 7, + 0, + 537, + }, + dictWord{8, 0, 64}, + dictWord{9, 0, 127}, + dictWord{10, 0, 496}, + dictWord{12, 0, 510}, + dictWord{141, 0, 384}, + dictWord{138, 11, 253}, + dictWord{4, 0, 244}, + dictWord{135, 0, 233}, + dictWord{133, 11, 237}, + dictWord{132, 10, 365}, + dictWord{6, 0, 1650}, + dictWord{10, 0, 702}, + dictWord{139, 0, 245}, + dictWord{ + 5, + 10, + 7, + }, + dictWord{139, 10, 774}, + dictWord{13, 0, 463}, + dictWord{20, 0, 49}, + dictWord{13, 11, 463}, + dictWord{148, 11, 49}, + dictWord{4, 10, 734}, + dictWord{ + 5, + 10, + 662, + }, + dictWord{134, 10, 430}, + dictWord{4, 10, 746}, + dictWord{135, 10, 1090}, + dictWord{5, 10, 360}, + dictWord{136, 10, 237}, + dictWord{137, 0, 338}, + dictWord{143, 11, 10}, + dictWord{7, 11, 571}, + dictWord{138, 11, 366}, + dictWord{134, 0, 1279}, + dictWord{9, 11, 513}, + dictWord{10, 11, 22}, + dictWord{10, 11, 39}, + dictWord{12, 11, 122}, + dictWord{140, 11, 187}, + dictWord{133, 0, 896}, + dictWord{146, 0, 178}, + dictWord{134, 0, 695}, + dictWord{137, 0, 808}, + dictWord{ + 134, + 11, + 587, + }, + dictWord{7, 11, 107}, + dictWord{7, 11, 838}, + dictWord{8, 11, 550}, + dictWord{138, 11, 401}, + dictWord{7, 0, 1117}, + dictWord{136, 0, 539}, + dictWord{ + 4, + 10, + 277, + }, + dictWord{5, 10, 608}, + dictWord{6, 10, 493}, + dictWord{7, 10, 457}, + dictWord{140, 10, 384}, + dictWord{133, 11, 768}, + dictWord{12, 0, 257}, + dictWord{ + 7, + 10, + 27, + }, + dictWord{135, 10, 316}, + dictWord{140, 0, 1003}, + dictWord{4, 0, 207}, + dictWord{5, 0, 586}, + dictWord{5, 0, 676}, + dictWord{6, 0, 448}, + dictWord{ + 8, + 0, + 244, + }, + dictWord{11, 0, 1}, + dictWord{13, 0, 3}, + dictWord{16, 0, 54}, + dictWord{17, 0, 4}, + dictWord{18, 0, 13}, + dictWord{133, 10, 552}, + dictWord{4, 10, 401}, + dictWord{ + 137, + 10, + 264, + }, + dictWord{5, 0, 516}, + dictWord{7, 0, 1883}, + dictWord{135, 11, 1883}, + dictWord{12, 0, 960}, + dictWord{132, 11, 894}, + dictWord{5, 0, 4}, + dictWord{ + 5, + 0, + 810, + }, + dictWord{6, 0, 13}, + dictWord{6, 0, 538}, + dictWord{6, 0, 1690}, + dictWord{6, 0, 1726}, + dictWord{7, 0, 499}, + dictWord{7, 0, 1819}, + dictWord{8, 0, 148}, + dictWord{ + 8, + 0, + 696, + }, + dictWord{8, 0, 791}, + dictWord{12, 0, 125}, + dictWord{143, 0, 9}, + dictWord{135, 0, 1268}, + dictWord{11, 0, 30}, + dictWord{14, 0, 315}, + dictWord{ + 9, + 10, + 543, + }, + dictWord{10, 10, 524}, + dictWord{12, 10, 524}, + dictWord{16, 10, 18}, + dictWord{20, 10, 26}, + dictWord{148, 10, 65}, + dictWord{6, 0, 748}, + dictWord{ + 4, + 10, + 205, + }, + dictWord{5, 10, 623}, + dictWord{7, 10, 104}, + dictWord{136, 10, 519}, + dictWord{11, 0, 542}, + dictWord{139, 0, 852}, + dictWord{140, 0, 6}, + dictWord{ + 132, + 0, + 848, + }, + dictWord{7, 0, 1385}, + dictWord{11, 0, 582}, + dictWord{11, 0, 650}, + dictWord{11, 0, 901}, + dictWord{11, 0, 949}, + dictWord{12, 0, 232}, + dictWord{12, 0, 236}, + dictWord{13, 0, 413}, + dictWord{13, 0, 501}, + dictWord{18, 0, 116}, + dictWord{7, 10, 579}, + dictWord{9, 10, 41}, + dictWord{9, 10, 244}, + dictWord{9, 10, 669}, + dictWord{10, 10, 5}, + dictWord{11, 10, 861}, + dictWord{11, 10, 951}, + dictWord{139, 10, 980}, + dictWord{4, 0, 945}, + dictWord{6, 0, 1811}, + dictWord{6, 0, 1845}, + dictWord{ + 6, + 0, + 1853, + }, + dictWord{6, 0, 1858}, + dictWord{8, 0, 862}, + dictWord{12, 0, 782}, + dictWord{12, 0, 788}, + dictWord{18, 0, 160}, + dictWord{148, 0, 117}, + dictWord{ + 132, + 10, + 717, + }, + dictWord{4, 0, 925}, + dictWord{5, 0, 803}, + dictWord{8, 0, 698}, + dictWord{138, 0, 828}, + dictWord{134, 0, 1416}, + dictWord{132, 0, 610}, + dictWord{ + 139, + 0, + 992, + }, + dictWord{6, 0, 878}, + dictWord{134, 0, 1477}, + dictWord{135, 0, 1847}, + dictWord{138, 11, 531}, + dictWord{137, 11, 539}, + dictWord{134, 11, 272}, + dictWord{133, 0, 383}, + dictWord{134, 0, 1404}, + dictWord{132, 10, 489}, + dictWord{4, 11, 9}, + dictWord{5, 11, 128}, + dictWord{7, 11, 368}, + dictWord{ + 11, + 11, + 480, + }, + dictWord{148, 11, 3}, + dictWord{136, 0, 986}, + dictWord{9, 0, 660}, + dictWord{138, 0, 347}, + dictWord{135, 10, 892}, + dictWord{136, 11, 682}, + dictWord{ + 7, + 0, + 572, + }, + dictWord{9, 0, 592}, + dictWord{11, 0, 680}, + dictWord{12, 0, 356}, + dictWord{140, 0, 550}, + dictWord{7, 0, 1411}, + dictWord{138, 11, 527}, + dictWord{ + 4, + 11, + 2, + }, + dictWord{7, 11, 545}, + dictWord{135, 11, 894}, + dictWord{137, 10, 473}, + dictWord{11, 0, 64}, + dictWord{7, 11, 481}, + dictWord{7, 10, 819}, + dictWord{9, 10, 26}, + dictWord{9, 10, 392}, + dictWord{9, 11, 792}, + dictWord{10, 10, 152}, + dictWord{10, 10, 226}, + dictWord{12, 10, 276}, + dictWord{12, 10, 426}, + dictWord{ + 12, + 10, + 589, + }, + dictWord{13, 10, 460}, + dictWord{15, 10, 97}, + dictWord{19, 10, 48}, + dictWord{148, 10, 104}, + dictWord{135, 10, 51}, + dictWord{136, 11, 445}, + dictWord{136, 11, 646}, + dictWord{135, 0, 606}, + dictWord{132, 10, 674}, + dictWord{6, 0, 1829}, + dictWord{134, 0, 1830}, + dictWord{132, 10, 770}, + dictWord{ + 5, + 10, + 79, + }, + dictWord{7, 10, 1027}, + dictWord{7, 10, 1477}, + dictWord{139, 10, 52}, + dictWord{5, 11, 530}, + dictWord{142, 11, 113}, + dictWord{134, 10, 1666}, + dictWord{ + 7, + 0, + 748, + }, + dictWord{139, 0, 700}, + dictWord{134, 10, 195}, + dictWord{133, 10, 789}, + dictWord{9, 0, 87}, + dictWord{10, 0, 365}, + dictWord{4, 10, 251}, + dictWord{ + 4, + 10, + 688, + }, + dictWord{7, 10, 513}, + dictWord{135, 10, 1284}, + dictWord{136, 11, 111}, + dictWord{133, 0, 127}, + dictWord{6, 0, 198}, + dictWord{140, 0, 83}, + dictWord{133, 11, 556}, + dictWord{133, 10, 889}, + dictWord{4, 10, 160}, + dictWord{5, 10, 330}, + dictWord{7, 10, 1434}, + dictWord{136, 10, 174}, + dictWord{5, 0, 276}, + dictWord{6, 0, 55}, + dictWord{7, 0, 1369}, + dictWord{138, 0, 864}, + dictWord{8, 11, 16}, + dictWord{140, 11, 568}, + dictWord{6, 0, 1752}, + dictWord{136, 0, 726}, + dictWord{135, 0, 1066}, + dictWord{133, 0, 764}, + dictWord{6, 11, 186}, + dictWord{137, 11, 426}, + dictWord{11, 0, 683}, + dictWord{139, 11, 683}, + dictWord{ + 6, + 0, + 309, + }, + dictWord{7, 0, 331}, + dictWord{138, 0, 550}, + dictWord{133, 10, 374}, + dictWord{6, 0, 1212}, + dictWord{6, 0, 1852}, + dictWord{7, 0, 1062}, + dictWord{ + 8, + 0, + 874, + }, + dictWord{8, 0, 882}, + dictWord{138, 0, 936}, + dictWord{132, 11, 585}, + dictWord{134, 0, 1364}, + dictWord{7, 0, 986}, + dictWord{133, 10, 731}, + dictWord{ + 6, + 0, + 723, + }, + dictWord{6, 0, 1408}, + dictWord{138, 0, 381}, + dictWord{135, 0, 1573}, + dictWord{134, 0, 1025}, + dictWord{4, 10, 626}, + dictWord{5, 10, 642}, + dictWord{ + 6, + 10, + 425, + }, + dictWord{10, 10, 202}, + dictWord{139, 10, 141}, + dictWord{4, 11, 93}, + dictWord{5, 11, 252}, + dictWord{6, 11, 229}, + dictWord{7, 11, 291}, + dictWord{ + 9, + 11, + 550, + }, + dictWord{139, 11, 644}, + dictWord{137, 11, 749}, + dictWord{137, 11, 162}, + dictWord{132, 11, 381}, + dictWord{135, 0, 1559}, + dictWord{ + 6, + 0, + 194, + }, + dictWord{7, 0, 133}, + dictWord{10, 0, 493}, + dictWord{10, 0, 570}, + dictWord{139, 0, 664}, + dictWord{5, 0, 24}, + dictWord{5, 0, 569}, + dictWord{6, 0, 3}, + dictWord{ + 6, + 0, + 119, + }, + dictWord{6, 0, 143}, + dictWord{6, 0, 440}, + dictWord{7, 0, 295}, + dictWord{7, 0, 599}, + dictWord{7, 0, 1686}, + dictWord{7, 0, 1854}, + dictWord{8, 0, 424}, + dictWord{ + 9, + 0, + 43, + }, + dictWord{9, 0, 584}, + dictWord{9, 0, 760}, + dictWord{10, 0, 148}, + dictWord{10, 0, 328}, + dictWord{11, 0, 159}, + dictWord{11, 0, 253}, + dictWord{11, 0, 506}, + dictWord{12, 0, 487}, + dictWord{140, 0, 531}, + dictWord{6, 0, 661}, + dictWord{134, 0, 1517}, + dictWord{136, 10, 835}, + dictWord{151, 10, 17}, + dictWord{5, 0, 14}, + dictWord{5, 0, 892}, + dictWord{6, 0, 283}, + dictWord{7, 0, 234}, + dictWord{136, 0, 537}, + dictWord{139, 0, 541}, + dictWord{4, 0, 126}, + dictWord{8, 0, 635}, + dictWord{ + 147, + 0, + 34, + }, + dictWord{4, 0, 316}, + dictWord{4, 0, 495}, + dictWord{135, 0, 1561}, + dictWord{4, 11, 187}, + dictWord{5, 11, 184}, + dictWord{5, 11, 690}, + dictWord{ + 7, + 11, + 1869, + }, + dictWord{138, 11, 756}, + dictWord{139, 11, 783}, + dictWord{4, 0, 998}, + dictWord{137, 0, 861}, + dictWord{136, 0, 1009}, + dictWord{139, 11, 292}, + dictWord{5, 11, 21}, + dictWord{6, 11, 77}, + dictWord{6, 11, 157}, + dictWord{7, 11, 974}, + dictWord{7, 11, 1301}, + dictWord{7, 11, 1339}, + dictWord{7, 11, 1490}, + dictWord{ + 7, + 11, + 1873, + }, + dictWord{137, 11, 628}, + dictWord{7, 11, 1283}, + dictWord{9, 11, 227}, + dictWord{9, 11, 499}, + dictWord{10, 11, 341}, + dictWord{11, 11, 325}, + dictWord{11, 11, 408}, + dictWord{14, 11, 180}, + dictWord{15, 11, 144}, + dictWord{18, 11, 47}, + dictWord{147, 11, 49}, + dictWord{4, 0, 64}, + dictWord{5, 0, 352}, + dictWord{5, 0, 720}, + dictWord{6, 0, 368}, + dictWord{139, 0, 359}, + dictWord{5, 10, 384}, + dictWord{8, 10, 455}, + dictWord{140, 10, 48}, + dictWord{5, 10, 264}, + dictWord{ + 134, + 10, + 184, + }, + dictWord{7, 0, 1577}, + dictWord{10, 0, 304}, + dictWord{10, 0, 549}, + dictWord{12, 0, 365}, + dictWord{13, 0, 220}, + dictWord{13, 0, 240}, + dictWord{ + 142, + 0, + 33, + }, + dictWord{134, 0, 1107}, + dictWord{134, 0, 929}, + dictWord{135, 0, 1142}, + dictWord{6, 0, 175}, + dictWord{137, 0, 289}, + dictWord{5, 0, 432}, + dictWord{ + 133, + 0, + 913, + }, + dictWord{6, 0, 279}, + dictWord{7, 0, 219}, + dictWord{5, 10, 633}, + dictWord{135, 10, 1323}, + dictWord{7, 0, 785}, + dictWord{7, 10, 359}, + dictWord{ + 8, + 10, + 243, + }, + dictWord{140, 10, 175}, + dictWord{139, 0, 595}, + dictWord{132, 10, 105}, + dictWord{8, 11, 398}, + dictWord{9, 11, 681}, + dictWord{139, 11, 632}, + dictWord{140, 0, 80}, + dictWord{5, 0, 931}, + dictWord{134, 0, 1698}, + dictWord{142, 11, 241}, + dictWord{134, 11, 20}, + dictWord{134, 0, 1323}, + dictWord{11, 0, 526}, + dictWord{11, 0, 939}, + dictWord{141, 0, 290}, + dictWord{5, 0, 774}, + dictWord{6, 0, 780}, + dictWord{6, 0, 1637}, + dictWord{6, 0, 1686}, + dictWord{6, 0, 1751}, + dictWord{ + 8, + 0, + 559, + }, + dictWord{141, 0, 109}, + dictWord{141, 0, 127}, + dictWord{7, 0, 1167}, + dictWord{11, 0, 934}, + dictWord{13, 0, 391}, + dictWord{17, 0, 76}, + dictWord{ + 135, + 11, + 709, + }, + dictWord{135, 0, 963}, + dictWord{6, 0, 260}, + dictWord{135, 0, 1484}, + dictWord{134, 0, 573}, + dictWord{4, 10, 758}, + dictWord{139, 11, 941}, + dictWord{135, 10, 1649}, + dictWord{145, 11, 36}, + dictWord{4, 0, 292}, + dictWord{137, 0, 580}, + dictWord{4, 0, 736}, + dictWord{5, 0, 871}, + dictWord{6, 0, 1689}, + dictWord{135, 0, 1944}, + dictWord{7, 11, 945}, + dictWord{11, 11, 713}, + dictWord{139, 11, 744}, + dictWord{134, 0, 1164}, + dictWord{135, 11, 937}, + dictWord{ + 6, + 0, + 1922, + }, + dictWord{9, 0, 982}, + dictWord{15, 0, 173}, + dictWord{15, 0, 178}, + dictWord{15, 0, 200}, + dictWord{18, 0, 189}, + dictWord{18, 0, 207}, + dictWord{21, 0, 47}, + dictWord{135, 11, 1652}, + dictWord{7, 0, 1695}, + dictWord{139, 10, 128}, + dictWord{6, 0, 63}, + dictWord{135, 0, 920}, + dictWord{133, 0, 793}, + dictWord{ + 143, + 11, + 134, + }, + dictWord{133, 10, 918}, + dictWord{5, 0, 67}, + dictWord{6, 0, 62}, + dictWord{6, 0, 374}, + dictWord{135, 0, 1391}, + dictWord{9, 0, 790}, + dictWord{12, 0, 47}, + dictWord{4, 11, 579}, + dictWord{5, 11, 226}, + dictWord{5, 11, 323}, + dictWord{135, 11, 960}, + dictWord{10, 11, 784}, + dictWord{141, 11, 191}, + dictWord{4, 0, 391}, + dictWord{135, 0, 1169}, + dictWord{137, 0, 443}, + dictWord{13, 11, 232}, + dictWord{146, 11, 35}, + dictWord{132, 10, 340}, + dictWord{132, 0, 271}, + dictWord{ + 137, + 11, + 313, + }, + dictWord{5, 11, 973}, + dictWord{137, 11, 659}, + dictWord{134, 0, 1140}, + dictWord{6, 11, 135}, + dictWord{135, 11, 1176}, + dictWord{4, 0, 253}, + dictWord{5, 0, 544}, + dictWord{7, 0, 300}, + dictWord{137, 0, 340}, + dictWord{7, 0, 897}, + dictWord{5, 10, 985}, + dictWord{7, 10, 509}, + dictWord{145, 10, 96}, + dictWord{ + 138, + 11, + 735, + }, + dictWord{135, 10, 1919}, + dictWord{138, 0, 890}, + dictWord{5, 0, 818}, + dictWord{134, 0, 1122}, + dictWord{5, 0, 53}, + dictWord{5, 0, 541}, + dictWord{ + 6, + 0, + 94, + }, + dictWord{6, 0, 499}, + dictWord{7, 0, 230}, + dictWord{139, 0, 321}, + dictWord{4, 0, 920}, + dictWord{5, 0, 25}, + dictWord{5, 0, 790}, + dictWord{6, 0, 457}, + dictWord{ + 7, + 0, + 853, + }, + dictWord{8, 0, 788}, + dictWord{142, 11, 31}, + dictWord{132, 10, 247}, + dictWord{135, 11, 314}, + dictWord{132, 0, 468}, + dictWord{7, 0, 243}, + dictWord{ + 6, + 10, + 337, + }, + dictWord{7, 10, 494}, + dictWord{8, 10, 27}, + dictWord{8, 10, 599}, + dictWord{138, 10, 153}, + dictWord{4, 10, 184}, + dictWord{5, 10, 390}, + dictWord{ + 7, + 10, + 618, + }, + dictWord{7, 10, 1456}, + dictWord{139, 10, 710}, + dictWord{134, 0, 870}, + dictWord{134, 0, 1238}, + dictWord{134, 0, 1765}, + dictWord{10, 0, 853}, + dictWord{10, 0, 943}, + dictWord{14, 0, 437}, + dictWord{14, 0, 439}, + dictWord{14, 0, 443}, + dictWord{14, 0, 446}, + dictWord{14, 0, 452}, + dictWord{14, 0, 469}, + dictWord{ + 14, + 0, + 471, + }, + dictWord{14, 0, 473}, + dictWord{16, 0, 93}, + dictWord{16, 0, 102}, + dictWord{16, 0, 110}, + dictWord{148, 0, 121}, + dictWord{4, 0, 605}, + dictWord{ + 7, + 0, + 518, + }, + dictWord{7, 0, 1282}, + dictWord{7, 0, 1918}, + dictWord{10, 0, 180}, + dictWord{139, 0, 218}, + dictWord{133, 0, 822}, + dictWord{4, 0, 634}, + dictWord{ + 11, + 0, + 916, + }, + dictWord{142, 0, 419}, + dictWord{6, 11, 281}, + dictWord{7, 11, 6}, + dictWord{8, 11, 282}, + dictWord{8, 11, 480}, + dictWord{8, 11, 499}, + dictWord{9, 11, 198}, + dictWord{10, 11, 143}, + dictWord{10, 11, 169}, + dictWord{10, 11, 211}, + dictWord{10, 11, 417}, + dictWord{10, 11, 574}, + dictWord{11, 11, 147}, + dictWord{ + 11, + 11, + 395, + }, + dictWord{12, 11, 75}, + dictWord{12, 11, 407}, + dictWord{12, 11, 608}, + dictWord{13, 11, 500}, + dictWord{142, 11, 251}, + dictWord{134, 0, 898}, + dictWord{ + 6, + 0, + 36, + }, + dictWord{7, 0, 658}, + dictWord{8, 0, 454}, + dictWord{150, 11, 48}, + dictWord{133, 11, 674}, + dictWord{135, 11, 1776}, + dictWord{4, 11, 419}, + dictWord{ + 10, + 10, + 227, + }, + dictWord{11, 10, 497}, + dictWord{11, 10, 709}, + dictWord{140, 10, 415}, + dictWord{6, 10, 360}, + dictWord{7, 10, 1664}, + dictWord{136, 10, 478}, + dictWord{137, 0, 806}, + dictWord{12, 11, 508}, + dictWord{14, 11, 102}, + dictWord{14, 11, 226}, + dictWord{144, 11, 57}, + dictWord{135, 11, 1123}, + dictWord{ + 4, + 11, + 138, + }, + dictWord{7, 11, 1012}, + dictWord{7, 11, 1280}, + dictWord{137, 11, 76}, + dictWord{5, 11, 29}, + dictWord{140, 11, 638}, + dictWord{136, 10, 699}, + dictWord{134, 0, 1326}, + dictWord{132, 0, 104}, + dictWord{135, 11, 735}, + dictWord{132, 10, 739}, + dictWord{134, 0, 1331}, + dictWord{7, 0, 260}, + dictWord{ + 135, + 11, + 260, + }, + dictWord{135, 11, 1063}, + dictWord{7, 0, 45}, + dictWord{9, 0, 542}, + dictWord{9, 0, 566}, + dictWord{10, 0, 728}, + dictWord{137, 10, 869}, + dictWord{ + 4, + 10, + 67, + }, + dictWord{5, 10, 422}, + dictWord{7, 10, 1037}, + dictWord{7, 10, 1289}, + dictWord{7, 10, 1555}, + dictWord{9, 10, 741}, + dictWord{145, 10, 108}, + dictWord{ + 139, + 0, + 263, + }, + dictWord{134, 0, 1516}, + dictWord{14, 0, 146}, + dictWord{15, 0, 42}, + dictWord{16, 0, 23}, + dictWord{17, 0, 86}, + dictWord{146, 0, 17}, + dictWord{ + 138, + 0, + 468, + }, + dictWord{136, 0, 1005}, + dictWord{4, 11, 17}, + dictWord{5, 11, 23}, + dictWord{7, 11, 995}, + dictWord{11, 11, 383}, + dictWord{11, 11, 437}, + dictWord{ + 12, + 11, + 460, + }, + dictWord{140, 11, 532}, + dictWord{7, 0, 87}, + dictWord{142, 0, 288}, + dictWord{138, 10, 96}, + dictWord{135, 11, 626}, + dictWord{144, 10, 26}, + dictWord{ + 7, + 0, + 988, + }, + dictWord{7, 0, 1939}, + dictWord{9, 0, 64}, + dictWord{9, 0, 502}, + dictWord{12, 0, 22}, + dictWord{12, 0, 34}, + dictWord{13, 0, 12}, + dictWord{13, 0, 234}, + dictWord{147, 0, 77}, + dictWord{13, 0, 133}, + dictWord{8, 10, 203}, + dictWord{11, 10, 823}, + dictWord{11, 10, 846}, + dictWord{12, 10, 482}, + dictWord{13, 10, 277}, + dictWord{13, 10, 302}, + dictWord{13, 10, 464}, + dictWord{14, 10, 205}, + dictWord{142, 10, 221}, + dictWord{4, 10, 449}, + dictWord{133, 10, 718}, + dictWord{ + 135, + 0, + 141, + }, + dictWord{6, 0, 1842}, + dictWord{136, 0, 872}, + dictWord{8, 11, 70}, + dictWord{12, 11, 171}, + dictWord{141, 11, 272}, + dictWord{4, 10, 355}, + dictWord{ + 6, + 10, + 311, + }, + dictWord{9, 10, 256}, + dictWord{138, 10, 404}, + dictWord{132, 0, 619}, + dictWord{137, 0, 261}, + dictWord{10, 11, 233}, + dictWord{10, 10, 758}, + dictWord{139, 11, 76}, + dictWord{5, 0, 246}, + dictWord{8, 0, 189}, + dictWord{9, 0, 355}, + dictWord{9, 0, 512}, + dictWord{10, 0, 124}, + dictWord{10, 0, 453}, + dictWord{ + 11, + 0, + 143, + }, + dictWord{11, 0, 416}, + dictWord{11, 0, 859}, + dictWord{141, 0, 341}, + dictWord{134, 11, 442}, + dictWord{133, 10, 827}, + dictWord{5, 10, 64}, + dictWord{ + 140, + 10, + 581, + }, + dictWord{4, 10, 442}, + dictWord{7, 10, 1047}, + dictWord{7, 10, 1352}, + dictWord{135, 10, 1643}, + dictWord{134, 11, 1709}, + dictWord{5, 0, 678}, + dictWord{6, 0, 305}, + dictWord{7, 0, 775}, + dictWord{7, 0, 1065}, + dictWord{133, 10, 977}, + dictWord{11, 11, 69}, + dictWord{12, 11, 105}, + dictWord{12, 11, 117}, + dictWord{13, 11, 213}, + dictWord{14, 11, 13}, + dictWord{14, 11, 62}, + dictWord{14, 11, 177}, + dictWord{14, 11, 421}, + dictWord{15, 11, 19}, + dictWord{146, 11, 141}, + dictWord{137, 11, 309}, + dictWord{5, 0, 35}, + dictWord{7, 0, 862}, + dictWord{7, 0, 1886}, + dictWord{138, 0, 179}, + dictWord{136, 0, 285}, + dictWord{132, 0, 517}, + dictWord{7, 11, 976}, + dictWord{9, 11, 146}, + dictWord{10, 11, 206}, + dictWord{10, 11, 596}, + dictWord{13, 11, 218}, + dictWord{142, 11, 153}, + dictWord{ + 132, + 10, + 254, + }, + dictWord{6, 0, 214}, + dictWord{12, 0, 540}, + dictWord{4, 10, 275}, + dictWord{7, 10, 1219}, + dictWord{140, 10, 376}, + dictWord{8, 0, 667}, + dictWord{ + 11, + 0, + 403, + }, + dictWord{146, 0, 83}, + dictWord{12, 0, 74}, + dictWord{10, 11, 648}, + dictWord{11, 11, 671}, + dictWord{143, 11, 46}, + dictWord{135, 0, 125}, + dictWord{ + 134, + 10, + 1753, + }, + dictWord{133, 0, 761}, + dictWord{6, 0, 912}, + dictWord{4, 11, 518}, + dictWord{6, 10, 369}, + dictWord{6, 10, 502}, + dictWord{7, 10, 1036}, + dictWord{ + 7, + 11, + 1136, + }, + dictWord{8, 10, 348}, + dictWord{9, 10, 452}, + dictWord{10, 10, 26}, + dictWord{11, 10, 224}, + dictWord{11, 10, 387}, + dictWord{11, 10, 772}, + dictWord{12, 10, 95}, + dictWord{12, 10, 629}, + dictWord{13, 10, 195}, + dictWord{13, 10, 207}, + dictWord{13, 10, 241}, + dictWord{14, 10, 260}, + dictWord{14, 10, 270}, + dictWord{143, 10, 140}, + dictWord{10, 0, 131}, + dictWord{140, 0, 72}, + dictWord{132, 10, 269}, + dictWord{5, 10, 480}, + dictWord{7, 10, 532}, + dictWord{ + 7, + 10, + 1197, + }, + dictWord{7, 10, 1358}, + dictWord{8, 10, 291}, + dictWord{11, 10, 349}, + dictWord{142, 10, 396}, + dictWord{8, 11, 689}, + dictWord{137, 11, 863}, + dictWord{ + 8, + 0, + 333, + }, + dictWord{138, 0, 182}, + dictWord{4, 11, 18}, + dictWord{7, 11, 145}, + dictWord{7, 11, 444}, + dictWord{7, 11, 1278}, + dictWord{8, 11, 49}, + dictWord{ + 8, + 11, + 400, + }, + dictWord{9, 11, 71}, + dictWord{9, 11, 250}, + dictWord{10, 11, 459}, + dictWord{12, 11, 160}, + dictWord{144, 11, 24}, + dictWord{14, 11, 35}, + dictWord{ + 142, + 11, + 191, + }, + dictWord{135, 11, 1864}, + dictWord{135, 0, 1338}, + dictWord{148, 10, 15}, + dictWord{14, 0, 94}, + dictWord{15, 0, 65}, + dictWord{16, 0, 4}, + dictWord{ + 16, + 0, + 77, + }, + dictWord{16, 0, 80}, + dictWord{145, 0, 5}, + dictWord{12, 11, 82}, + dictWord{143, 11, 36}, + dictWord{133, 11, 1010}, + dictWord{133, 0, 449}, + dictWord{ + 133, + 0, + 646, + }, + dictWord{7, 0, 86}, + dictWord{8, 0, 103}, + dictWord{135, 10, 657}, + dictWord{7, 0, 2028}, + dictWord{138, 0, 641}, + dictWord{136, 10, 533}, + dictWord{ + 134, + 0, + 1, + }, + dictWord{139, 11, 970}, + dictWord{5, 11, 87}, + dictWord{7, 11, 313}, + dictWord{7, 11, 1103}, + dictWord{10, 11, 112}, + dictWord{10, 11, 582}, + dictWord{ + 11, + 11, + 389, + }, + dictWord{11, 11, 813}, + dictWord{12, 11, 385}, + dictWord{13, 11, 286}, + dictWord{14, 11, 124}, + dictWord{146, 11, 108}, + dictWord{6, 0, 869}, + dictWord{ + 132, + 11, + 267, + }, + dictWord{6, 0, 277}, + dictWord{7, 0, 1274}, + dictWord{7, 0, 1386}, + dictWord{146, 0, 87}, + dictWord{6, 0, 187}, + dictWord{7, 0, 39}, + dictWord{7, 0, 1203}, + dictWord{8, 0, 380}, + dictWord{14, 0, 117}, + dictWord{149, 0, 28}, + dictWord{4, 10, 211}, + dictWord{4, 10, 332}, + dictWord{5, 10, 335}, + dictWord{6, 10, 238}, + dictWord{ + 7, + 10, + 269, + }, + dictWord{7, 10, 811}, + dictWord{7, 10, 1797}, + dictWord{8, 10, 836}, + dictWord{9, 10, 507}, + dictWord{141, 10, 242}, + dictWord{4, 0, 785}, + dictWord{ + 5, + 0, + 368, + }, + dictWord{6, 0, 297}, + dictWord{7, 0, 793}, + dictWord{139, 0, 938}, + dictWord{7, 0, 464}, + dictWord{8, 0, 558}, + dictWord{11, 0, 105}, + dictWord{12, 0, 231}, + dictWord{14, 0, 386}, + dictWord{15, 0, 102}, + dictWord{148, 0, 75}, + dictWord{133, 10, 1009}, + dictWord{8, 0, 877}, + dictWord{140, 0, 731}, + dictWord{ + 139, + 11, + 289, + }, + dictWord{10, 11, 249}, + dictWord{139, 11, 209}, + dictWord{132, 11, 561}, + dictWord{134, 0, 1608}, + dictWord{132, 11, 760}, + dictWord{134, 0, 1429}, + dictWord{9, 11, 154}, + dictWord{140, 11, 485}, + dictWord{5, 10, 228}, + dictWord{6, 10, 203}, + dictWord{7, 10, 156}, + dictWord{8, 10, 347}, + dictWord{ + 137, + 10, + 265, + }, + dictWord{7, 0, 1010}, + dictWord{11, 0, 733}, + dictWord{11, 0, 759}, + dictWord{13, 0, 34}, + dictWord{14, 0, 427}, + dictWord{146, 0, 45}, + dictWord{7, 10, 1131}, + dictWord{135, 10, 1468}, + dictWord{136, 11, 255}, + dictWord{7, 0, 1656}, + dictWord{9, 0, 369}, + dictWord{10, 0, 338}, + dictWord{10, 0, 490}, + dictWord{ + 11, + 0, + 154, + }, + dictWord{11, 0, 545}, + dictWord{11, 0, 775}, + dictWord{13, 0, 77}, + dictWord{141, 0, 274}, + dictWord{133, 11, 621}, + dictWord{134, 0, 1038}, + dictWord{ + 4, + 11, + 368, + }, + dictWord{135, 11, 641}, + dictWord{6, 0, 2010}, + dictWord{8, 0, 979}, + dictWord{8, 0, 985}, + dictWord{10, 0, 951}, + dictWord{138, 0, 1011}, + dictWord{ + 134, + 0, + 1005, + }, + dictWord{19, 0, 121}, + dictWord{5, 10, 291}, + dictWord{5, 10, 318}, + dictWord{7, 10, 765}, + dictWord{9, 10, 389}, + dictWord{140, 10, 548}, + dictWord{ + 5, + 0, + 20, + }, + dictWord{6, 0, 298}, + dictWord{7, 0, 659}, + dictWord{137, 0, 219}, + dictWord{7, 0, 1440}, + dictWord{11, 0, 854}, + dictWord{11, 0, 872}, + dictWord{11, 0, 921}, + dictWord{12, 0, 551}, + dictWord{13, 0, 472}, + dictWord{142, 0, 367}, + dictWord{5, 0, 490}, + dictWord{6, 0, 615}, + dictWord{6, 0, 620}, + dictWord{135, 0, 683}, + dictWord{ + 6, + 0, + 1070, + }, + dictWord{134, 0, 1597}, + dictWord{139, 0, 522}, + dictWord{132, 0, 439}, + dictWord{136, 0, 669}, + dictWord{6, 0, 766}, + dictWord{6, 0, 1143}, + dictWord{ + 6, + 0, + 1245, + }, + dictWord{10, 10, 525}, + dictWord{139, 10, 82}, + dictWord{9, 11, 92}, + dictWord{147, 11, 91}, + dictWord{6, 0, 668}, + dictWord{134, 0, 1218}, + dictWord{ + 6, + 11, + 525, + }, + dictWord{9, 11, 876}, + dictWord{140, 11, 284}, + dictWord{132, 0, 233}, + dictWord{136, 0, 547}, + dictWord{132, 10, 422}, + dictWord{5, 10, 355}, + dictWord{145, 10, 0}, + dictWord{6, 11, 300}, + dictWord{135, 11, 1515}, + dictWord{4, 0, 482}, + dictWord{137, 10, 905}, + dictWord{4, 0, 886}, + dictWord{7, 0, 346}, + dictWord{133, 11, 594}, + dictWord{133, 10, 865}, + dictWord{5, 10, 914}, + dictWord{134, 10, 1625}, + dictWord{135, 0, 334}, + dictWord{5, 0, 795}, + dictWord{ + 6, + 0, + 1741, + }, + dictWord{133, 10, 234}, + dictWord{135, 10, 1383}, + dictWord{6, 11, 1641}, + dictWord{136, 11, 820}, + dictWord{135, 0, 371}, + dictWord{7, 11, 1313}, + dictWord{138, 11, 660}, + dictWord{135, 10, 1312}, + dictWord{135, 0, 622}, + dictWord{7, 0, 625}, + dictWord{135, 0, 1750}, + dictWord{135, 0, 339}, + dictWord{ + 4, + 0, + 203, + }, + dictWord{135, 0, 1936}, + dictWord{15, 0, 29}, + dictWord{16, 0, 38}, + dictWord{15, 11, 29}, + dictWord{144, 11, 38}, + dictWord{5, 0, 338}, + dictWord{ + 135, + 0, + 1256, + }, + dictWord{135, 10, 1493}, + dictWord{10, 0, 130}, + dictWord{6, 10, 421}, + dictWord{7, 10, 61}, + dictWord{7, 10, 1540}, + dictWord{138, 10, 501}, + dictWord{ + 6, + 11, + 389, + }, + dictWord{7, 11, 149}, + dictWord{9, 11, 142}, + dictWord{138, 11, 94}, + dictWord{137, 10, 341}, + dictWord{11, 0, 678}, + dictWord{12, 0, 307}, + dictWord{142, 10, 98}, + dictWord{6, 11, 8}, + dictWord{7, 11, 1881}, + dictWord{136, 11, 91}, + dictWord{135, 0, 2044}, + dictWord{6, 0, 770}, + dictWord{6, 0, 802}, + dictWord{ + 6, + 0, + 812, + }, + dictWord{7, 0, 311}, + dictWord{9, 0, 308}, + dictWord{12, 0, 255}, + dictWord{6, 10, 102}, + dictWord{7, 10, 72}, + dictWord{15, 10, 142}, + dictWord{ + 147, + 10, + 67, + }, + dictWord{151, 10, 30}, + dictWord{135, 10, 823}, + dictWord{135, 0, 1266}, + dictWord{135, 11, 1746}, + dictWord{135, 10, 1870}, + dictWord{4, 0, 400}, + dictWord{5, 0, 267}, + dictWord{135, 0, 232}, + dictWord{7, 11, 24}, + dictWord{11, 11, 542}, + dictWord{139, 11, 852}, + dictWord{135, 11, 1739}, + dictWord{4, 11, 503}, + dictWord{135, 11, 1661}, + dictWord{5, 11, 130}, + dictWord{7, 11, 1314}, + dictWord{9, 11, 610}, + dictWord{10, 11, 718}, + dictWord{11, 11, 601}, + dictWord{ + 11, + 11, + 819, + }, + dictWord{11, 11, 946}, + dictWord{140, 11, 536}, + dictWord{10, 11, 149}, + dictWord{11, 11, 280}, + dictWord{142, 11, 336}, + dictWord{7, 0, 739}, + dictWord{11, 0, 690}, + dictWord{7, 11, 1946}, + dictWord{8, 10, 48}, + dictWord{8, 10, 88}, + dictWord{8, 10, 582}, + dictWord{8, 10, 681}, + dictWord{9, 10, 373}, + dictWord{ + 9, + 10, + 864, + }, + dictWord{11, 10, 157}, + dictWord{11, 10, 843}, + dictWord{148, 10, 27}, + dictWord{134, 0, 990}, + dictWord{4, 10, 88}, + dictWord{5, 10, 137}, + dictWord{ + 5, + 10, + 174, + }, + dictWord{5, 10, 777}, + dictWord{6, 10, 1664}, + dictWord{6, 10, 1725}, + dictWord{7, 10, 77}, + dictWord{7, 10, 426}, + dictWord{7, 10, 1317}, + dictWord{ + 7, + 10, + 1355, + }, + dictWord{8, 10, 126}, + dictWord{8, 10, 563}, + dictWord{9, 10, 523}, + dictWord{9, 10, 750}, + dictWord{10, 10, 310}, + dictWord{10, 10, 836}, + dictWord{ + 11, + 10, + 42, + }, + dictWord{11, 10, 318}, + dictWord{11, 10, 731}, + dictWord{12, 10, 68}, + dictWord{12, 10, 92}, + dictWord{12, 10, 507}, + dictWord{12, 10, 692}, + dictWord{ + 13, + 10, + 81, + }, + dictWord{13, 10, 238}, + dictWord{13, 10, 374}, + dictWord{14, 10, 436}, + dictWord{18, 10, 138}, + dictWord{19, 10, 78}, + dictWord{19, 10, 111}, + dictWord{20, 10, 55}, + dictWord{20, 10, 77}, + dictWord{148, 10, 92}, + dictWord{141, 10, 418}, + dictWord{7, 0, 1831}, + dictWord{132, 10, 938}, + dictWord{6, 0, 776}, + dictWord{134, 0, 915}, + dictWord{138, 10, 351}, + dictWord{5, 11, 348}, + dictWord{6, 11, 522}, + dictWord{6, 10, 1668}, + dictWord{7, 10, 1499}, + dictWord{8, 10, 117}, + dictWord{9, 10, 314}, + dictWord{138, 10, 174}, + dictWord{135, 10, 707}, + dictWord{132, 0, 613}, + dictWord{133, 10, 403}, + dictWord{132, 11, 392}, + dictWord{ + 5, + 11, + 433, + }, + dictWord{9, 11, 633}, + dictWord{139, 11, 629}, + dictWord{133, 0, 763}, + dictWord{132, 0, 878}, + dictWord{132, 0, 977}, + dictWord{132, 0, 100}, + dictWord{6, 0, 463}, + dictWord{4, 10, 44}, + dictWord{5, 10, 311}, + dictWord{7, 10, 639}, + dictWord{7, 10, 762}, + dictWord{7, 10, 1827}, + dictWord{9, 10, 8}, + dictWord{ + 9, + 10, + 462, + }, + dictWord{148, 10, 83}, + dictWord{134, 11, 234}, + dictWord{4, 10, 346}, + dictWord{7, 10, 115}, + dictWord{9, 10, 180}, + dictWord{9, 10, 456}, + dictWord{ + 138, + 10, + 363, + }, + dictWord{5, 0, 362}, + dictWord{5, 0, 443}, + dictWord{6, 0, 318}, + dictWord{7, 0, 1019}, + dictWord{139, 0, 623}, + dictWord{5, 0, 463}, + dictWord{8, 0, 296}, + dictWord{7, 11, 140}, + dictWord{7, 11, 1950}, + dictWord{8, 11, 680}, + dictWord{11, 11, 817}, + dictWord{147, 11, 88}, + dictWord{7, 11, 1222}, + dictWord{ + 138, + 11, + 386, + }, + dictWord{142, 0, 137}, + dictWord{132, 0, 454}, + dictWord{7, 0, 1914}, + dictWord{6, 11, 5}, + dictWord{7, 10, 1051}, + dictWord{9, 10, 545}, + dictWord{ + 11, + 11, + 249, + }, + dictWord{12, 11, 313}, + dictWord{16, 11, 66}, + dictWord{145, 11, 26}, + dictWord{135, 0, 1527}, + dictWord{145, 0, 58}, + dictWord{148, 11, 59}, + dictWord{ + 5, + 0, + 48, + }, + dictWord{5, 0, 404}, + dictWord{6, 0, 557}, + dictWord{7, 0, 458}, + dictWord{8, 0, 597}, + dictWord{10, 0, 455}, + dictWord{10, 0, 606}, + dictWord{11, 0, 49}, + dictWord{ + 11, + 0, + 548, + }, + dictWord{12, 0, 476}, + dictWord{13, 0, 18}, + dictWord{141, 0, 450}, + dictWord{5, 11, 963}, + dictWord{134, 11, 1773}, + dictWord{133, 0, 729}, + dictWord{138, 11, 586}, + dictWord{5, 0, 442}, + dictWord{135, 0, 1984}, + dictWord{134, 0, 449}, + dictWord{144, 0, 40}, + dictWord{4, 0, 853}, + dictWord{7, 11, 180}, + dictWord{8, 11, 509}, + dictWord{136, 11, 792}, + dictWord{6, 10, 185}, + dictWord{7, 10, 1899}, + dictWord{9, 10, 875}, + dictWord{139, 10, 673}, + dictWord{ + 134, + 11, + 524, + }, + dictWord{12, 0, 227}, + dictWord{4, 10, 327}, + dictWord{5, 10, 478}, + dictWord{7, 10, 1332}, + dictWord{136, 10, 753}, + dictWord{6, 0, 1491}, + dictWord{ + 5, + 10, + 1020, + }, + dictWord{133, 10, 1022}, + dictWord{4, 10, 103}, + dictWord{133, 10, 401}, + dictWord{132, 11, 931}, + dictWord{4, 10, 499}, + dictWord{135, 10, 1421}, + dictWord{5, 0, 55}, + dictWord{7, 0, 376}, + dictWord{140, 0, 161}, + dictWord{133, 0, 450}, + dictWord{6, 0, 1174}, + dictWord{134, 0, 1562}, + dictWord{10, 0, 62}, + dictWord{13, 0, 400}, + dictWord{135, 11, 1837}, + dictWord{140, 0, 207}, + dictWord{135, 0, 869}, + dictWord{4, 11, 773}, + dictWord{5, 11, 618}, + dictWord{ + 137, + 11, + 756, + }, + dictWord{132, 10, 96}, + dictWord{4, 0, 213}, + dictWord{7, 0, 223}, + dictWord{8, 0, 80}, + dictWord{135, 10, 968}, + dictWord{4, 11, 90}, + dictWord{5, 11, 337}, + dictWord{5, 11, 545}, + dictWord{7, 11, 754}, + dictWord{9, 11, 186}, + dictWord{10, 11, 72}, + dictWord{10, 11, 782}, + dictWord{11, 11, 513}, + dictWord{11, 11, 577}, + dictWord{11, 11, 610}, + dictWord{11, 11, 889}, + dictWord{11, 11, 961}, + dictWord{12, 11, 354}, + dictWord{12, 11, 362}, + dictWord{12, 11, 461}, + dictWord{ + 12, + 11, + 595, + }, + dictWord{13, 11, 79}, + dictWord{143, 11, 121}, + dictWord{7, 0, 381}, + dictWord{7, 0, 806}, + dictWord{7, 0, 820}, + dictWord{8, 0, 354}, + dictWord{8, 0, 437}, + dictWord{8, 0, 787}, + dictWord{9, 0, 657}, + dictWord{10, 0, 58}, + dictWord{10, 0, 339}, + dictWord{10, 0, 749}, + dictWord{11, 0, 914}, + dictWord{12, 0, 162}, + dictWord{ + 13, + 0, + 75, + }, + dictWord{14, 0, 106}, + dictWord{14, 0, 198}, + dictWord{14, 0, 320}, + dictWord{14, 0, 413}, + dictWord{146, 0, 43}, + dictWord{136, 0, 747}, + dictWord{ + 136, + 0, + 954, + }, + dictWord{134, 0, 1073}, + dictWord{135, 0, 556}, + dictWord{7, 11, 151}, + dictWord{9, 11, 329}, + dictWord{139, 11, 254}, + dictWord{5, 0, 692}, + dictWord{ + 134, + 0, + 1395, + }, + dictWord{6, 10, 563}, + dictWord{137, 10, 224}, + dictWord{134, 0, 191}, + dictWord{132, 0, 804}, + dictWord{9, 11, 187}, + dictWord{10, 11, 36}, + dictWord{17, 11, 44}, + dictWord{146, 11, 64}, + dictWord{7, 11, 165}, + dictWord{7, 11, 919}, + dictWord{136, 11, 517}, + dictWord{4, 11, 506}, + dictWord{5, 11, 295}, + dictWord{7, 11, 1680}, + dictWord{15, 11, 14}, + dictWord{144, 11, 5}, + dictWord{4, 0, 706}, + dictWord{6, 0, 162}, + dictWord{7, 0, 1960}, + dictWord{136, 0, 831}, + dictWord{ + 135, + 11, + 1376, + }, + dictWord{7, 11, 987}, + dictWord{9, 11, 688}, + dictWord{10, 11, 522}, + dictWord{11, 11, 788}, + dictWord{140, 11, 566}, + dictWord{150, 0, 35}, + dictWord{138, 0, 426}, + dictWord{135, 0, 1235}, + dictWord{135, 11, 1741}, + dictWord{7, 11, 389}, + dictWord{7, 11, 700}, + dictWord{7, 11, 940}, + dictWord{ + 8, + 11, + 514, + }, + dictWord{9, 11, 116}, + dictWord{9, 11, 535}, + dictWord{10, 11, 118}, + dictWord{11, 11, 107}, + dictWord{11, 11, 148}, + dictWord{11, 11, 922}, + dictWord{ + 12, + 11, + 254, + }, + dictWord{12, 11, 421}, + dictWord{142, 11, 238}, + dictWord{134, 0, 1234}, + dictWord{132, 11, 743}, + dictWord{4, 10, 910}, + dictWord{5, 10, 832}, + dictWord{135, 11, 1335}, + dictWord{141, 0, 96}, + dictWord{135, 11, 185}, + dictWord{146, 0, 149}, + dictWord{4, 0, 204}, + dictWord{137, 0, 902}, + dictWord{ + 4, + 11, + 784, + }, + dictWord{133, 11, 745}, + dictWord{136, 0, 833}, + dictWord{136, 0, 949}, + dictWord{7, 0, 366}, + dictWord{9, 0, 287}, + dictWord{12, 0, 199}, + dictWord{ + 12, + 0, + 556, + }, + dictWord{12, 0, 577}, + dictWord{5, 11, 81}, + dictWord{7, 11, 146}, + dictWord{7, 11, 1342}, + dictWord{7, 11, 1446}, + dictWord{8, 11, 53}, + dictWord{8, 11, 561}, + dictWord{8, 11, 694}, + dictWord{8, 11, 754}, + dictWord{9, 11, 97}, + dictWord{9, 11, 115}, + dictWord{9, 11, 894}, + dictWord{10, 11, 462}, + dictWord{10, 11, 813}, + dictWord{11, 11, 230}, + dictWord{11, 11, 657}, + dictWord{11, 11, 699}, + dictWord{11, 11, 748}, + dictWord{12, 11, 119}, + dictWord{12, 11, 200}, + dictWord{ + 12, + 11, + 283, + }, + dictWord{14, 11, 273}, + dictWord{145, 11, 15}, + dictWord{5, 11, 408}, + dictWord{137, 11, 747}, + dictWord{9, 11, 498}, + dictWord{140, 11, 181}, + dictWord{ + 6, + 0, + 2020, + }, + dictWord{136, 0, 992}, + dictWord{5, 0, 356}, + dictWord{135, 0, 224}, + dictWord{134, 0, 784}, + dictWord{7, 0, 630}, + dictWord{9, 0, 567}, + dictWord{ + 11, + 0, + 150, + }, + dictWord{11, 0, 444}, + dictWord{13, 0, 119}, + dictWord{8, 10, 528}, + dictWord{137, 10, 348}, + dictWord{134, 0, 539}, + dictWord{4, 10, 20}, + dictWord{ + 133, + 10, + 616, + }, + dictWord{142, 0, 27}, + dictWord{7, 11, 30}, + dictWord{8, 11, 86}, + dictWord{8, 11, 315}, + dictWord{8, 11, 700}, + dictWord{9, 11, 576}, + dictWord{9, 11, 858}, + dictWord{11, 11, 310}, + dictWord{11, 11, 888}, + dictWord{11, 11, 904}, + dictWord{12, 11, 361}, + dictWord{141, 11, 248}, + dictWord{138, 11, 839}, + dictWord{ + 134, + 0, + 755, + }, + dictWord{134, 0, 1063}, + dictWord{7, 10, 1091}, + dictWord{135, 10, 1765}, + dictWord{134, 11, 428}, + dictWord{7, 11, 524}, + dictWord{8, 11, 169}, + dictWord{8, 11, 234}, + dictWord{9, 11, 480}, + dictWord{138, 11, 646}, + dictWord{139, 0, 814}, + dictWord{7, 11, 1462}, + dictWord{139, 11, 659}, + dictWord{ + 4, + 10, + 26, + }, + dictWord{5, 10, 429}, + dictWord{6, 10, 245}, + dictWord{7, 10, 704}, + dictWord{7, 10, 1379}, + dictWord{135, 10, 1474}, + dictWord{7, 11, 1205}, + dictWord{ + 138, + 11, + 637, + }, + dictWord{139, 11, 803}, + dictWord{132, 10, 621}, + dictWord{136, 0, 987}, + dictWord{4, 11, 266}, + dictWord{8, 11, 4}, + dictWord{9, 11, 39}, + dictWord{ + 10, + 11, + 166, + }, + dictWord{11, 11, 918}, + dictWord{12, 11, 635}, + dictWord{20, 11, 10}, + dictWord{22, 11, 27}, + dictWord{150, 11, 43}, + dictWord{4, 0, 235}, + dictWord{ + 135, + 0, + 255, + }, + dictWord{4, 0, 194}, + dictWord{5, 0, 584}, + dictWord{6, 0, 384}, + dictWord{7, 0, 583}, + dictWord{10, 0, 761}, + dictWord{11, 0, 760}, + dictWord{139, 0, 851}, + dictWord{133, 10, 542}, + dictWord{134, 0, 1086}, + dictWord{133, 10, 868}, + dictWord{8, 0, 1016}, + dictWord{136, 0, 1018}, + dictWord{7, 0, 1396}, + dictWord{ + 7, + 11, + 1396, + }, + dictWord{136, 10, 433}, + dictWord{135, 10, 1495}, + dictWord{138, 10, 215}, + dictWord{141, 10, 124}, + dictWord{7, 11, 157}, + dictWord{ + 8, + 11, + 279, + }, + dictWord{9, 11, 759}, + dictWord{16, 11, 31}, + dictWord{16, 11, 39}, + dictWord{16, 11, 75}, + dictWord{18, 11, 24}, + dictWord{20, 11, 42}, + dictWord{152, 11, 1}, + dictWord{5, 0, 562}, + dictWord{134, 11, 604}, + dictWord{134, 0, 913}, + dictWord{5, 0, 191}, + dictWord{137, 0, 271}, + dictWord{4, 0, 470}, + dictWord{6, 0, 153}, + dictWord{7, 0, 1503}, + dictWord{7, 0, 1923}, + dictWord{10, 0, 701}, + dictWord{11, 0, 132}, + dictWord{11, 0, 227}, + dictWord{11, 0, 320}, + dictWord{11, 0, 436}, + dictWord{ + 11, + 0, + 525, + }, + dictWord{11, 0, 855}, + dictWord{11, 0, 873}, + dictWord{12, 0, 41}, + dictWord{12, 0, 286}, + dictWord{13, 0, 103}, + dictWord{13, 0, 284}, + dictWord{ + 14, + 0, + 255, + }, + dictWord{14, 0, 262}, + dictWord{15, 0, 117}, + dictWord{143, 0, 127}, + dictWord{7, 0, 475}, + dictWord{12, 0, 45}, + dictWord{147, 10, 112}, + dictWord{ + 132, + 11, + 567, + }, + dictWord{137, 11, 859}, + dictWord{6, 0, 713}, + dictWord{6, 0, 969}, + dictWord{6, 0, 1290}, + dictWord{134, 0, 1551}, + dictWord{133, 0, 327}, + dictWord{ + 6, + 0, + 552, + }, + dictWord{6, 0, 1292}, + dictWord{7, 0, 1754}, + dictWord{137, 0, 604}, + dictWord{4, 0, 223}, + dictWord{6, 0, 359}, + dictWord{11, 0, 3}, + dictWord{13, 0, 108}, + dictWord{14, 0, 89}, + dictWord{16, 0, 22}, + dictWord{5, 11, 762}, + dictWord{7, 11, 1880}, + dictWord{9, 11, 680}, + dictWord{139, 11, 798}, + dictWord{5, 0, 80}, + dictWord{ + 6, + 0, + 405, + }, + dictWord{7, 0, 403}, + dictWord{7, 0, 1502}, + dictWord{8, 0, 456}, + dictWord{9, 0, 487}, + dictWord{9, 0, 853}, + dictWord{9, 0, 889}, + dictWord{10, 0, 309}, + dictWord{ + 11, + 0, + 721, + }, + dictWord{11, 0, 994}, + dictWord{12, 0, 430}, + dictWord{141, 0, 165}, + dictWord{133, 11, 298}, + dictWord{132, 10, 647}, + dictWord{134, 0, 2016}, + dictWord{18, 10, 10}, + dictWord{146, 11, 10}, + dictWord{4, 0, 453}, + dictWord{5, 0, 887}, + dictWord{6, 0, 535}, + dictWord{8, 0, 6}, + dictWord{8, 0, 543}, + dictWord{ + 136, + 0, + 826, + }, + dictWord{136, 0, 975}, + dictWord{10, 0, 961}, + dictWord{138, 0, 962}, + dictWord{138, 10, 220}, + dictWord{6, 0, 1891}, + dictWord{6, 0, 1893}, + dictWord{ + 9, + 0, + 916, + }, + dictWord{9, 0, 965}, + dictWord{9, 0, 972}, + dictWord{12, 0, 801}, + dictWord{12, 0, 859}, + dictWord{12, 0, 883}, + dictWord{15, 0, 226}, + dictWord{149, 0, 51}, + dictWord{132, 10, 109}, + dictWord{135, 11, 267}, + dictWord{7, 11, 92}, + dictWord{7, 11, 182}, + dictWord{8, 11, 453}, + dictWord{9, 11, 204}, + dictWord{11, 11, 950}, + dictWord{12, 11, 94}, + dictWord{12, 11, 644}, + dictWord{16, 11, 20}, + dictWord{16, 11, 70}, + dictWord{16, 11, 90}, + dictWord{147, 11, 55}, + dictWord{ + 134, + 10, + 1746, + }, + dictWord{6, 11, 71}, + dictWord{7, 11, 845}, + dictWord{7, 11, 1308}, + dictWord{8, 11, 160}, + dictWord{137, 11, 318}, + dictWord{5, 0, 101}, + dictWord{6, 0, 88}, + dictWord{7, 0, 263}, + dictWord{7, 0, 628}, + dictWord{7, 0, 1677}, + dictWord{8, 0, 349}, + dictWord{9, 0, 100}, + dictWord{10, 0, 677}, + dictWord{14, 0, 169}, + dictWord{ + 14, + 0, + 302, + }, + dictWord{14, 0, 313}, + dictWord{15, 0, 48}, + dictWord{15, 0, 84}, + dictWord{7, 11, 237}, + dictWord{8, 11, 664}, + dictWord{9, 11, 42}, + dictWord{9, 11, 266}, + dictWord{9, 11, 380}, + dictWord{9, 11, 645}, + dictWord{10, 11, 177}, + dictWord{138, 11, 276}, + dictWord{138, 11, 69}, + dictWord{4, 0, 310}, + dictWord{7, 0, 708}, + dictWord{7, 0, 996}, + dictWord{9, 0, 795}, + dictWord{10, 0, 390}, + dictWord{10, 0, 733}, + dictWord{11, 0, 451}, + dictWord{12, 0, 249}, + dictWord{14, 0, 115}, + dictWord{ + 14, + 0, + 286, + }, + dictWord{143, 0, 100}, + dictWord{5, 0, 587}, + dictWord{4, 10, 40}, + dictWord{10, 10, 67}, + dictWord{11, 10, 117}, + dictWord{11, 10, 768}, + dictWord{ + 139, + 10, + 935, + }, + dictWord{6, 0, 1942}, + dictWord{7, 0, 512}, + dictWord{136, 0, 983}, + dictWord{7, 10, 992}, + dictWord{8, 10, 301}, + dictWord{9, 10, 722}, + dictWord{12, 10, 63}, + dictWord{13, 10, 29}, + dictWord{14, 10, 161}, + dictWord{143, 10, 18}, + dictWord{136, 11, 76}, + dictWord{139, 10, 923}, + dictWord{134, 0, 645}, + dictWord{ + 134, + 0, + 851, + }, + dictWord{4, 0, 498}, + dictWord{132, 11, 293}, + dictWord{7, 0, 217}, + dictWord{8, 0, 140}, + dictWord{10, 0, 610}, + dictWord{14, 11, 352}, + dictWord{ + 17, + 11, + 53, + }, + dictWord{18, 11, 146}, + dictWord{18, 11, 152}, + dictWord{19, 11, 11}, + dictWord{150, 11, 54}, + dictWord{134, 0, 1448}, + dictWord{138, 11, 841}, + dictWord{133, 0, 905}, + dictWord{4, 11, 605}, + dictWord{7, 11, 518}, + dictWord{7, 11, 1282}, + dictWord{7, 11, 1918}, + dictWord{10, 11, 180}, + dictWord{139, 11, 218}, + dictWord{139, 11, 917}, + dictWord{135, 10, 825}, + dictWord{140, 10, 328}, + dictWord{4, 0, 456}, + dictWord{7, 0, 105}, + dictWord{7, 0, 358}, + dictWord{7, 0, 1637}, + dictWord{8, 0, 643}, + dictWord{139, 0, 483}, + dictWord{134, 0, 792}, + dictWord{6, 11, 96}, + dictWord{135, 11, 1426}, + dictWord{137, 11, 691}, + dictWord{ + 4, + 11, + 651, + }, + dictWord{133, 11, 289}, + dictWord{7, 11, 688}, + dictWord{8, 11, 35}, + dictWord{9, 11, 511}, + dictWord{10, 11, 767}, + dictWord{147, 11, 118}, + dictWord{ + 150, + 0, + 56, + }, + dictWord{5, 0, 243}, + dictWord{5, 0, 535}, + dictWord{6, 10, 204}, + dictWord{10, 10, 320}, + dictWord{10, 10, 583}, + dictWord{13, 10, 502}, + dictWord{ + 14, + 10, + 72, + }, + dictWord{14, 10, 274}, + dictWord{14, 10, 312}, + dictWord{14, 10, 344}, + dictWord{15, 10, 159}, + dictWord{16, 10, 62}, + dictWord{16, 10, 69}, + dictWord{ + 17, + 10, + 30, + }, + dictWord{18, 10, 42}, + dictWord{18, 10, 53}, + dictWord{18, 10, 84}, + dictWord{18, 10, 140}, + dictWord{19, 10, 68}, + dictWord{19, 10, 85}, + dictWord{20, 10, 5}, + dictWord{20, 10, 45}, + dictWord{20, 10, 101}, + dictWord{22, 10, 7}, + dictWord{150, 10, 20}, + dictWord{4, 10, 558}, + dictWord{6, 10, 390}, + dictWord{7, 10, 162}, + dictWord{7, 10, 689}, + dictWord{9, 10, 360}, + dictWord{138, 10, 653}, + dictWord{146, 11, 23}, + dictWord{135, 0, 1748}, + dictWord{5, 10, 856}, + dictWord{ + 6, + 10, + 1672, + }, + dictWord{6, 10, 1757}, + dictWord{134, 10, 1781}, + dictWord{5, 0, 539}, + dictWord{5, 0, 754}, + dictWord{6, 0, 876}, + dictWord{132, 11, 704}, + dictWord{ + 135, + 11, + 1078, + }, + dictWord{5, 10, 92}, + dictWord{10, 10, 736}, + dictWord{140, 10, 102}, + dictWord{17, 0, 91}, + dictWord{5, 10, 590}, + dictWord{137, 10, 213}, + dictWord{134, 0, 1565}, + dictWord{6, 0, 91}, + dictWord{135, 0, 435}, + dictWord{4, 0, 939}, + dictWord{140, 0, 792}, + dictWord{134, 0, 1399}, + dictWord{4, 0, 16}, + dictWord{ + 5, + 0, + 316, + }, + dictWord{5, 0, 842}, + dictWord{6, 0, 370}, + dictWord{6, 0, 1778}, + dictWord{8, 0, 166}, + dictWord{11, 0, 812}, + dictWord{12, 0, 206}, + dictWord{12, 0, 351}, + dictWord{14, 0, 418}, + dictWord{16, 0, 15}, + dictWord{16, 0, 34}, + dictWord{18, 0, 3}, + dictWord{19, 0, 3}, + dictWord{19, 0, 7}, + dictWord{20, 0, 4}, + dictWord{21, 0, 21}, + dictWord{ + 4, + 11, + 720, + }, + dictWord{133, 11, 306}, + dictWord{144, 0, 95}, + dictWord{133, 11, 431}, + dictWord{132, 11, 234}, + dictWord{135, 0, 551}, + dictWord{4, 0, 999}, + dictWord{6, 0, 1966}, + dictWord{134, 0, 2042}, + dictWord{7, 0, 619}, + dictWord{10, 0, 547}, + dictWord{11, 0, 122}, + dictWord{12, 0, 601}, + dictWord{15, 0, 7}, + dictWord{148, 0, 20}, + dictWord{5, 11, 464}, + dictWord{6, 11, 236}, + dictWord{7, 11, 276}, + dictWord{7, 11, 696}, + dictWord{7, 11, 914}, + dictWord{7, 11, 1108}, + dictWord{ + 7, + 11, + 1448, + }, + dictWord{9, 11, 15}, + dictWord{9, 11, 564}, + dictWord{10, 11, 14}, + dictWord{12, 11, 565}, + dictWord{13, 11, 449}, + dictWord{14, 11, 53}, + dictWord{ + 15, + 11, + 13, + }, + dictWord{16, 11, 64}, + dictWord{145, 11, 41}, + dictWord{6, 0, 884}, + dictWord{6, 0, 1019}, + dictWord{134, 0, 1150}, + dictWord{6, 11, 1767}, + dictWord{ + 12, + 11, + 194, + }, + dictWord{145, 11, 107}, + dictWord{136, 10, 503}, + dictWord{133, 11, 840}, + dictWord{7, 0, 671}, + dictWord{134, 10, 466}, + dictWord{132, 0, 888}, + dictWord{4, 0, 149}, + dictWord{138, 0, 368}, + dictWord{4, 0, 154}, + dictWord{7, 0, 1134}, + dictWord{136, 0, 105}, + dictWord{135, 0, 983}, + dictWord{9, 11, 642}, + dictWord{11, 11, 236}, + dictWord{142, 11, 193}, + dictWord{4, 0, 31}, + dictWord{6, 0, 429}, + dictWord{7, 0, 962}, + dictWord{9, 0, 458}, + dictWord{139, 0, 691}, + dictWord{ + 6, + 0, + 643, + }, + dictWord{134, 0, 1102}, + dictWord{132, 0, 312}, + dictWord{4, 11, 68}, + dictWord{5, 11, 634}, + dictWord{6, 11, 386}, + dictWord{7, 11, 794}, + dictWord{ + 8, + 11, + 273, + }, + dictWord{9, 11, 563}, + dictWord{10, 11, 105}, + dictWord{10, 11, 171}, + dictWord{11, 11, 94}, + dictWord{139, 11, 354}, + dictWord{133, 0, 740}, + dictWord{ + 135, + 0, + 1642, + }, + dictWord{4, 11, 95}, + dictWord{7, 11, 416}, + dictWord{8, 11, 211}, + dictWord{139, 11, 830}, + dictWord{132, 0, 236}, + dictWord{138, 10, 241}, + dictWord{7, 11, 731}, + dictWord{13, 11, 20}, + dictWord{143, 11, 11}, + dictWord{5, 0, 836}, + dictWord{5, 0, 857}, + dictWord{6, 0, 1680}, + dictWord{135, 0, 59}, + dictWord{ + 10, + 0, + 68, + }, + dictWord{11, 0, 494}, + dictWord{152, 11, 6}, + dictWord{4, 0, 81}, + dictWord{139, 0, 867}, + dictWord{135, 0, 795}, + dictWord{133, 11, 689}, + dictWord{ + 4, + 0, + 1001, + }, + dictWord{5, 0, 282}, + dictWord{6, 0, 1932}, + dictWord{6, 0, 1977}, + dictWord{6, 0, 1987}, + dictWord{6, 0, 1992}, + dictWord{8, 0, 650}, + dictWord{8, 0, 919}, + dictWord{8, 0, 920}, + dictWord{8, 0, 923}, + dictWord{8, 0, 926}, + dictWord{8, 0, 927}, + dictWord{8, 0, 931}, + dictWord{8, 0, 939}, + dictWord{8, 0, 947}, + dictWord{8, 0, 956}, + dictWord{8, 0, 997}, + dictWord{9, 0, 907}, + dictWord{10, 0, 950}, + dictWord{10, 0, 953}, + dictWord{10, 0, 954}, + dictWord{10, 0, 956}, + dictWord{10, 0, 958}, + dictWord{ + 10, + 0, + 959, + }, + dictWord{10, 0, 964}, + dictWord{10, 0, 970}, + dictWord{10, 0, 972}, + dictWord{10, 0, 973}, + dictWord{10, 0, 975}, + dictWord{10, 0, 976}, + dictWord{ + 10, + 0, + 980, + }, + dictWord{10, 0, 981}, + dictWord{10, 0, 984}, + dictWord{10, 0, 988}, + dictWord{10, 0, 990}, + dictWord{10, 0, 995}, + dictWord{10, 0, 999}, + dictWord{ + 10, + 0, + 1002, + }, + dictWord{10, 0, 1003}, + dictWord{10, 0, 1005}, + dictWord{10, 0, 1006}, + dictWord{10, 0, 1008}, + dictWord{10, 0, 1009}, + dictWord{10, 0, 1012}, + dictWord{10, 0, 1014}, + dictWord{10, 0, 1015}, + dictWord{10, 0, 1019}, + dictWord{10, 0, 1020}, + dictWord{10, 0, 1022}, + dictWord{12, 0, 959}, + dictWord{12, 0, 961}, + dictWord{12, 0, 962}, + dictWord{12, 0, 963}, + dictWord{12, 0, 964}, + dictWord{12, 0, 965}, + dictWord{12, 0, 967}, + dictWord{12, 0, 968}, + dictWord{12, 0, 969}, + dictWord{12, 0, 970}, + dictWord{12, 0, 971}, + dictWord{12, 0, 972}, + dictWord{12, 0, 973}, + dictWord{12, 0, 974}, + dictWord{12, 0, 975}, + dictWord{12, 0, 976}, + dictWord{ + 12, + 0, + 977, + }, + dictWord{12, 0, 979}, + dictWord{12, 0, 981}, + dictWord{12, 0, 982}, + dictWord{12, 0, 983}, + dictWord{12, 0, 984}, + dictWord{12, 0, 985}, + dictWord{ + 12, + 0, + 986, + }, + dictWord{12, 0, 987}, + dictWord{12, 0, 989}, + dictWord{12, 0, 990}, + dictWord{12, 0, 992}, + dictWord{12, 0, 993}, + dictWord{12, 0, 995}, + dictWord{12, 0, 998}, + dictWord{12, 0, 999}, + dictWord{12, 0, 1000}, + dictWord{12, 0, 1001}, + dictWord{12, 0, 1002}, + dictWord{12, 0, 1004}, + dictWord{12, 0, 1005}, + dictWord{ + 12, + 0, + 1006, + }, + dictWord{12, 0, 1007}, + dictWord{12, 0, 1008}, + dictWord{12, 0, 1009}, + dictWord{12, 0, 1010}, + dictWord{12, 0, 1011}, + dictWord{12, 0, 1012}, + dictWord{12, 0, 1014}, + dictWord{12, 0, 1015}, + dictWord{12, 0, 1016}, + dictWord{12, 0, 1017}, + dictWord{12, 0, 1018}, + dictWord{12, 0, 1019}, + dictWord{ + 12, + 0, + 1022, + }, + dictWord{12, 0, 1023}, + dictWord{14, 0, 475}, + dictWord{14, 0, 477}, + dictWord{14, 0, 478}, + dictWord{14, 0, 479}, + dictWord{14, 0, 480}, + dictWord{ + 14, + 0, + 482, + }, + dictWord{14, 0, 483}, + dictWord{14, 0, 484}, + dictWord{14, 0, 485}, + dictWord{14, 0, 486}, + dictWord{14, 0, 487}, + dictWord{14, 0, 488}, + dictWord{14, 0, 489}, + dictWord{14, 0, 490}, + dictWord{14, 0, 491}, + dictWord{14, 0, 492}, + dictWord{14, 0, 493}, + dictWord{14, 0, 494}, + dictWord{14, 0, 495}, + dictWord{14, 0, 496}, + dictWord{14, 0, 497}, + dictWord{14, 0, 498}, + dictWord{14, 0, 499}, + dictWord{14, 0, 500}, + dictWord{14, 0, 501}, + dictWord{14, 0, 502}, + dictWord{14, 0, 503}, + dictWord{ + 14, + 0, + 504, + }, + dictWord{14, 0, 506}, + dictWord{14, 0, 507}, + dictWord{14, 0, 508}, + dictWord{14, 0, 509}, + dictWord{14, 0, 510}, + dictWord{14, 0, 511}, + dictWord{ + 16, + 0, + 113, + }, + dictWord{16, 0, 114}, + dictWord{16, 0, 115}, + dictWord{16, 0, 117}, + dictWord{16, 0, 118}, + dictWord{16, 0, 119}, + dictWord{16, 0, 121}, + dictWord{16, 0, 122}, + dictWord{16, 0, 123}, + dictWord{16, 0, 124}, + dictWord{16, 0, 125}, + dictWord{16, 0, 126}, + dictWord{16, 0, 127}, + dictWord{18, 0, 242}, + dictWord{18, 0, 243}, + dictWord{18, 0, 244}, + dictWord{18, 0, 245}, + dictWord{18, 0, 248}, + dictWord{18, 0, 249}, + dictWord{18, 0, 250}, + dictWord{18, 0, 251}, + dictWord{18, 0, 252}, + dictWord{ + 18, + 0, + 253, + }, + dictWord{18, 0, 254}, + dictWord{18, 0, 255}, + dictWord{20, 0, 125}, + dictWord{20, 0, 126}, + dictWord{148, 0, 127}, + dictWord{7, 11, 1717}, + dictWord{ + 7, + 11, + 1769, + }, + dictWord{138, 11, 546}, + dictWord{7, 11, 1127}, + dictWord{7, 11, 1572}, + dictWord{10, 11, 297}, + dictWord{10, 11, 422}, + dictWord{11, 11, 764}, + dictWord{11, 11, 810}, + dictWord{12, 11, 264}, + dictWord{13, 11, 102}, + dictWord{13, 11, 300}, + dictWord{13, 11, 484}, + dictWord{14, 11, 147}, + dictWord{ + 14, + 11, + 229, + }, + dictWord{17, 11, 71}, + dictWord{18, 11, 118}, + dictWord{147, 11, 120}, + dictWord{6, 0, 1148}, + dictWord{134, 0, 1586}, + dictWord{132, 0, 775}, + dictWord{135, 10, 954}, + dictWord{133, 11, 864}, + dictWord{133, 11, 928}, + dictWord{138, 11, 189}, + dictWord{135, 10, 1958}, + dictWord{6, 10, 549}, + dictWord{ + 8, + 10, + 34, + }, + dictWord{8, 10, 283}, + dictWord{9, 10, 165}, + dictWord{138, 10, 475}, + dictWord{5, 10, 652}, + dictWord{5, 10, 701}, + dictWord{135, 10, 449}, + dictWord{135, 11, 695}, + dictWord{4, 10, 655}, + dictWord{7, 10, 850}, + dictWord{17, 10, 75}, + dictWord{146, 10, 137}, + dictWord{140, 11, 682}, + dictWord{ + 133, + 11, + 523, + }, + dictWord{8, 0, 970}, + dictWord{136, 10, 670}, + dictWord{136, 11, 555}, + dictWord{7, 11, 76}, + dictWord{8, 11, 44}, + dictWord{9, 11, 884}, + dictWord{ + 10, + 11, + 580, + }, + dictWord{11, 11, 399}, + dictWord{11, 11, 894}, + dictWord{15, 11, 122}, + dictWord{18, 11, 144}, + dictWord{147, 11, 61}, + dictWord{6, 10, 159}, + dictWord{ + 6, + 10, + 364, + }, + dictWord{7, 10, 516}, + dictWord{7, 10, 1439}, + dictWord{137, 10, 518}, + dictWord{4, 0, 71}, + dictWord{5, 0, 376}, + dictWord{7, 0, 119}, + dictWord{ + 138, + 0, + 665, + }, + dictWord{141, 10, 151}, + dictWord{11, 0, 827}, + dictWord{14, 0, 34}, + dictWord{143, 0, 148}, + dictWord{133, 11, 518}, + dictWord{4, 0, 479}, + dictWord{ + 135, + 11, + 1787, + }, + dictWord{135, 11, 1852}, + dictWord{135, 10, 993}, + dictWord{7, 0, 607}, + dictWord{136, 0, 99}, + dictWord{134, 0, 1960}, + dictWord{132, 0, 793}, + dictWord{4, 0, 41}, + dictWord{5, 0, 74}, + dictWord{7, 0, 1627}, + dictWord{11, 0, 871}, + dictWord{140, 0, 619}, + dictWord{7, 0, 94}, + dictWord{11, 0, 329}, + dictWord{ + 11, + 0, + 965, + }, + dictWord{12, 0, 241}, + dictWord{14, 0, 354}, + dictWord{15, 0, 22}, + dictWord{148, 0, 63}, + dictWord{7, 10, 501}, + dictWord{9, 10, 111}, + dictWord{10, 10, 141}, + dictWord{11, 10, 332}, + dictWord{13, 10, 43}, + dictWord{13, 10, 429}, + dictWord{14, 10, 130}, + dictWord{14, 10, 415}, + dictWord{145, 10, 102}, + dictWord{ + 9, + 0, + 209, + }, + dictWord{137, 0, 300}, + dictWord{134, 0, 1497}, + dictWord{138, 11, 255}, + dictWord{4, 11, 934}, + dictWord{5, 11, 138}, + dictWord{136, 11, 610}, + dictWord{133, 0, 98}, + dictWord{6, 0, 1316}, + dictWord{10, 11, 804}, + dictWord{138, 11, 832}, + dictWord{8, 11, 96}, + dictWord{9, 11, 36}, + dictWord{10, 11, 607}, + dictWord{11, 11, 423}, + dictWord{11, 11, 442}, + dictWord{12, 11, 309}, + dictWord{14, 11, 199}, + dictWord{15, 11, 90}, + dictWord{145, 11, 110}, + dictWord{ + 132, + 0, + 463, + }, + dictWord{5, 10, 149}, + dictWord{136, 10, 233}, + dictWord{133, 10, 935}, + dictWord{4, 11, 652}, + dictWord{8, 11, 320}, + dictWord{9, 11, 13}, + dictWord{ + 9, + 11, + 398, + }, + dictWord{9, 11, 727}, + dictWord{10, 11, 75}, + dictWord{10, 11, 184}, + dictWord{10, 11, 230}, + dictWord{10, 11, 564}, + dictWord{10, 11, 569}, + dictWord{ + 11, + 11, + 973, + }, + dictWord{12, 11, 70}, + dictWord{12, 11, 189}, + dictWord{13, 11, 57}, + dictWord{13, 11, 257}, + dictWord{22, 11, 6}, + dictWord{150, 11, 16}, + dictWord{ + 142, + 0, + 291, + }, + dictWord{12, 10, 582}, + dictWord{146, 10, 131}, + dictWord{136, 10, 801}, + dictWord{133, 0, 984}, + dictWord{145, 11, 116}, + dictWord{4, 11, 692}, + dictWord{133, 11, 321}, + dictWord{4, 0, 182}, + dictWord{6, 0, 205}, + dictWord{135, 0, 220}, + dictWord{4, 0, 42}, + dictWord{9, 0, 205}, + dictWord{9, 0, 786}, + dictWord{ + 138, + 0, + 659, + }, + dictWord{6, 0, 801}, + dictWord{11, 11, 130}, + dictWord{140, 11, 609}, + dictWord{132, 0, 635}, + dictWord{5, 11, 345}, + dictWord{135, 11, 1016}, + dictWord{139, 0, 533}, + dictWord{132, 0, 371}, + dictWord{4, 0, 272}, + dictWord{135, 0, 836}, + dictWord{6, 0, 1282}, + dictWord{135, 11, 1100}, + dictWord{5, 0, 825}, + dictWord{134, 0, 1640}, + dictWord{135, 11, 1325}, + dictWord{133, 11, 673}, + dictWord{4, 11, 287}, + dictWord{133, 11, 1018}, + dictWord{135, 0, 357}, + dictWord{ + 6, + 0, + 467, + }, + dictWord{137, 0, 879}, + dictWord{7, 0, 317}, + dictWord{135, 0, 569}, + dictWord{6, 0, 924}, + dictWord{134, 0, 1588}, + dictWord{5, 11, 34}, + dictWord{ + 5, + 10, + 406, + }, + dictWord{10, 11, 724}, + dictWord{12, 11, 444}, + dictWord{13, 11, 354}, + dictWord{18, 11, 32}, + dictWord{23, 11, 24}, + dictWord{23, 11, 31}, + dictWord{ + 152, + 11, + 5, + }, + dictWord{6, 0, 1795}, + dictWord{6, 0, 1835}, + dictWord{6, 0, 1836}, + dictWord{6, 0, 1856}, + dictWord{8, 0, 844}, + dictWord{8, 0, 849}, + dictWord{8, 0, 854}, + dictWord{8, 0, 870}, + dictWord{8, 0, 887}, + dictWord{10, 0, 852}, + dictWord{138, 0, 942}, + dictWord{6, 10, 69}, + dictWord{135, 10, 117}, + dictWord{137, 0, 307}, + dictWord{ + 4, + 0, + 944, + }, + dictWord{6, 0, 1799}, + dictWord{6, 0, 1825}, + dictWord{10, 0, 848}, + dictWord{10, 0, 875}, + dictWord{10, 0, 895}, + dictWord{10, 0, 899}, + dictWord{ + 10, + 0, + 902, + }, + dictWord{140, 0, 773}, + dictWord{11, 0, 43}, + dictWord{13, 0, 72}, + dictWord{141, 0, 142}, + dictWord{135, 10, 1830}, + dictWord{134, 11, 382}, + dictWord{ + 4, + 10, + 432, + }, + dictWord{135, 10, 824}, + dictWord{132, 11, 329}, + dictWord{7, 0, 1820}, + dictWord{139, 11, 124}, + dictWord{133, 10, 826}, + dictWord{ + 133, + 0, + 525, + }, + dictWord{132, 11, 906}, + dictWord{7, 11, 1940}, + dictWord{136, 11, 366}, + dictWord{138, 11, 10}, + dictWord{4, 11, 123}, + dictWord{4, 11, 649}, + dictWord{ + 5, + 11, + 605, + }, + dictWord{7, 11, 1509}, + dictWord{136, 11, 36}, + dictWord{6, 0, 110}, + dictWord{135, 0, 1681}, + dictWord{133, 0, 493}, + dictWord{133, 11, 767}, + dictWord{4, 0, 174}, + dictWord{135, 0, 911}, + dictWord{138, 11, 786}, + dictWord{8, 0, 417}, + dictWord{137, 0, 782}, + dictWord{133, 10, 1000}, + dictWord{7, 0, 733}, + dictWord{137, 0, 583}, + dictWord{4, 10, 297}, + dictWord{6, 10, 529}, + dictWord{7, 10, 152}, + dictWord{7, 10, 713}, + dictWord{7, 10, 1845}, + dictWord{8, 10, 710}, + dictWord{8, 10, 717}, + dictWord{12, 10, 639}, + dictWord{140, 10, 685}, + dictWord{4, 0, 32}, + dictWord{5, 0, 215}, + dictWord{6, 0, 269}, + dictWord{7, 0, 1782}, + dictWord{ + 7, + 0, + 1892, + }, + dictWord{10, 0, 16}, + dictWord{11, 0, 822}, + dictWord{11, 0, 954}, + dictWord{141, 0, 481}, + dictWord{4, 11, 273}, + dictWord{5, 11, 658}, + dictWord{ + 133, + 11, + 995, + }, + dictWord{136, 0, 477}, + dictWord{134, 11, 72}, + dictWord{135, 11, 1345}, + dictWord{5, 0, 308}, + dictWord{7, 0, 1088}, + dictWord{4, 10, 520}, + dictWord{ + 135, + 10, + 575, + }, + dictWord{133, 11, 589}, + dictWord{5, 0, 126}, + dictWord{8, 0, 297}, + dictWord{9, 0, 366}, + dictWord{140, 0, 374}, + dictWord{7, 0, 1551}, + dictWord{ + 139, + 0, + 361, + }, + dictWord{5, 11, 117}, + dictWord{6, 11, 514}, + dictWord{6, 11, 541}, + dictWord{7, 11, 1164}, + dictWord{7, 11, 1436}, + dictWord{8, 11, 220}, + dictWord{ + 8, + 11, + 648, + }, + dictWord{10, 11, 688}, + dictWord{139, 11, 560}, + dictWord{133, 11, 686}, + dictWord{4, 0, 946}, + dictWord{6, 0, 1807}, + dictWord{8, 0, 871}, + dictWord{ + 10, + 0, + 854, + }, + dictWord{10, 0, 870}, + dictWord{10, 0, 888}, + dictWord{10, 0, 897}, + dictWord{10, 0, 920}, + dictWord{12, 0, 722}, + dictWord{12, 0, 761}, + dictWord{ + 12, + 0, + 763, + }, + dictWord{12, 0, 764}, + dictWord{14, 0, 454}, + dictWord{14, 0, 465}, + dictWord{16, 0, 107}, + dictWord{18, 0, 167}, + dictWord{18, 0, 168}, + dictWord{ + 146, + 0, + 172, + }, + dictWord{132, 0, 175}, + dictWord{135, 0, 1307}, + dictWord{132, 0, 685}, + dictWord{135, 11, 1834}, + dictWord{133, 0, 797}, + dictWord{6, 0, 745}, + dictWord{ + 6, + 0, + 858, + }, + dictWord{134, 0, 963}, + dictWord{133, 0, 565}, + dictWord{5, 10, 397}, + dictWord{6, 10, 154}, + dictWord{7, 11, 196}, + dictWord{7, 10, 676}, + dictWord{ + 8, + 10, + 443, + }, + dictWord{8, 10, 609}, + dictWord{9, 10, 24}, + dictWord{9, 10, 325}, + dictWord{10, 10, 35}, + dictWord{10, 11, 765}, + dictWord{11, 11, 347}, + dictWord{ + 11, + 10, + 535, + }, + dictWord{11, 11, 552}, + dictWord{11, 11, 576}, + dictWord{11, 10, 672}, + dictWord{11, 11, 790}, + dictWord{11, 10, 1018}, + dictWord{12, 11, 263}, + dictWord{12, 10, 637}, + dictWord{13, 11, 246}, + dictWord{13, 11, 270}, + dictWord{13, 11, 395}, + dictWord{14, 11, 74}, + dictWord{14, 11, 176}, + dictWord{ + 14, + 11, + 190, + }, + dictWord{14, 11, 398}, + dictWord{14, 11, 412}, + dictWord{15, 11, 32}, + dictWord{15, 11, 63}, + dictWord{16, 10, 30}, + dictWord{16, 11, 88}, + dictWord{ + 147, + 11, + 105, + }, + dictWord{13, 11, 84}, + dictWord{141, 11, 122}, + dictWord{4, 0, 252}, + dictWord{7, 0, 1068}, + dictWord{10, 0, 434}, + dictWord{11, 0, 228}, + dictWord{ + 11, + 0, + 426, + }, + dictWord{13, 0, 231}, + dictWord{18, 0, 106}, + dictWord{148, 0, 87}, + dictWord{137, 0, 826}, + dictWord{4, 11, 589}, + dictWord{139, 11, 282}, + dictWord{ + 5, + 11, + 381, + }, + dictWord{135, 11, 1792}, + dictWord{132, 0, 791}, + dictWord{5, 0, 231}, + dictWord{10, 0, 509}, + dictWord{133, 10, 981}, + dictWord{7, 0, 601}, + dictWord{ + 9, + 0, + 277, + }, + dictWord{9, 0, 674}, + dictWord{10, 0, 178}, + dictWord{10, 0, 418}, + dictWord{10, 0, 571}, + dictWord{11, 0, 531}, + dictWord{12, 0, 113}, + dictWord{12, 0, 475}, + dictWord{13, 0, 99}, + dictWord{142, 0, 428}, + dictWord{4, 10, 56}, + dictWord{7, 11, 616}, + dictWord{7, 10, 1791}, + dictWord{8, 10, 607}, + dictWord{8, 10, 651}, + dictWord{10, 11, 413}, + dictWord{11, 10, 465}, + dictWord{11, 10, 835}, + dictWord{12, 10, 337}, + dictWord{141, 10, 480}, + dictWord{7, 0, 1591}, + dictWord{144, 0, 43}, + dictWord{9, 10, 158}, + dictWord{138, 10, 411}, + dictWord{135, 0, 1683}, + dictWord{8, 0, 289}, + dictWord{11, 0, 45}, + dictWord{12, 0, 278}, + dictWord{140, 0, 537}, + dictWord{6, 11, 120}, + dictWord{7, 11, 1188}, + dictWord{7, 11, 1710}, + dictWord{8, 11, 286}, + dictWord{9, 11, 667}, + dictWord{11, 11, 592}, + dictWord{ + 139, + 11, + 730, + }, + dictWord{136, 10, 617}, + dictWord{135, 0, 1120}, + dictWord{135, 11, 1146}, + dictWord{139, 10, 563}, + dictWord{4, 11, 352}, + dictWord{4, 10, 369}, + dictWord{135, 11, 687}, + dictWord{143, 11, 38}, + dictWord{4, 0, 399}, + dictWord{5, 0, 119}, + dictWord{5, 0, 494}, + dictWord{7, 0, 751}, + dictWord{9, 0, 556}, + dictWord{ + 14, + 11, + 179, + }, + dictWord{15, 11, 151}, + dictWord{150, 11, 11}, + dictWord{4, 11, 192}, + dictWord{5, 11, 49}, + dictWord{6, 11, 200}, + dictWord{6, 11, 293}, + dictWord{ + 6, + 11, + 1696, + }, + dictWord{135, 11, 488}, + dictWord{4, 0, 398}, + dictWord{133, 0, 660}, + dictWord{7, 0, 1030}, + dictWord{134, 10, 622}, + dictWord{135, 11, 595}, + dictWord{141, 0, 168}, + dictWord{132, 11, 147}, + dictWord{7, 0, 973}, + dictWord{10, 10, 624}, + dictWord{142, 10, 279}, + dictWord{132, 10, 363}, + dictWord{ + 132, + 0, + 642, + }, + dictWord{133, 11, 934}, + dictWord{134, 0, 1615}, + dictWord{7, 11, 505}, + dictWord{135, 11, 523}, + dictWord{7, 0, 594}, + dictWord{7, 0, 851}, + dictWord{ + 7, + 0, + 1858, + }, + dictWord{9, 0, 411}, + dictWord{9, 0, 574}, + dictWord{9, 0, 666}, + dictWord{9, 0, 737}, + dictWord{10, 0, 346}, + dictWord{10, 0, 712}, + dictWord{11, 0, 246}, + dictWord{11, 0, 432}, + dictWord{11, 0, 517}, + dictWord{11, 0, 647}, + dictWord{11, 0, 679}, + dictWord{11, 0, 727}, + dictWord{12, 0, 304}, + dictWord{12, 0, 305}, + dictWord{ + 12, + 0, + 323, + }, + dictWord{12, 0, 483}, + dictWord{12, 0, 572}, + dictWord{12, 0, 593}, + dictWord{12, 0, 602}, + dictWord{13, 0, 95}, + dictWord{13, 0, 101}, + dictWord{ + 13, + 0, + 171, + }, + dictWord{13, 0, 315}, + dictWord{13, 0, 378}, + dictWord{13, 0, 425}, + dictWord{13, 0, 475}, + dictWord{14, 0, 63}, + dictWord{14, 0, 380}, + dictWord{14, 0, 384}, + dictWord{15, 0, 133}, + dictWord{18, 0, 112}, + dictWord{148, 0, 72}, + dictWord{135, 0, 1093}, + dictWord{132, 0, 679}, + dictWord{8, 0, 913}, + dictWord{10, 0, 903}, + dictWord{10, 0, 915}, + dictWord{12, 0, 648}, + dictWord{12, 0, 649}, + dictWord{14, 0, 455}, + dictWord{16, 0, 112}, + dictWord{138, 11, 438}, + dictWord{137, 0, 203}, + dictWord{134, 10, 292}, + dictWord{134, 0, 1492}, + dictWord{7, 0, 1374}, + dictWord{8, 0, 540}, + dictWord{5, 10, 177}, + dictWord{6, 10, 616}, + dictWord{7, 10, 827}, + dictWord{9, 10, 525}, + dictWord{138, 10, 656}, + dictWord{135, 0, 1486}, + dictWord{9, 0, 714}, + dictWord{138, 10, 31}, + dictWord{136, 0, 825}, + dictWord{ + 134, + 0, + 1511, + }, + dictWord{132, 11, 637}, + dictWord{134, 0, 952}, + dictWord{4, 10, 161}, + dictWord{133, 10, 631}, + dictWord{5, 0, 143}, + dictWord{5, 0, 769}, + dictWord{ + 6, + 0, + 1760, + }, + dictWord{7, 0, 682}, + dictWord{7, 0, 1992}, + dictWord{136, 0, 736}, + dictWord{132, 0, 700}, + dictWord{134, 0, 1540}, + dictWord{132, 11, 777}, + dictWord{ + 9, + 11, + 867, + }, + dictWord{138, 11, 837}, + dictWord{7, 0, 1557}, + dictWord{135, 10, 1684}, + dictWord{133, 0, 860}, + dictWord{6, 0, 422}, + dictWord{7, 0, 0}, + dictWord{ + 7, + 0, + 1544, + }, + dictWord{9, 0, 605}, + dictWord{11, 0, 990}, + dictWord{12, 0, 235}, + dictWord{12, 0, 453}, + dictWord{13, 0, 47}, + dictWord{13, 0, 266}, + dictWord{9, 10, 469}, + dictWord{9, 10, 709}, + dictWord{12, 10, 512}, + dictWord{14, 10, 65}, + dictWord{145, 10, 12}, + dictWord{11, 0, 807}, + dictWord{10, 10, 229}, + dictWord{11, 10, 73}, + dictWord{139, 10, 376}, + dictWord{6, 11, 170}, + dictWord{7, 11, 1080}, + dictWord{8, 11, 395}, + dictWord{8, 11, 487}, + dictWord{11, 11, 125}, + dictWord{ + 141, + 11, + 147, + }, + dictWord{5, 0, 515}, + dictWord{137, 0, 131}, + dictWord{7, 0, 1605}, + dictWord{11, 0, 962}, + dictWord{146, 0, 139}, + dictWord{132, 0, 646}, + dictWord{ + 4, + 0, + 396, + }, + dictWord{7, 0, 728}, + dictWord{9, 0, 117}, + dictWord{13, 0, 202}, + dictWord{148, 0, 51}, + dictWord{6, 0, 121}, + dictWord{6, 0, 124}, + dictWord{6, 0, 357}, + dictWord{ + 7, + 0, + 1138, + }, + dictWord{7, 0, 1295}, + dictWord{8, 0, 162}, + dictWord{8, 0, 508}, + dictWord{11, 0, 655}, + dictWord{4, 11, 535}, + dictWord{6, 10, 558}, + dictWord{ + 7, + 10, + 651, + }, + dictWord{8, 11, 618}, + dictWord{9, 10, 0}, + dictWord{10, 10, 34}, + dictWord{139, 10, 1008}, + dictWord{135, 11, 1245}, + dictWord{138, 0, 357}, + dictWord{ + 150, + 11, + 23, + }, + dictWord{133, 0, 237}, + dictWord{135, 0, 1784}, + dictWord{7, 10, 1832}, + dictWord{138, 10, 374}, + dictWord{132, 0, 713}, + dictWord{132, 11, 46}, + dictWord{6, 0, 1536}, + dictWord{10, 0, 348}, + dictWord{5, 11, 811}, + dictWord{6, 11, 1679}, + dictWord{6, 11, 1714}, + dictWord{135, 11, 2032}, + dictWord{ + 11, + 11, + 182, + }, + dictWord{142, 11, 195}, + dictWord{6, 0, 523}, + dictWord{7, 0, 738}, + dictWord{7, 10, 771}, + dictWord{7, 10, 1731}, + dictWord{9, 10, 405}, + dictWord{ + 138, + 10, + 421, + }, + dictWord{7, 11, 1458}, + dictWord{9, 11, 407}, + dictWord{139, 11, 15}, + dictWord{6, 11, 34}, + dictWord{7, 11, 69}, + dictWord{7, 11, 640}, + dictWord{ + 7, + 11, + 1089, + }, + dictWord{8, 11, 708}, + dictWord{8, 11, 721}, + dictWord{9, 11, 363}, + dictWord{9, 11, 643}, + dictWord{10, 11, 628}, + dictWord{148, 11, 98}, + dictWord{ + 133, + 0, + 434, + }, + dictWord{135, 0, 1877}, + dictWord{7, 0, 571}, + dictWord{138, 0, 366}, + dictWord{5, 10, 881}, + dictWord{133, 10, 885}, + dictWord{9, 0, 513}, + dictWord{ + 10, + 0, + 25, + }, + dictWord{10, 0, 39}, + dictWord{12, 0, 122}, + dictWord{140, 0, 187}, + dictWord{132, 0, 580}, + dictWord{5, 10, 142}, + dictWord{134, 10, 546}, + dictWord{ + 132, + 11, + 462, + }, + dictWord{137, 0, 873}, + dictWord{5, 10, 466}, + dictWord{11, 10, 571}, + dictWord{12, 10, 198}, + dictWord{13, 10, 283}, + dictWord{14, 10, 186}, + dictWord{15, 10, 21}, + dictWord{143, 10, 103}, + dictWord{7, 0, 171}, + dictWord{4, 10, 185}, + dictWord{5, 10, 257}, + dictWord{5, 10, 839}, + dictWord{5, 10, 936}, + dictWord{ + 9, + 10, + 399, + }, + dictWord{10, 10, 258}, + dictWord{10, 10, 395}, + dictWord{10, 10, 734}, + dictWord{11, 10, 1014}, + dictWord{12, 10, 23}, + dictWord{13, 10, 350}, + dictWord{14, 10, 150}, + dictWord{147, 10, 6}, + dictWord{134, 0, 625}, + dictWord{7, 0, 107}, + dictWord{7, 0, 838}, + dictWord{8, 0, 550}, + dictWord{138, 0, 401}, + dictWord{ + 5, + 11, + 73, + }, + dictWord{6, 11, 23}, + dictWord{134, 11, 338}, + dictWord{4, 0, 943}, + dictWord{6, 0, 1850}, + dictWord{12, 0, 713}, + dictWord{142, 0, 434}, + dictWord{ + 11, + 0, + 588, + }, + dictWord{11, 0, 864}, + dictWord{11, 0, 936}, + dictWord{11, 0, 968}, + dictWord{12, 0, 73}, + dictWord{12, 0, 343}, + dictWord{12, 0, 394}, + dictWord{13, 0, 275}, + dictWord{14, 0, 257}, + dictWord{15, 0, 160}, + dictWord{7, 10, 404}, + dictWord{7, 10, 1377}, + dictWord{7, 10, 1430}, + dictWord{7, 10, 2017}, + dictWord{8, 10, 149}, + dictWord{8, 10, 239}, + dictWord{8, 10, 512}, + dictWord{8, 10, 793}, + dictWord{8, 10, 818}, + dictWord{9, 10, 474}, + dictWord{9, 10, 595}, + dictWord{10, 10, 122}, + dictWord{10, 10, 565}, + dictWord{10, 10, 649}, + dictWord{10, 10, 783}, + dictWord{11, 10, 239}, + dictWord{11, 10, 295}, + dictWord{11, 10, 447}, + dictWord{ + 11, + 10, + 528, + }, + dictWord{11, 10, 639}, + dictWord{11, 10, 800}, + dictWord{12, 10, 25}, + dictWord{12, 10, 157}, + dictWord{12, 10, 316}, + dictWord{12, 10, 390}, + dictWord{ + 12, + 10, + 391, + }, + dictWord{12, 10, 395}, + dictWord{12, 10, 478}, + dictWord{12, 10, 503}, + dictWord{12, 10, 592}, + dictWord{12, 10, 680}, + dictWord{13, 10, 50}, + dictWord{13, 10, 53}, + dictWord{13, 10, 132}, + dictWord{13, 10, 198}, + dictWord{13, 10, 322}, + dictWord{13, 10, 415}, + dictWord{13, 10, 511}, + dictWord{14, 10, 71}, + dictWord{14, 10, 395}, + dictWord{15, 10, 71}, + dictWord{15, 10, 136}, + dictWord{17, 10, 123}, + dictWord{18, 10, 93}, + dictWord{147, 10, 58}, + dictWord{ + 133, + 0, + 768, + }, + dictWord{11, 0, 103}, + dictWord{142, 0, 0}, + dictWord{136, 10, 712}, + dictWord{132, 0, 799}, + dictWord{132, 0, 894}, + dictWord{7, 11, 725}, + dictWord{ + 8, + 11, + 498, + }, + dictWord{139, 11, 268}, + dictWord{135, 11, 1798}, + dictWord{135, 11, 773}, + dictWord{141, 11, 360}, + dictWord{4, 10, 377}, + dictWord{152, 10, 13}, + dictWord{135, 0, 1673}, + dictWord{132, 11, 583}, + dictWord{134, 0, 1052}, + dictWord{133, 11, 220}, + dictWord{140, 11, 69}, + dictWord{132, 11, 544}, + dictWord{ + 4, + 10, + 180, + }, + dictWord{135, 10, 1906}, + dictWord{134, 0, 272}, + dictWord{4, 0, 441}, + dictWord{134, 0, 1421}, + dictWord{4, 0, 9}, + dictWord{5, 0, 128}, + dictWord{ + 7, + 0, + 368, + }, + dictWord{11, 0, 480}, + dictWord{148, 0, 3}, + dictWord{5, 11, 176}, + dictWord{6, 11, 437}, + dictWord{6, 11, 564}, + dictWord{11, 11, 181}, + dictWord{ + 141, + 11, + 183, + }, + dictWord{132, 10, 491}, + dictWord{7, 0, 1182}, + dictWord{141, 11, 67}, + dictWord{6, 0, 1346}, + dictWord{4, 10, 171}, + dictWord{138, 10, 234}, + dictWord{ + 4, + 10, + 586, + }, + dictWord{7, 10, 1186}, + dictWord{138, 10, 631}, + dictWord{136, 0, 682}, + dictWord{134, 0, 1004}, + dictWord{15, 0, 24}, + dictWord{143, 11, 24}, + dictWord{134, 0, 968}, + dictWord{4, 0, 2}, + dictWord{6, 0, 742}, + dictWord{6, 0, 793}, + dictWord{7, 0, 545}, + dictWord{7, 0, 894}, + dictWord{9, 10, 931}, + dictWord{ + 10, + 10, + 334, + }, + dictWord{148, 10, 71}, + dictWord{136, 11, 600}, + dictWord{133, 10, 765}, + dictWord{9, 0, 769}, + dictWord{140, 0, 185}, + dictWord{4, 11, 790}, + dictWord{ + 5, + 11, + 273, + }, + dictWord{134, 11, 394}, + dictWord{7, 0, 474}, + dictWord{137, 0, 578}, + dictWord{4, 11, 135}, + dictWord{6, 11, 127}, + dictWord{7, 11, 1185}, + dictWord{ + 7, + 11, + 1511, + }, + dictWord{8, 11, 613}, + dictWord{11, 11, 5}, + dictWord{12, 11, 133}, + dictWord{12, 11, 495}, + dictWord{12, 11, 586}, + dictWord{14, 11, 385}, + dictWord{15, 11, 118}, + dictWord{17, 11, 20}, + dictWord{146, 11, 98}, + dictWord{133, 10, 424}, + dictWord{5, 0, 530}, + dictWord{142, 0, 113}, + dictWord{6, 11, 230}, + dictWord{7, 11, 961}, + dictWord{7, 11, 1085}, + dictWord{136, 11, 462}, + dictWord{7, 11, 1954}, + dictWord{137, 11, 636}, + dictWord{136, 10, 714}, + dictWord{ + 149, + 11, + 6, + }, + dictWord{135, 10, 685}, + dictWord{9, 10, 420}, + dictWord{10, 10, 269}, + dictWord{10, 10, 285}, + dictWord{10, 10, 576}, + dictWord{11, 10, 397}, + dictWord{13, 10, 175}, + dictWord{145, 10, 90}, + dictWord{132, 10, 429}, + dictWord{5, 0, 556}, + dictWord{5, 11, 162}, + dictWord{136, 11, 68}, + dictWord{132, 11, 654}, + dictWord{4, 11, 156}, + dictWord{7, 11, 998}, + dictWord{7, 11, 1045}, + dictWord{7, 11, 1860}, + dictWord{9, 11, 48}, + dictWord{9, 11, 692}, + dictWord{11, 11, 419}, + dictWord{139, 11, 602}, + dictWord{6, 0, 1317}, + dictWord{8, 0, 16}, + dictWord{9, 0, 825}, + dictWord{12, 0, 568}, + dictWord{7, 11, 1276}, + dictWord{8, 11, 474}, + dictWord{137, 11, 652}, + dictWord{18, 0, 97}, + dictWord{7, 10, 18}, + dictWord{7, 10, 699}, + dictWord{7, 10, 1966}, + dictWord{8, 10, 752}, + dictWord{9, 10, 273}, + dictWord{ + 9, + 10, + 412, + }, + dictWord{9, 10, 703}, + dictWord{10, 10, 71}, + dictWord{10, 10, 427}, + dictWord{138, 10, 508}, + dictWord{10, 0, 703}, + dictWord{7, 11, 1454}, + dictWord{138, 11, 703}, + dictWord{4, 10, 53}, + dictWord{5, 10, 186}, + dictWord{135, 10, 752}, + dictWord{134, 0, 892}, + dictWord{134, 0, 1571}, + dictWord{8, 10, 575}, + dictWord{10, 10, 289}, + dictWord{139, 10, 319}, + dictWord{6, 0, 186}, + dictWord{137, 0, 426}, + dictWord{134, 0, 1101}, + dictWord{132, 10, 675}, + dictWord{ + 132, + 0, + 585, + }, + dictWord{6, 0, 1870}, + dictWord{137, 0, 937}, + dictWord{152, 11, 10}, + dictWord{9, 11, 197}, + dictWord{10, 11, 300}, + dictWord{12, 11, 473}, + dictWord{ + 13, + 11, + 90, + }, + dictWord{141, 11, 405}, + dictWord{4, 0, 93}, + dictWord{5, 0, 252}, + dictWord{6, 0, 229}, + dictWord{7, 0, 291}, + dictWord{9, 0, 550}, + dictWord{139, 0, 644}, + dictWord{137, 0, 749}, + dictWord{9, 0, 162}, + dictWord{6, 10, 209}, + dictWord{8, 10, 468}, + dictWord{9, 10, 210}, + dictWord{11, 10, 36}, + dictWord{12, 10, 28}, + dictWord{12, 10, 630}, + dictWord{13, 10, 21}, + dictWord{13, 10, 349}, + dictWord{14, 10, 7}, + dictWord{145, 10, 13}, + dictWord{132, 0, 381}, + dictWord{132, 11, 606}, + dictWord{4, 10, 342}, + dictWord{135, 10, 1179}, + dictWord{7, 11, 1587}, + dictWord{7, 11, 1707}, + dictWord{10, 11, 528}, + dictWord{139, 11, 504}, + dictWord{ + 12, + 11, + 39, + }, + dictWord{13, 11, 265}, + dictWord{141, 11, 439}, + dictWord{4, 10, 928}, + dictWord{133, 10, 910}, + dictWord{7, 10, 1838}, + dictWord{7, 11, 1978}, + dictWord{136, 11, 676}, + dictWord{6, 0, 762}, + dictWord{6, 0, 796}, + dictWord{134, 0, 956}, + dictWord{4, 10, 318}, + dictWord{4, 10, 496}, + dictWord{7, 10, 856}, + dictWord{139, 10, 654}, + dictWord{137, 11, 242}, + dictWord{4, 11, 361}, + dictWord{133, 11, 315}, + dictWord{132, 11, 461}, + dictWord{132, 11, 472}, + dictWord{ + 132, + 0, + 857, + }, + dictWord{5, 0, 21}, + dictWord{6, 0, 77}, + dictWord{6, 0, 157}, + dictWord{7, 0, 974}, + dictWord{7, 0, 1301}, + dictWord{7, 0, 1339}, + dictWord{7, 0, 1490}, + dictWord{ + 7, + 0, + 1873, + }, + dictWord{9, 0, 628}, + dictWord{7, 10, 915}, + dictWord{8, 10, 247}, + dictWord{147, 10, 0}, + dictWord{4, 10, 202}, + dictWord{5, 10, 382}, + dictWord{ + 6, + 10, + 454, + }, + dictWord{7, 10, 936}, + dictWord{7, 10, 1803}, + dictWord{8, 10, 758}, + dictWord{9, 10, 375}, + dictWord{9, 10, 895}, + dictWord{10, 10, 743}, + dictWord{ + 10, + 10, + 792, + }, + dictWord{11, 10, 978}, + dictWord{11, 10, 1012}, + dictWord{142, 10, 109}, + dictWord{7, 11, 617}, + dictWord{10, 11, 498}, + dictWord{11, 11, 501}, + dictWord{12, 11, 16}, + dictWord{140, 11, 150}, + dictWord{7, 10, 1150}, + dictWord{7, 10, 1425}, + dictWord{7, 10, 1453}, + dictWord{10, 11, 747}, + dictWord{ + 140, + 10, + 513, + }, + dictWord{133, 11, 155}, + dictWord{11, 0, 919}, + dictWord{141, 0, 409}, + dictWord{138, 10, 791}, + dictWord{10, 0, 633}, + dictWord{139, 11, 729}, + dictWord{ + 7, + 11, + 163, + }, + dictWord{8, 11, 319}, + dictWord{9, 11, 402}, + dictWord{10, 11, 24}, + dictWord{10, 11, 681}, + dictWord{11, 11, 200}, + dictWord{11, 11, 567}, + dictWord{12, 11, 253}, + dictWord{12, 11, 410}, + dictWord{142, 11, 219}, + dictWord{5, 11, 475}, + dictWord{7, 11, 1780}, + dictWord{9, 11, 230}, + dictWord{11, 11, 297}, + dictWord{11, 11, 558}, + dictWord{14, 11, 322}, + dictWord{147, 11, 76}, + dictWord{7, 0, 332}, + dictWord{6, 10, 445}, + dictWord{137, 10, 909}, + dictWord{ + 135, + 11, + 1956, + }, + dictWord{136, 11, 274}, + dictWord{134, 10, 578}, + dictWord{135, 0, 1489}, + dictWord{135, 11, 1848}, + dictWord{5, 11, 944}, + dictWord{ + 134, + 11, + 1769, + }, + dictWord{132, 11, 144}, + dictWord{136, 10, 766}, + dictWord{4, 0, 832}, + dictWord{135, 10, 541}, + dictWord{8, 0, 398}, + dictWord{9, 0, 681}, + dictWord{ + 139, + 0, + 632, + }, + dictWord{136, 0, 645}, + dictWord{9, 0, 791}, + dictWord{10, 0, 93}, + dictWord{16, 0, 13}, + dictWord{17, 0, 23}, + dictWord{18, 0, 135}, + dictWord{19, 0, 12}, + dictWord{20, 0, 1}, + dictWord{20, 0, 12}, + dictWord{148, 0, 14}, + dictWord{6, 11, 247}, + dictWord{137, 11, 555}, + dictWord{134, 0, 20}, + dictWord{132, 0, 800}, + dictWord{135, 0, 1841}, + dictWord{139, 10, 983}, + dictWord{137, 10, 768}, + dictWord{132, 10, 584}, + dictWord{141, 11, 51}, + dictWord{6, 0, 1993}, + dictWord{ + 4, + 11, + 620, + }, + dictWord{138, 11, 280}, + dictWord{136, 0, 769}, + dictWord{11, 0, 290}, + dictWord{11, 0, 665}, + dictWord{7, 11, 1810}, + dictWord{11, 11, 866}, + dictWord{ + 12, + 11, + 103, + }, + dictWord{13, 11, 495}, + dictWord{17, 11, 67}, + dictWord{147, 11, 74}, + dictWord{134, 0, 1426}, + dictWord{139, 0, 60}, + dictWord{4, 10, 326}, + dictWord{135, 10, 1770}, + dictWord{7, 0, 1874}, + dictWord{9, 0, 641}, + dictWord{132, 10, 226}, + dictWord{6, 0, 644}, + dictWord{5, 10, 426}, + dictWord{8, 10, 30}, + dictWord{ + 9, + 10, + 2, + }, + dictWord{11, 10, 549}, + dictWord{147, 10, 122}, + dictWord{5, 11, 428}, + dictWord{138, 11, 442}, + dictWord{135, 11, 1871}, + dictWord{ + 135, + 0, + 1757, + }, + dictWord{147, 10, 117}, + dictWord{135, 0, 937}, + dictWord{135, 0, 1652}, + dictWord{6, 0, 654}, + dictWord{134, 0, 1476}, + dictWord{133, 11, 99}, + dictWord{135, 0, 527}, + dictWord{132, 10, 345}, + dictWord{4, 10, 385}, + dictWord{4, 11, 397}, + dictWord{7, 10, 265}, + dictWord{135, 10, 587}, + dictWord{4, 0, 579}, + dictWord{5, 0, 226}, + dictWord{5, 0, 323}, + dictWord{135, 0, 960}, + dictWord{134, 0, 1486}, + dictWord{8, 11, 502}, + dictWord{144, 11, 9}, + dictWord{4, 10, 347}, + dictWord{ + 5, + 10, + 423, + }, + dictWord{5, 10, 996}, + dictWord{135, 10, 1329}, + dictWord{7, 11, 727}, + dictWord{146, 11, 73}, + dictWord{4, 11, 485}, + dictWord{7, 11, 353}, + dictWord{7, 10, 1259}, + dictWord{7, 11, 1523}, + dictWord{9, 10, 125}, + dictWord{139, 10, 65}, + dictWord{6, 0, 325}, + dictWord{5, 10, 136}, + dictWord{6, 11, 366}, + dictWord{ + 7, + 11, + 1384, + }, + dictWord{7, 11, 1601}, + dictWord{136, 10, 644}, + dictWord{138, 11, 160}, + dictWord{6, 0, 1345}, + dictWord{137, 11, 282}, + dictWord{18, 0, 91}, + dictWord{147, 0, 70}, + dictWord{136, 0, 404}, + dictWord{4, 11, 157}, + dictWord{133, 11, 471}, + dictWord{133, 0, 973}, + dictWord{6, 0, 135}, + dictWord{ + 135, + 0, + 1176, + }, + dictWord{8, 11, 116}, + dictWord{11, 11, 551}, + dictWord{142, 11, 159}, + dictWord{4, 0, 549}, + dictWord{4, 10, 433}, + dictWord{133, 10, 719}, + dictWord{ + 136, + 0, + 976, + }, + dictWord{5, 11, 160}, + dictWord{7, 11, 363}, + dictWord{7, 11, 589}, + dictWord{10, 11, 170}, + dictWord{141, 11, 55}, + dictWord{144, 0, 21}, + dictWord{ + 144, + 0, + 51, + }, + dictWord{135, 0, 314}, + dictWord{135, 10, 1363}, + dictWord{4, 11, 108}, + dictWord{7, 11, 405}, + dictWord{10, 11, 491}, + dictWord{139, 11, 498}, + dictWord{146, 0, 4}, + dictWord{4, 10, 555}, + dictWord{8, 10, 536}, + dictWord{10, 10, 288}, + dictWord{139, 10, 1005}, + dictWord{135, 11, 1005}, + dictWord{6, 0, 281}, + dictWord{7, 0, 6}, + dictWord{8, 0, 282}, + dictWord{8, 0, 480}, + dictWord{8, 0, 499}, + dictWord{9, 0, 198}, + dictWord{10, 0, 143}, + dictWord{10, 0, 169}, + dictWord{ + 10, + 0, + 211, + }, + dictWord{10, 0, 417}, + dictWord{10, 0, 574}, + dictWord{11, 0, 147}, + dictWord{11, 0, 395}, + dictWord{12, 0, 75}, + dictWord{12, 0, 407}, + dictWord{12, 0, 608}, + dictWord{13, 0, 500}, + dictWord{142, 0, 251}, + dictWord{6, 0, 1093}, + dictWord{6, 0, 1405}, + dictWord{9, 10, 370}, + dictWord{138, 10, 90}, + dictWord{4, 11, 926}, + dictWord{133, 11, 983}, + dictWord{135, 0, 1776}, + dictWord{134, 0, 1528}, + dictWord{132, 0, 419}, + dictWord{132, 11, 538}, + dictWord{6, 11, 294}, + dictWord{ + 7, + 11, + 1267, + }, + dictWord{136, 11, 624}, + dictWord{135, 11, 1772}, + dictWord{138, 11, 301}, + dictWord{4, 10, 257}, + dictWord{135, 10, 2031}, + dictWord{4, 0, 138}, + dictWord{7, 0, 1012}, + dictWord{7, 0, 1280}, + dictWord{9, 0, 76}, + dictWord{135, 10, 1768}, + dictWord{132, 11, 757}, + dictWord{5, 0, 29}, + dictWord{140, 0, 638}, + dictWord{7, 11, 655}, + dictWord{135, 11, 1844}, + dictWord{7, 0, 1418}, + dictWord{6, 11, 257}, + dictWord{135, 11, 1522}, + dictWord{8, 11, 469}, + dictWord{ + 138, + 11, + 47, + }, + dictWord{142, 11, 278}, + dictWord{6, 10, 83}, + dictWord{6, 10, 1733}, + dictWord{135, 10, 1389}, + dictWord{11, 11, 204}, + dictWord{11, 11, 243}, + dictWord{140, 11, 293}, + dictWord{135, 11, 1875}, + dictWord{6, 0, 1710}, + dictWord{135, 0, 2038}, + dictWord{137, 11, 299}, + dictWord{4, 0, 17}, + dictWord{5, 0, 23}, + dictWord{7, 0, 995}, + dictWord{11, 0, 383}, + dictWord{11, 0, 437}, + dictWord{12, 0, 460}, + dictWord{140, 0, 532}, + dictWord{133, 0, 862}, + dictWord{137, 10, 696}, + dictWord{6, 0, 592}, + dictWord{138, 0, 946}, + dictWord{138, 11, 599}, + dictWord{7, 10, 1718}, + dictWord{9, 10, 95}, + dictWord{9, 10, 274}, + dictWord{10, 10, 279}, + dictWord{10, 10, 317}, + dictWord{10, 10, 420}, + dictWord{11, 10, 303}, + dictWord{11, 10, 808}, + dictWord{12, 10, 134}, + dictWord{12, 10, 367}, + dictWord{ + 13, + 10, + 149, + }, + dictWord{13, 10, 347}, + dictWord{14, 10, 349}, + dictWord{14, 10, 406}, + dictWord{18, 10, 22}, + dictWord{18, 10, 89}, + dictWord{18, 10, 122}, + dictWord{ + 147, + 10, + 47, + }, + dictWord{8, 0, 70}, + dictWord{12, 0, 171}, + dictWord{141, 0, 272}, + dictWord{133, 10, 26}, + dictWord{132, 10, 550}, + dictWord{137, 0, 812}, + dictWord{ + 10, + 0, + 233, + }, + dictWord{139, 0, 76}, + dictWord{134, 0, 988}, + dictWord{134, 0, 442}, + dictWord{136, 10, 822}, + dictWord{7, 0, 896}, + dictWord{4, 10, 902}, + dictWord{ + 5, + 10, + 809, + }, + dictWord{134, 10, 122}, + dictWord{5, 11, 150}, + dictWord{7, 11, 106}, + dictWord{8, 11, 603}, + dictWord{9, 11, 593}, + dictWord{9, 11, 634}, + dictWord{ + 10, + 11, + 44, + }, + dictWord{10, 11, 173}, + dictWord{11, 11, 462}, + dictWord{11, 11, 515}, + dictWord{13, 11, 216}, + dictWord{13, 11, 288}, + dictWord{142, 11, 400}, + dictWord{136, 0, 483}, + dictWord{135, 10, 262}, + dictWord{6, 0, 1709}, + dictWord{133, 10, 620}, + dictWord{4, 10, 34}, + dictWord{5, 10, 574}, + dictWord{7, 10, 279}, + dictWord{7, 10, 1624}, + dictWord{136, 10, 601}, + dictWord{137, 10, 170}, + dictWord{147, 0, 119}, + dictWord{12, 11, 108}, + dictWord{141, 11, 291}, + dictWord{ + 11, + 0, + 69, + }, + dictWord{12, 0, 105}, + dictWord{12, 0, 117}, + dictWord{13, 0, 213}, + dictWord{14, 0, 13}, + dictWord{14, 0, 62}, + dictWord{14, 0, 177}, + dictWord{14, 0, 421}, + dictWord{15, 0, 19}, + dictWord{146, 0, 141}, + dictWord{137, 0, 309}, + dictWord{11, 11, 278}, + dictWord{142, 11, 73}, + dictWord{7, 0, 608}, + dictWord{7, 0, 976}, + dictWord{9, 0, 146}, + dictWord{10, 0, 206}, + dictWord{10, 0, 596}, + dictWord{13, 0, 218}, + dictWord{142, 0, 153}, + dictWord{133, 10, 332}, + dictWord{6, 10, 261}, + dictWord{ + 8, + 10, + 182, + }, + dictWord{139, 10, 943}, + dictWord{4, 11, 493}, + dictWord{144, 11, 55}, + dictWord{134, 10, 1721}, + dictWord{132, 0, 768}, + dictWord{4, 10, 933}, + dictWord{133, 10, 880}, + dictWord{7, 11, 555}, + dictWord{7, 11, 1316}, + dictWord{7, 11, 1412}, + dictWord{7, 11, 1839}, + dictWord{9, 11, 192}, + dictWord{ + 9, + 11, + 589, + }, + dictWord{11, 11, 241}, + dictWord{11, 11, 676}, + dictWord{11, 11, 811}, + dictWord{11, 11, 891}, + dictWord{12, 11, 140}, + dictWord{12, 11, 346}, + dictWord{ + 12, + 11, + 479, + }, + dictWord{13, 11, 30}, + dictWord{13, 11, 49}, + dictWord{13, 11, 381}, + dictWord{14, 11, 188}, + dictWord{15, 11, 150}, + dictWord{16, 11, 76}, + dictWord{18, 11, 30}, + dictWord{148, 11, 52}, + dictWord{4, 0, 518}, + dictWord{135, 0, 1136}, + dictWord{6, 11, 568}, + dictWord{7, 11, 112}, + dictWord{7, 11, 1804}, + dictWord{8, 11, 362}, + dictWord{8, 11, 410}, + dictWord{8, 11, 830}, + dictWord{9, 11, 514}, + dictWord{11, 11, 649}, + dictWord{142, 11, 157}, + dictWord{135, 11, 673}, + dictWord{8, 0, 689}, + dictWord{137, 0, 863}, + dictWord{4, 0, 18}, + dictWord{7, 0, 145}, + dictWord{7, 0, 444}, + dictWord{7, 0, 1278}, + dictWord{8, 0, 49}, + dictWord{8, 0, 400}, + dictWord{9, 0, 71}, + dictWord{9, 0, 250}, + dictWord{10, 0, 459}, + dictWord{12, 0, 160}, + dictWord{16, 0, 24}, + dictWord{132, 11, 625}, + dictWord{140, 0, 1020}, + dictWord{4, 0, 997}, + dictWord{6, 0, 1946}, + dictWord{6, 0, 1984}, + dictWord{134, 0, 1998}, + dictWord{6, 11, 16}, + dictWord{6, 11, 158}, + dictWord{7, 11, 43}, + dictWord{ + 7, + 11, + 129, + }, + dictWord{7, 11, 181}, + dictWord{8, 11, 276}, + dictWord{8, 11, 377}, + dictWord{10, 11, 523}, + dictWord{11, 11, 816}, + dictWord{12, 11, 455}, + dictWord{ + 13, + 11, + 303, + }, + dictWord{142, 11, 135}, + dictWord{133, 10, 812}, + dictWord{134, 0, 658}, + dictWord{4, 11, 1}, + dictWord{7, 11, 1143}, + dictWord{7, 11, 1463}, + dictWord{8, 11, 61}, + dictWord{9, 11, 207}, + dictWord{9, 11, 390}, + dictWord{9, 11, 467}, + dictWord{139, 11, 836}, + dictWord{150, 11, 26}, + dictWord{140, 0, 106}, + dictWord{6, 0, 1827}, + dictWord{10, 0, 931}, + dictWord{18, 0, 166}, + dictWord{20, 0, 114}, + dictWord{4, 10, 137}, + dictWord{7, 10, 1178}, + dictWord{7, 11, 1319}, + dictWord{135, 10, 1520}, + dictWord{133, 0, 1010}, + dictWord{4, 11, 723}, + dictWord{5, 11, 895}, + dictWord{7, 11, 1031}, + dictWord{8, 11, 199}, + dictWord{8, 11, 340}, + dictWord{9, 11, 153}, + dictWord{9, 11, 215}, + dictWord{10, 11, 21}, + dictWord{10, 11, 59}, + dictWord{10, 11, 80}, + dictWord{10, 11, 224}, + dictWord{11, 11, 229}, + dictWord{11, 11, 652}, + dictWord{12, 11, 192}, + dictWord{13, 11, 146}, + dictWord{142, 11, 91}, + dictWord{132, 11, 295}, + dictWord{6, 11, 619}, + dictWord{ + 7, + 11, + 898, + }, + dictWord{7, 11, 1092}, + dictWord{8, 11, 485}, + dictWord{18, 11, 28}, + dictWord{147, 11, 116}, + dictWord{137, 11, 51}, + dictWord{6, 10, 1661}, + dictWord{ + 7, + 10, + 1975, + }, + dictWord{7, 10, 2009}, + dictWord{135, 10, 2011}, + dictWord{5, 11, 309}, + dictWord{140, 11, 211}, + dictWord{5, 0, 87}, + dictWord{7, 0, 313}, + dictWord{ + 7, + 0, + 1103, + }, + dictWord{10, 0, 208}, + dictWord{10, 0, 582}, + dictWord{11, 0, 389}, + dictWord{11, 0, 813}, + dictWord{12, 0, 385}, + dictWord{13, 0, 286}, + dictWord{ + 14, + 0, + 124, + }, + dictWord{146, 0, 108}, + dictWord{5, 11, 125}, + dictWord{8, 11, 77}, + dictWord{138, 11, 15}, + dictWord{132, 0, 267}, + dictWord{133, 0, 703}, + dictWord{ + 137, + 11, + 155, + }, + dictWord{133, 11, 439}, + dictWord{11, 11, 164}, + dictWord{140, 11, 76}, + dictWord{9, 0, 496}, + dictWord{5, 10, 89}, + dictWord{7, 10, 1915}, + dictWord{ + 9, + 10, + 185, + }, + dictWord{9, 10, 235}, + dictWord{10, 10, 64}, + dictWord{10, 10, 270}, + dictWord{10, 10, 403}, + dictWord{10, 10, 469}, + dictWord{10, 10, 529}, + dictWord{10, 10, 590}, + dictWord{11, 10, 140}, + dictWord{11, 10, 860}, + dictWord{13, 10, 1}, + dictWord{13, 10, 422}, + dictWord{14, 10, 341}, + dictWord{14, 10, 364}, + dictWord{17, 10, 93}, + dictWord{18, 10, 113}, + dictWord{19, 10, 97}, + dictWord{147, 10, 113}, + dictWord{133, 10, 695}, + dictWord{135, 0, 1121}, + dictWord{ + 5, + 10, + 6, + }, + dictWord{6, 10, 183}, + dictWord{7, 10, 680}, + dictWord{7, 10, 978}, + dictWord{7, 10, 1013}, + dictWord{7, 10, 1055}, + dictWord{12, 10, 230}, + dictWord{ + 13, + 10, + 172, + }, + dictWord{146, 10, 29}, + dictWord{4, 11, 8}, + dictWord{7, 11, 1152}, + dictWord{7, 11, 1153}, + dictWord{7, 11, 1715}, + dictWord{9, 11, 374}, + dictWord{ + 10, + 11, + 478, + }, + dictWord{139, 11, 648}, + dictWord{135, 11, 1099}, + dictWord{6, 10, 29}, + dictWord{139, 10, 63}, + dictWord{4, 0, 561}, + dictWord{10, 0, 249}, + dictWord{ + 139, + 0, + 209, + }, + dictWord{132, 0, 760}, + dictWord{7, 11, 799}, + dictWord{138, 11, 511}, + dictWord{136, 11, 87}, + dictWord{9, 0, 154}, + dictWord{140, 0, 485}, + dictWord{136, 0, 255}, + dictWord{132, 0, 323}, + dictWord{140, 0, 419}, + dictWord{132, 10, 311}, + dictWord{134, 10, 1740}, + dictWord{4, 0, 368}, + dictWord{ + 135, + 0, + 641, + }, + dictWord{7, 10, 170}, + dictWord{8, 10, 90}, + dictWord{8, 10, 177}, + dictWord{8, 10, 415}, + dictWord{11, 10, 714}, + dictWord{142, 10, 281}, + dictWord{ + 4, + 11, + 69, + }, + dictWord{5, 11, 122}, + dictWord{9, 11, 656}, + dictWord{138, 11, 464}, + dictWord{5, 11, 849}, + dictWord{134, 11, 1633}, + dictWord{8, 0, 522}, + dictWord{ + 142, + 0, + 328, + }, + dictWord{11, 10, 91}, + dictWord{13, 10, 129}, + dictWord{15, 10, 101}, + dictWord{145, 10, 125}, + dictWord{7, 0, 562}, + dictWord{8, 0, 551}, + dictWord{ + 4, + 10, + 494, + }, + dictWord{6, 10, 74}, + dictWord{7, 10, 44}, + dictWord{11, 11, 499}, + dictWord{12, 10, 17}, + dictWord{15, 10, 5}, + dictWord{148, 10, 11}, + dictWord{4, 10, 276}, + dictWord{133, 10, 296}, + dictWord{9, 0, 92}, + dictWord{147, 0, 91}, + dictWord{4, 10, 7}, + dictWord{5, 10, 90}, + dictWord{5, 10, 158}, + dictWord{6, 10, 542}, + dictWord{ + 7, + 10, + 221, + }, + dictWord{7, 10, 1574}, + dictWord{9, 10, 490}, + dictWord{10, 10, 540}, + dictWord{11, 10, 443}, + dictWord{139, 10, 757}, + dictWord{6, 0, 525}, + dictWord{ + 6, + 0, + 1976, + }, + dictWord{8, 0, 806}, + dictWord{9, 0, 876}, + dictWord{140, 0, 284}, + dictWord{5, 11, 859}, + dictWord{7, 10, 588}, + dictWord{7, 11, 1160}, + dictWord{ + 8, + 11, + 107, + }, + dictWord{9, 10, 175}, + dictWord{9, 11, 291}, + dictWord{9, 11, 439}, + dictWord{10, 10, 530}, + dictWord{10, 11, 663}, + dictWord{11, 11, 609}, + dictWord{ + 140, + 11, + 197, + }, + dictWord{7, 11, 168}, + dictWord{13, 11, 196}, + dictWord{141, 11, 237}, + dictWord{139, 0, 958}, + dictWord{133, 0, 594}, + dictWord{135, 10, 580}, + dictWord{7, 10, 88}, + dictWord{136, 10, 627}, + dictWord{6, 0, 479}, + dictWord{6, 0, 562}, + dictWord{7, 0, 1060}, + dictWord{13, 0, 6}, + dictWord{5, 10, 872}, + dictWord{ + 6, + 10, + 57, + }, + dictWord{7, 10, 471}, + dictWord{9, 10, 447}, + dictWord{137, 10, 454}, + dictWord{136, 11, 413}, + dictWord{145, 11, 19}, + dictWord{4, 11, 117}, + dictWord{ + 6, + 11, + 372, + }, + dictWord{7, 11, 1905}, + dictWord{142, 11, 323}, + dictWord{4, 11, 722}, + dictWord{139, 11, 471}, + dictWord{17, 0, 61}, + dictWord{5, 10, 31}, + dictWord{134, 10, 614}, + dictWord{8, 10, 330}, + dictWord{140, 10, 477}, + dictWord{7, 10, 1200}, + dictWord{138, 10, 460}, + dictWord{6, 10, 424}, + dictWord{ + 135, + 10, + 1866, + }, + dictWord{6, 0, 1641}, + dictWord{136, 0, 820}, + dictWord{6, 0, 1556}, + dictWord{134, 0, 1618}, + dictWord{9, 11, 5}, + dictWord{12, 11, 216}, + dictWord{ + 12, + 11, + 294, + }, + dictWord{12, 11, 298}, + dictWord{12, 11, 400}, + dictWord{12, 11, 518}, + dictWord{13, 11, 229}, + dictWord{143, 11, 139}, + dictWord{15, 11, 155}, + dictWord{144, 11, 79}, + dictWord{4, 0, 302}, + dictWord{135, 0, 1766}, + dictWord{5, 10, 13}, + dictWord{134, 10, 142}, + dictWord{6, 0, 148}, + dictWord{7, 0, 1313}, + dictWord{ + 7, + 10, + 116, + }, + dictWord{8, 10, 322}, + dictWord{8, 10, 755}, + dictWord{9, 10, 548}, + dictWord{10, 10, 714}, + dictWord{11, 10, 884}, + dictWord{141, 10, 324}, + dictWord{137, 0, 676}, + dictWord{9, 11, 88}, + dictWord{139, 11, 270}, + dictWord{5, 11, 12}, + dictWord{7, 11, 375}, + dictWord{137, 11, 438}, + dictWord{134, 0, 1674}, + dictWord{7, 10, 1472}, + dictWord{135, 10, 1554}, + dictWord{11, 0, 178}, + dictWord{7, 10, 1071}, + dictWord{7, 10, 1541}, + dictWord{7, 10, 1767}, + dictWord{ + 7, + 10, + 1806, + }, + dictWord{11, 10, 162}, + dictWord{11, 10, 242}, + dictWord{12, 10, 605}, + dictWord{15, 10, 26}, + dictWord{144, 10, 44}, + dictWord{6, 0, 389}, + dictWord{ + 7, + 0, + 149, + }, + dictWord{9, 0, 142}, + dictWord{138, 0, 94}, + dictWord{140, 11, 71}, + dictWord{145, 10, 115}, + dictWord{6, 0, 8}, + dictWord{7, 0, 1881}, + dictWord{8, 0, 91}, + dictWord{11, 11, 966}, + dictWord{12, 11, 287}, + dictWord{13, 11, 342}, + dictWord{13, 11, 402}, + dictWord{15, 11, 110}, + dictWord{143, 11, 163}, + dictWord{ + 4, + 11, + 258, + }, + dictWord{136, 11, 639}, + dictWord{6, 11, 22}, + dictWord{7, 11, 903}, + dictWord{138, 11, 577}, + dictWord{133, 11, 681}, + dictWord{135, 10, 1111}, + dictWord{135, 11, 1286}, + dictWord{9, 0, 112}, + dictWord{8, 10, 1}, + dictWord{138, 10, 326}, + dictWord{5, 10, 488}, + dictWord{6, 10, 527}, + dictWord{7, 10, 489}, + dictWord{ + 7, + 10, + 1636, + }, + dictWord{8, 10, 121}, + dictWord{8, 10, 144}, + dictWord{8, 10, 359}, + dictWord{9, 10, 193}, + dictWord{9, 10, 241}, + dictWord{9, 10, 336}, + dictWord{ + 9, + 10, + 882, + }, + dictWord{11, 10, 266}, + dictWord{11, 10, 372}, + dictWord{11, 10, 944}, + dictWord{12, 10, 401}, + dictWord{140, 10, 641}, + dictWord{4, 11, 664}, + dictWord{133, 11, 804}, + dictWord{6, 0, 747}, + dictWord{134, 0, 1015}, + dictWord{135, 0, 1746}, + dictWord{9, 10, 31}, + dictWord{10, 10, 244}, + dictWord{ + 10, + 10, + 699, + }, + dictWord{12, 10, 149}, + dictWord{141, 10, 497}, + dictWord{133, 10, 377}, + dictWord{135, 0, 24}, + dictWord{6, 0, 1352}, + dictWord{5, 11, 32}, + dictWord{ + 145, + 10, + 101, + }, + dictWord{7, 0, 1530}, + dictWord{10, 0, 158}, + dictWord{13, 0, 13}, + dictWord{13, 0, 137}, + dictWord{13, 0, 258}, + dictWord{14, 0, 111}, + dictWord{ + 14, + 0, + 225, + }, + dictWord{14, 0, 253}, + dictWord{14, 0, 304}, + dictWord{14, 0, 339}, + dictWord{14, 0, 417}, + dictWord{146, 0, 33}, + dictWord{4, 0, 503}, + dictWord{ + 135, + 0, + 1661, + }, + dictWord{5, 0, 130}, + dictWord{6, 0, 845}, + dictWord{7, 0, 1314}, + dictWord{9, 0, 610}, + dictWord{10, 0, 718}, + dictWord{11, 0, 601}, + dictWord{11, 0, 819}, + dictWord{11, 0, 946}, + dictWord{140, 0, 536}, + dictWord{10, 0, 149}, + dictWord{11, 0, 280}, + dictWord{142, 0, 336}, + dictWord{134, 0, 1401}, + dictWord{ + 135, + 0, + 1946, + }, + dictWord{8, 0, 663}, + dictWord{144, 0, 8}, + dictWord{134, 0, 1607}, + dictWord{135, 10, 2023}, + dictWord{4, 11, 289}, + dictWord{7, 11, 629}, + dictWord{ + 7, + 11, + 1698, + }, + dictWord{7, 11, 1711}, + dictWord{140, 11, 215}, + dictWord{6, 11, 450}, + dictWord{136, 11, 109}, + dictWord{10, 0, 882}, + dictWord{10, 0, 883}, + dictWord{10, 0, 914}, + dictWord{138, 0, 928}, + dictWord{133, 10, 843}, + dictWord{136, 11, 705}, + dictWord{132, 10, 554}, + dictWord{133, 10, 536}, + dictWord{ + 5, + 0, + 417, + }, + dictWord{9, 10, 79}, + dictWord{11, 10, 625}, + dictWord{145, 10, 7}, + dictWord{7, 11, 1238}, + dictWord{142, 11, 37}, + dictWord{4, 0, 392}, + dictWord{ + 135, + 0, + 1597, + }, + dictWord{5, 0, 433}, + dictWord{9, 0, 633}, + dictWord{11, 0, 629}, + dictWord{132, 10, 424}, + dictWord{7, 10, 336}, + dictWord{136, 10, 785}, + dictWord{ + 134, + 11, + 355, + }, + dictWord{6, 0, 234}, + dictWord{7, 0, 769}, + dictWord{9, 0, 18}, + dictWord{138, 0, 358}, + dictWord{4, 10, 896}, + dictWord{134, 10, 1777}, + dictWord{ + 138, + 11, + 323, + }, + dictWord{7, 0, 140}, + dictWord{7, 0, 1950}, + dictWord{8, 0, 680}, + dictWord{11, 0, 817}, + dictWord{147, 0, 88}, + dictWord{7, 0, 1222}, + dictWord{ + 138, + 0, + 386, + }, + dictWord{139, 11, 908}, + dictWord{11, 0, 249}, + dictWord{12, 0, 313}, + dictWord{16, 0, 66}, + dictWord{145, 0, 26}, + dictWord{134, 0, 5}, + dictWord{7, 10, 750}, + dictWord{9, 10, 223}, + dictWord{11, 10, 27}, + dictWord{11, 10, 466}, + dictWord{12, 10, 624}, + dictWord{14, 10, 265}, + dictWord{146, 10, 61}, + dictWord{ + 134, + 11, + 26, + }, + dictWord{134, 0, 1216}, + dictWord{5, 0, 963}, + dictWord{134, 0, 1773}, + dictWord{4, 11, 414}, + dictWord{5, 11, 467}, + dictWord{9, 11, 654}, + dictWord{ + 10, + 11, + 451, + }, + dictWord{12, 11, 59}, + dictWord{141, 11, 375}, + dictWord{135, 11, 17}, + dictWord{4, 10, 603}, + dictWord{133, 10, 661}, + dictWord{4, 10, 11}, + dictWord{ + 6, + 10, + 128, + }, + dictWord{7, 10, 231}, + dictWord{7, 10, 1533}, + dictWord{138, 10, 725}, + dictWord{135, 11, 955}, + dictWord{7, 0, 180}, + dictWord{8, 0, 509}, + dictWord{ + 136, + 0, + 792, + }, + dictWord{132, 10, 476}, + dictWord{132, 0, 1002}, + dictWord{133, 11, 538}, + dictWord{135, 10, 1807}, + dictWord{132, 0, 931}, + dictWord{7, 0, 943}, + dictWord{11, 0, 614}, + dictWord{140, 0, 747}, + dictWord{135, 0, 1837}, + dictWord{9, 10, 20}, + dictWord{10, 10, 324}, + dictWord{10, 10, 807}, + dictWord{ + 139, + 10, + 488, + }, + dictWord{134, 0, 641}, + dictWord{6, 11, 280}, + dictWord{10, 11, 502}, + dictWord{11, 11, 344}, + dictWord{140, 11, 38}, + dictWord{5, 11, 45}, + dictWord{ + 7, + 11, + 1161, + }, + dictWord{11, 11, 448}, + dictWord{11, 11, 880}, + dictWord{13, 11, 139}, + dictWord{13, 11, 407}, + dictWord{15, 11, 16}, + dictWord{17, 11, 95}, + dictWord{ + 18, + 11, + 66, + }, + dictWord{18, 11, 88}, + dictWord{18, 11, 123}, + dictWord{149, 11, 7}, + dictWord{9, 0, 280}, + dictWord{138, 0, 134}, + dictWord{22, 0, 22}, + dictWord{23, 0, 5}, + dictWord{151, 0, 29}, + dictWord{136, 11, 777}, + dictWord{4, 0, 90}, + dictWord{5, 0, 545}, + dictWord{7, 0, 754}, + dictWord{9, 0, 186}, + dictWord{10, 0, 72}, + dictWord{ + 10, + 0, + 782, + }, + dictWord{11, 0, 577}, + dictWord{11, 0, 610}, + dictWord{11, 0, 960}, + dictWord{12, 0, 354}, + dictWord{12, 0, 362}, + dictWord{12, 0, 595}, + dictWord{ + 4, + 11, + 410, + }, + dictWord{135, 11, 521}, + dictWord{135, 11, 1778}, + dictWord{5, 10, 112}, + dictWord{6, 10, 103}, + dictWord{134, 10, 150}, + dictWord{138, 10, 356}, + dictWord{132, 0, 742}, + dictWord{7, 0, 151}, + dictWord{9, 0, 329}, + dictWord{139, 0, 254}, + dictWord{8, 0, 853}, + dictWord{8, 0, 881}, + dictWord{8, 0, 911}, + dictWord{ + 8, + 0, + 912, + }, + dictWord{10, 0, 872}, + dictWord{12, 0, 741}, + dictWord{12, 0, 742}, + dictWord{152, 0, 18}, + dictWord{4, 11, 573}, + dictWord{136, 11, 655}, + dictWord{ + 6, + 0, + 921, + }, + dictWord{134, 0, 934}, + dictWord{9, 0, 187}, + dictWord{10, 0, 36}, + dictWord{11, 0, 1016}, + dictWord{17, 0, 44}, + dictWord{146, 0, 64}, + dictWord{7, 0, 833}, + dictWord{136, 0, 517}, + dictWord{4, 0, 506}, + dictWord{5, 0, 295}, + dictWord{135, 0, 1680}, + dictWord{4, 10, 708}, + dictWord{8, 10, 15}, + dictWord{9, 10, 50}, + dictWord{ + 9, + 10, + 386, + }, + dictWord{11, 10, 18}, + dictWord{11, 10, 529}, + dictWord{140, 10, 228}, + dictWord{7, 0, 251}, + dictWord{7, 0, 1701}, + dictWord{8, 0, 436}, + dictWord{ + 4, + 10, + 563, + }, + dictWord{7, 10, 592}, + dictWord{7, 10, 637}, + dictWord{7, 10, 770}, + dictWord{8, 10, 463}, + dictWord{9, 10, 60}, + dictWord{9, 10, 335}, + dictWord{9, 10, 904}, + dictWord{10, 10, 73}, + dictWord{11, 10, 434}, + dictWord{12, 10, 585}, + dictWord{13, 10, 331}, + dictWord{18, 10, 110}, + dictWord{148, 10, 60}, + dictWord{ + 132, + 10, + 502, + }, + dictWord{136, 0, 584}, + dictWord{6, 10, 347}, + dictWord{138, 10, 161}, + dictWord{7, 0, 987}, + dictWord{9, 0, 688}, + dictWord{10, 0, 522}, + dictWord{ + 11, + 0, + 788, + }, + dictWord{12, 0, 137}, + dictWord{12, 0, 566}, + dictWord{14, 0, 9}, + dictWord{14, 0, 24}, + dictWord{14, 0, 64}, + dictWord{7, 11, 899}, + dictWord{142, 11, 325}, + dictWord{4, 0, 214}, + dictWord{5, 0, 500}, + dictWord{5, 10, 102}, + dictWord{6, 10, 284}, + dictWord{7, 10, 1079}, + dictWord{7, 10, 1423}, + dictWord{7, 10, 1702}, + dictWord{ + 8, + 10, + 470, + }, + dictWord{9, 10, 554}, + dictWord{9, 10, 723}, + dictWord{139, 10, 333}, + dictWord{7, 10, 246}, + dictWord{135, 10, 840}, + dictWord{6, 10, 10}, + dictWord{ + 8, + 10, + 571, + }, + dictWord{9, 10, 739}, + dictWord{143, 10, 91}, + dictWord{133, 10, 626}, + dictWord{146, 0, 195}, + dictWord{134, 0, 1775}, + dictWord{7, 0, 389}, + dictWord{7, 0, 700}, + dictWord{7, 0, 940}, + dictWord{8, 0, 514}, + dictWord{9, 0, 116}, + dictWord{9, 0, 535}, + dictWord{10, 0, 118}, + dictWord{11, 0, 107}, + dictWord{ + 11, + 0, + 148, + }, + dictWord{11, 0, 922}, + dictWord{12, 0, 254}, + dictWord{12, 0, 421}, + dictWord{142, 0, 238}, + dictWord{5, 10, 18}, + dictWord{6, 10, 526}, + dictWord{13, 10, 24}, + dictWord{13, 10, 110}, + dictWord{19, 10, 5}, + dictWord{147, 10, 44}, + dictWord{132, 0, 743}, + dictWord{11, 0, 292}, + dictWord{4, 10, 309}, + dictWord{5, 10, 462}, + dictWord{7, 10, 970}, + dictWord{135, 10, 1097}, + dictWord{22, 10, 30}, + dictWord{150, 10, 33}, + dictWord{139, 11, 338}, + dictWord{135, 11, 1598}, + dictWord{ + 7, + 0, + 1283, + }, + dictWord{9, 0, 227}, + dictWord{11, 0, 325}, + dictWord{11, 0, 408}, + dictWord{14, 0, 180}, + dictWord{146, 0, 47}, + dictWord{4, 0, 953}, + dictWord{6, 0, 1805}, + dictWord{6, 0, 1814}, + dictWord{6, 0, 1862}, + dictWord{140, 0, 774}, + dictWord{6, 11, 611}, + dictWord{135, 11, 1733}, + dictWord{135, 11, 1464}, + dictWord{ + 5, + 0, + 81, + }, + dictWord{7, 0, 146}, + dictWord{7, 0, 1342}, + dictWord{8, 0, 53}, + dictWord{8, 0, 561}, + dictWord{8, 0, 694}, + dictWord{8, 0, 754}, + dictWord{9, 0, 115}, + dictWord{ + 9, + 0, + 179, + }, + dictWord{9, 0, 894}, + dictWord{10, 0, 462}, + dictWord{10, 0, 813}, + dictWord{11, 0, 230}, + dictWord{11, 0, 657}, + dictWord{11, 0, 699}, + dictWord{11, 0, 748}, + dictWord{12, 0, 119}, + dictWord{12, 0, 200}, + dictWord{12, 0, 283}, + dictWord{142, 0, 273}, + dictWord{5, 0, 408}, + dictWord{6, 0, 789}, + dictWord{6, 0, 877}, + dictWord{ + 6, + 0, + 1253, + }, + dictWord{6, 0, 1413}, + dictWord{137, 0, 747}, + dictWord{134, 10, 1704}, + dictWord{135, 11, 663}, + dictWord{6, 0, 1910}, + dictWord{6, 0, 1915}, + dictWord{6, 0, 1923}, + dictWord{9, 0, 913}, + dictWord{9, 0, 928}, + dictWord{9, 0, 950}, + dictWord{9, 0, 954}, + dictWord{9, 0, 978}, + dictWord{9, 0, 993}, + dictWord{12, 0, 812}, + dictWord{12, 0, 819}, + dictWord{12, 0, 831}, + dictWord{12, 0, 833}, + dictWord{12, 0, 838}, + dictWord{12, 0, 909}, + dictWord{12, 0, 928}, + dictWord{12, 0, 931}, + dictWord{12, 0, 950}, + dictWord{15, 0, 186}, + dictWord{15, 0, 187}, + dictWord{15, 0, 195}, + dictWord{15, 0, 196}, + dictWord{15, 0, 209}, + dictWord{15, 0, 215}, + dictWord{ + 15, + 0, + 236, + }, + dictWord{15, 0, 241}, + dictWord{15, 0, 249}, + dictWord{15, 0, 253}, + dictWord{18, 0, 180}, + dictWord{18, 0, 221}, + dictWord{18, 0, 224}, + dictWord{ + 18, + 0, + 227, + }, + dictWord{18, 0, 229}, + dictWord{149, 0, 60}, + dictWord{7, 0, 1826}, + dictWord{135, 0, 1938}, + dictWord{11, 0, 490}, + dictWord{18, 0, 143}, + dictWord{ + 5, + 10, + 86, + }, + dictWord{7, 10, 743}, + dictWord{9, 10, 85}, + dictWord{10, 10, 281}, + dictWord{10, 10, 432}, + dictWord{12, 10, 251}, + dictWord{13, 10, 118}, + dictWord{ + 142, + 10, + 378, + }, + dictWord{5, 10, 524}, + dictWord{133, 10, 744}, + dictWord{141, 11, 442}, + dictWord{10, 10, 107}, + dictWord{140, 10, 436}, + dictWord{135, 11, 503}, + dictWord{134, 0, 1162}, + dictWord{132, 10, 927}, + dictWord{7, 0, 30}, + dictWord{8, 0, 86}, + dictWord{8, 0, 315}, + dictWord{8, 0, 700}, + dictWord{9, 0, 576}, + dictWord{ + 9, + 0, + 858, + }, + dictWord{10, 0, 414}, + dictWord{11, 0, 310}, + dictWord{11, 0, 888}, + dictWord{11, 0, 904}, + dictWord{12, 0, 361}, + dictWord{13, 0, 248}, + dictWord{13, 0, 371}, + dictWord{14, 0, 142}, + dictWord{12, 10, 670}, + dictWord{146, 10, 94}, + dictWord{134, 0, 721}, + dictWord{4, 11, 113}, + dictWord{5, 11, 163}, + dictWord{5, 11, 735}, + dictWord{7, 11, 1009}, + dictWord{7, 10, 1149}, + dictWord{9, 11, 9}, + dictWord{9, 10, 156}, + dictWord{9, 11, 771}, + dictWord{12, 11, 90}, + dictWord{13, 11, 138}, + dictWord{13, 11, 410}, + dictWord{143, 11, 128}, + dictWord{138, 0, 839}, + dictWord{133, 10, 778}, + dictWord{137, 0, 617}, + dictWord{133, 10, 502}, + dictWord{ + 8, + 10, + 196, + }, + dictWord{10, 10, 283}, + dictWord{139, 10, 406}, + dictWord{6, 0, 428}, + dictWord{7, 0, 524}, + dictWord{8, 0, 169}, + dictWord{8, 0, 234}, + dictWord{9, 0, 480}, + dictWord{138, 0, 646}, + dictWord{133, 10, 855}, + dictWord{134, 0, 1648}, + dictWord{7, 0, 1205}, + dictWord{138, 0, 637}, + dictWord{7, 0, 1596}, + dictWord{ + 4, + 11, + 935, + }, + dictWord{133, 11, 823}, + dictWord{5, 11, 269}, + dictWord{7, 11, 434}, + dictWord{7, 11, 891}, + dictWord{8, 11, 339}, + dictWord{9, 11, 702}, + dictWord{ + 11, + 11, + 594, + }, + dictWord{11, 11, 718}, + dictWord{145, 11, 100}, + dictWord{7, 11, 878}, + dictWord{9, 11, 485}, + dictWord{141, 11, 264}, + dictWord{4, 0, 266}, + dictWord{ + 8, + 0, + 4, + }, + dictWord{9, 0, 39}, + dictWord{10, 0, 166}, + dictWord{11, 0, 918}, + dictWord{12, 0, 635}, + dictWord{20, 0, 10}, + dictWord{22, 0, 27}, + dictWord{22, 0, 43}, + dictWord{ + 22, + 0, + 52, + }, + dictWord{134, 11, 1713}, + dictWord{7, 10, 1400}, + dictWord{9, 10, 446}, + dictWord{138, 10, 45}, + dictWord{135, 11, 900}, + dictWord{132, 0, 862}, + dictWord{134, 0, 1554}, + dictWord{135, 11, 1033}, + dictWord{19, 0, 16}, + dictWord{147, 11, 16}, + dictWord{135, 11, 1208}, + dictWord{7, 0, 157}, + dictWord{ + 136, + 0, + 279, + }, + dictWord{6, 0, 604}, + dictWord{136, 0, 391}, + dictWord{13, 10, 455}, + dictWord{15, 10, 99}, + dictWord{15, 10, 129}, + dictWord{144, 10, 68}, + dictWord{ + 135, + 10, + 172, + }, + dictWord{7, 0, 945}, + dictWord{11, 0, 713}, + dictWord{139, 0, 744}, + dictWord{4, 0, 973}, + dictWord{10, 0, 877}, + dictWord{10, 0, 937}, + dictWord{ + 10, + 0, + 938, + }, + dictWord{140, 0, 711}, + dictWord{139, 0, 1022}, + dictWord{132, 10, 568}, + dictWord{142, 11, 143}, + dictWord{4, 0, 567}, + dictWord{9, 0, 859}, + dictWord{ + 132, + 10, + 732, + }, + dictWord{7, 0, 1846}, + dictWord{136, 0, 628}, + dictWord{136, 10, 733}, + dictWord{133, 0, 762}, + dictWord{4, 10, 428}, + dictWord{135, 10, 1789}, + dictWord{10, 0, 784}, + dictWord{13, 0, 191}, + dictWord{7, 10, 2015}, + dictWord{140, 10, 665}, + dictWord{133, 0, 298}, + dictWord{7, 0, 633}, + dictWord{7, 0, 905}, + dictWord{7, 0, 909}, + dictWord{7, 0, 1538}, + dictWord{9, 0, 767}, + dictWord{140, 0, 636}, + dictWord{138, 10, 806}, + dictWord{132, 0, 795}, + dictWord{139, 0, 301}, + dictWord{135, 0, 1970}, + dictWord{5, 11, 625}, + dictWord{135, 11, 1617}, + dictWord{135, 11, 275}, + dictWord{7, 11, 37}, + dictWord{8, 11, 425}, + dictWord{ + 8, + 11, + 693, + }, + dictWord{9, 11, 720}, + dictWord{10, 11, 380}, + dictWord{10, 11, 638}, + dictWord{11, 11, 273}, + dictWord{11, 11, 307}, + dictWord{11, 11, 473}, + dictWord{ + 12, + 11, + 61, + }, + dictWord{143, 11, 43}, + dictWord{135, 11, 198}, + dictWord{134, 0, 1236}, + dictWord{7, 0, 369}, + dictWord{12, 0, 644}, + dictWord{12, 0, 645}, + dictWord{144, 0, 90}, + dictWord{19, 0, 15}, + dictWord{149, 0, 27}, + dictWord{6, 0, 71}, + dictWord{7, 0, 845}, + dictWord{8, 0, 160}, + dictWord{9, 0, 318}, + dictWord{6, 10, 1623}, + dictWord{134, 10, 1681}, + dictWord{134, 0, 1447}, + dictWord{134, 0, 1255}, + dictWord{138, 0, 735}, + dictWord{8, 0, 76}, + dictWord{132, 11, 168}, + dictWord{ + 6, + 10, + 1748, + }, + dictWord{8, 10, 715}, + dictWord{9, 10, 802}, + dictWord{10, 10, 46}, + dictWord{10, 10, 819}, + dictWord{13, 10, 308}, + dictWord{14, 10, 351}, + dictWord{14, 10, 363}, + dictWord{146, 10, 67}, + dictWord{135, 11, 91}, + dictWord{6, 0, 474}, + dictWord{4, 10, 63}, + dictWord{133, 10, 347}, + dictWord{133, 10, 749}, + dictWord{138, 0, 841}, + dictWord{133, 10, 366}, + dictWord{6, 0, 836}, + dictWord{132, 11, 225}, + dictWord{135, 0, 1622}, + dictWord{135, 10, 89}, + dictWord{ + 140, + 0, + 735, + }, + dictWord{134, 0, 1601}, + dictWord{138, 11, 145}, + dictWord{6, 0, 1390}, + dictWord{137, 0, 804}, + dictWord{142, 0, 394}, + dictWord{6, 11, 15}, + dictWord{ + 7, + 11, + 70, + }, + dictWord{10, 11, 240}, + dictWord{147, 11, 93}, + dictWord{6, 0, 96}, + dictWord{135, 0, 1426}, + dictWord{4, 0, 651}, + dictWord{133, 0, 289}, + dictWord{ + 7, + 11, + 956, + }, + dictWord{7, 10, 977}, + dictWord{7, 11, 1157}, + dictWord{7, 11, 1506}, + dictWord{7, 11, 1606}, + dictWord{7, 11, 1615}, + dictWord{7, 11, 1619}, + dictWord{ + 7, + 11, + 1736, + }, + dictWord{7, 11, 1775}, + dictWord{8, 11, 590}, + dictWord{9, 11, 324}, + dictWord{9, 11, 736}, + dictWord{9, 11, 774}, + dictWord{9, 11, 776}, + dictWord{ + 9, + 11, + 784, + }, + dictWord{10, 11, 567}, + dictWord{10, 11, 708}, + dictWord{11, 11, 518}, + dictWord{11, 11, 613}, + dictWord{11, 11, 695}, + dictWord{11, 11, 716}, + dictWord{11, 11, 739}, + dictWord{11, 11, 770}, + dictWord{11, 11, 771}, + dictWord{11, 11, 848}, + dictWord{11, 11, 857}, + dictWord{11, 11, 931}, + dictWord{ + 11, + 11, + 947, + }, + dictWord{12, 11, 326}, + dictWord{12, 11, 387}, + dictWord{12, 11, 484}, + dictWord{12, 11, 528}, + dictWord{12, 11, 552}, + dictWord{12, 11, 613}, + dictWord{ + 13, + 11, + 189, + }, + dictWord{13, 11, 256}, + dictWord{13, 11, 340}, + dictWord{13, 11, 432}, + dictWord{13, 11, 436}, + dictWord{13, 11, 440}, + dictWord{13, 11, 454}, + dictWord{14, 11, 174}, + dictWord{14, 11, 220}, + dictWord{14, 11, 284}, + dictWord{14, 11, 390}, + dictWord{145, 11, 121}, + dictWord{7, 0, 688}, + dictWord{8, 0, 35}, + dictWord{9, 0, 511}, + dictWord{10, 0, 767}, + dictWord{147, 0, 118}, + dictWord{134, 0, 667}, + dictWord{4, 0, 513}, + dictWord{5, 10, 824}, + dictWord{133, 10, 941}, + dictWord{7, 10, 440}, + dictWord{8, 10, 230}, + dictWord{139, 10, 106}, + dictWord{134, 0, 2034}, + dictWord{135, 11, 1399}, + dictWord{143, 11, 66}, + dictWord{ + 135, + 11, + 1529, + }, + dictWord{4, 11, 145}, + dictWord{6, 11, 176}, + dictWord{7, 11, 395}, + dictWord{9, 11, 562}, + dictWord{144, 11, 28}, + dictWord{132, 11, 501}, + dictWord{132, 0, 704}, + dictWord{134, 0, 1524}, + dictWord{7, 0, 1078}, + dictWord{134, 11, 464}, + dictWord{6, 11, 509}, + dictWord{10, 11, 82}, + dictWord{20, 11, 91}, + dictWord{151, 11, 13}, + dictWord{4, 0, 720}, + dictWord{133, 0, 306}, + dictWord{133, 0, 431}, + dictWord{7, 0, 1196}, + dictWord{4, 10, 914}, + dictWord{5, 10, 800}, + dictWord{133, 10, 852}, + dictWord{135, 11, 1189}, + dictWord{10, 0, 54}, + dictWord{141, 10, 115}, + dictWord{7, 10, 564}, + dictWord{142, 10, 168}, + dictWord{ + 5, + 0, + 464, + }, + dictWord{6, 0, 236}, + dictWord{7, 0, 696}, + dictWord{7, 0, 914}, + dictWord{7, 0, 1108}, + dictWord{7, 0, 1448}, + dictWord{9, 0, 15}, + dictWord{9, 0, 564}, + dictWord{ + 10, + 0, + 14, + }, + dictWord{12, 0, 565}, + dictWord{13, 0, 449}, + dictWord{14, 0, 53}, + dictWord{15, 0, 13}, + dictWord{16, 0, 64}, + dictWord{17, 0, 41}, + dictWord{4, 10, 918}, + dictWord{133, 10, 876}, + dictWord{6, 0, 1418}, + dictWord{134, 10, 1764}, + dictWord{4, 10, 92}, + dictWord{133, 10, 274}, + dictWord{134, 0, 907}, + dictWord{ + 4, + 11, + 114, + }, + dictWord{8, 10, 501}, + dictWord{9, 11, 492}, + dictWord{13, 11, 462}, + dictWord{142, 11, 215}, + dictWord{4, 11, 77}, + dictWord{5, 11, 361}, + dictWord{ + 6, + 11, + 139, + }, + dictWord{6, 11, 401}, + dictWord{6, 11, 404}, + dictWord{7, 11, 413}, + dictWord{7, 11, 715}, + dictWord{7, 11, 1716}, + dictWord{11, 11, 279}, + dictWord{ + 12, + 11, + 179, + }, + dictWord{12, 11, 258}, + dictWord{13, 11, 244}, + dictWord{142, 11, 358}, + dictWord{6, 0, 1767}, + dictWord{12, 0, 194}, + dictWord{145, 0, 107}, + dictWord{ + 134, + 11, + 1717, + }, + dictWord{5, 10, 743}, + dictWord{142, 11, 329}, + dictWord{4, 10, 49}, + dictWord{7, 10, 280}, + dictWord{135, 10, 1633}, + dictWord{5, 0, 840}, + dictWord{7, 11, 1061}, + dictWord{8, 11, 82}, + dictWord{11, 11, 250}, + dictWord{12, 11, 420}, + dictWord{141, 11, 184}, + dictWord{135, 11, 724}, + dictWord{ + 134, + 0, + 900, + }, + dictWord{136, 10, 47}, + dictWord{134, 0, 1436}, + dictWord{144, 11, 0}, + dictWord{6, 0, 675}, + dictWord{7, 0, 1008}, + dictWord{7, 0, 1560}, + dictWord{ + 9, + 0, + 642, + }, + dictWord{11, 0, 236}, + dictWord{14, 0, 193}, + dictWord{5, 10, 272}, + dictWord{5, 10, 908}, + dictWord{5, 10, 942}, + dictWord{8, 10, 197}, + dictWord{9, 10, 47}, + dictWord{11, 10, 538}, + dictWord{139, 10, 742}, + dictWord{4, 0, 68}, + dictWord{5, 0, 628}, + dictWord{5, 0, 634}, + dictWord{6, 0, 386}, + dictWord{7, 0, 794}, + dictWord{ + 8, + 0, + 273, + }, + dictWord{9, 0, 563}, + dictWord{10, 0, 105}, + dictWord{10, 0, 171}, + dictWord{11, 0, 94}, + dictWord{139, 0, 354}, + dictWord{135, 10, 1911}, + dictWord{ + 137, + 10, + 891, + }, + dictWord{4, 0, 95}, + dictWord{6, 0, 1297}, + dictWord{6, 0, 1604}, + dictWord{7, 0, 416}, + dictWord{139, 0, 830}, + dictWord{6, 11, 513}, + dictWord{ + 135, + 11, + 1052, + }, + dictWord{7, 0, 731}, + dictWord{13, 0, 20}, + dictWord{143, 0, 11}, + dictWord{137, 11, 899}, + dictWord{10, 0, 850}, + dictWord{140, 0, 697}, + dictWord{ + 4, + 0, + 662, + }, + dictWord{7, 11, 1417}, + dictWord{12, 11, 382}, + dictWord{17, 11, 48}, + dictWord{152, 11, 12}, + dictWord{133, 0, 736}, + dictWord{132, 0, 861}, + dictWord{ + 4, + 10, + 407, + }, + dictWord{132, 10, 560}, + dictWord{141, 10, 490}, + dictWord{6, 11, 545}, + dictWord{7, 11, 565}, + dictWord{7, 11, 1669}, + dictWord{10, 11, 114}, + dictWord{11, 11, 642}, + dictWord{140, 11, 618}, + dictWord{6, 0, 871}, + dictWord{134, 0, 1000}, + dictWord{5, 0, 864}, + dictWord{10, 0, 648}, + dictWord{11, 0, 671}, + dictWord{15, 0, 46}, + dictWord{133, 11, 5}, + dictWord{133, 0, 928}, + dictWord{11, 0, 90}, + dictWord{13, 0, 7}, + dictWord{4, 10, 475}, + dictWord{11, 10, 35}, + dictWord{ + 13, + 10, + 71, + }, + dictWord{13, 10, 177}, + dictWord{142, 10, 422}, + dictWord{136, 0, 332}, + dictWord{135, 11, 192}, + dictWord{134, 0, 1055}, + dictWord{136, 11, 763}, + dictWord{11, 0, 986}, + dictWord{140, 0, 682}, + dictWord{7, 0, 76}, + dictWord{8, 0, 44}, + dictWord{9, 0, 884}, + dictWord{10, 0, 580}, + dictWord{11, 0, 399}, + dictWord{ + 11, + 0, + 894, + }, + dictWord{143, 0, 122}, + dictWord{135, 11, 1237}, + dictWord{135, 10, 636}, + dictWord{11, 0, 300}, + dictWord{6, 10, 222}, + dictWord{7, 10, 1620}, + dictWord{ + 8, + 10, + 409, + }, + dictWord{137, 10, 693}, + dictWord{4, 11, 87}, + dictWord{5, 11, 250}, + dictWord{10, 11, 601}, + dictWord{13, 11, 298}, + dictWord{13, 11, 353}, + dictWord{141, 11, 376}, + dictWord{5, 0, 518}, + dictWord{10, 0, 340}, + dictWord{11, 0, 175}, + dictWord{149, 0, 16}, + dictWord{140, 0, 771}, + dictWord{6, 0, 1108}, + dictWord{137, 0, 831}, + dictWord{132, 0, 836}, + dictWord{135, 0, 1852}, + dictWord{4, 0, 957}, + dictWord{6, 0, 1804}, + dictWord{8, 0, 842}, + dictWord{8, 0, 843}, + dictWord{ + 8, + 0, + 851, + }, + dictWord{8, 0, 855}, + dictWord{140, 0, 767}, + dictWord{135, 11, 814}, + dictWord{4, 11, 57}, + dictWord{7, 11, 1195}, + dictWord{7, 11, 1438}, + dictWord{ + 7, + 11, + 1548, + }, + dictWord{7, 11, 1835}, + dictWord{7, 11, 1904}, + dictWord{9, 11, 757}, + dictWord{10, 11, 604}, + dictWord{139, 11, 519}, + dictWord{133, 10, 882}, + dictWord{138, 0, 246}, + dictWord{4, 0, 934}, + dictWord{5, 0, 202}, + dictWord{8, 0, 610}, + dictWord{7, 11, 1897}, + dictWord{12, 11, 290}, + dictWord{13, 11, 80}, + dictWord{13, 11, 437}, + dictWord{145, 11, 74}, + dictWord{8, 0, 96}, + dictWord{9, 0, 36}, + dictWord{10, 0, 607}, + dictWord{10, 0, 804}, + dictWord{10, 0, 832}, + dictWord{ + 11, + 0, + 423, + }, + dictWord{11, 0, 442}, + dictWord{12, 0, 309}, + dictWord{14, 0, 199}, + dictWord{15, 0, 90}, + dictWord{145, 0, 110}, + dictWord{132, 10, 426}, + dictWord{ + 7, + 0, + 654, + }, + dictWord{8, 0, 240}, + dictWord{6, 10, 58}, + dictWord{7, 10, 745}, + dictWord{7, 10, 1969}, + dictWord{8, 10, 675}, + dictWord{9, 10, 479}, + dictWord{9, 10, 731}, + dictWord{10, 10, 330}, + dictWord{10, 10, 593}, + dictWord{10, 10, 817}, + dictWord{11, 10, 32}, + dictWord{11, 10, 133}, + dictWord{11, 10, 221}, + dictWord{ + 145, + 10, + 68, + }, + dictWord{9, 0, 13}, + dictWord{9, 0, 398}, + dictWord{9, 0, 727}, + dictWord{10, 0, 75}, + dictWord{10, 0, 184}, + dictWord{10, 0, 230}, + dictWord{10, 0, 564}, + dictWord{ + 10, + 0, + 569, + }, + dictWord{11, 0, 973}, + dictWord{12, 0, 70}, + dictWord{12, 0, 189}, + dictWord{13, 0, 57}, + dictWord{141, 0, 257}, + dictWord{4, 11, 209}, + dictWord{ + 135, + 11, + 902, + }, + dictWord{7, 0, 391}, + dictWord{137, 10, 538}, + dictWord{134, 0, 403}, + dictWord{6, 11, 303}, + dictWord{7, 11, 335}, + dictWord{7, 11, 1437}, + dictWord{ + 7, + 11, + 1668, + }, + dictWord{8, 11, 553}, + dictWord{8, 11, 652}, + dictWord{8, 11, 656}, + dictWord{9, 11, 558}, + dictWord{11, 11, 743}, + dictWord{149, 11, 18}, + dictWord{ + 132, + 11, + 559, + }, + dictWord{11, 0, 75}, + dictWord{142, 0, 267}, + dictWord{6, 0, 815}, + dictWord{141, 11, 2}, + dictWord{141, 0, 366}, + dictWord{137, 0, 631}, + dictWord{ + 133, + 11, + 1017, + }, + dictWord{5, 0, 345}, + dictWord{135, 0, 1016}, + dictWord{133, 11, 709}, + dictWord{134, 11, 1745}, + dictWord{133, 10, 566}, + dictWord{7, 0, 952}, + dictWord{6, 10, 48}, + dictWord{9, 10, 139}, + dictWord{10, 10, 399}, + dictWord{11, 10, 469}, + dictWord{12, 10, 634}, + dictWord{141, 10, 223}, + dictWord{ + 133, + 0, + 673, + }, + dictWord{9, 0, 850}, + dictWord{7, 11, 8}, + dictWord{136, 11, 206}, + dictWord{6, 0, 662}, + dictWord{149, 0, 35}, + dictWord{4, 0, 287}, + dictWord{133, 0, 1018}, + dictWord{6, 10, 114}, + dictWord{7, 10, 1224}, + dictWord{7, 10, 1556}, + dictWord{136, 10, 3}, + dictWord{8, 10, 576}, + dictWord{137, 10, 267}, + dictWord{4, 0, 884}, + dictWord{5, 0, 34}, + dictWord{10, 0, 724}, + dictWord{12, 0, 444}, + dictWord{13, 0, 354}, + dictWord{18, 0, 32}, + dictWord{23, 0, 24}, + dictWord{23, 0, 31}, + dictWord{ + 152, + 0, + 5, + }, + dictWord{133, 10, 933}, + dictWord{132, 11, 776}, + dictWord{138, 0, 151}, + dictWord{136, 0, 427}, + dictWord{134, 0, 382}, + dictWord{132, 0, 329}, + dictWord{ + 9, + 0, + 846, + }, + dictWord{10, 0, 827}, + dictWord{138, 11, 33}, + dictWord{9, 0, 279}, + dictWord{10, 0, 407}, + dictWord{14, 0, 84}, + dictWord{22, 0, 18}, + dictWord{ + 135, + 11, + 1297, + }, + dictWord{136, 11, 406}, + dictWord{132, 0, 906}, + dictWord{136, 0, 366}, + dictWord{134, 0, 843}, + dictWord{134, 0, 1443}, + dictWord{135, 0, 1372}, + dictWord{138, 0, 992}, + dictWord{4, 0, 123}, + dictWord{5, 0, 605}, + dictWord{7, 0, 1509}, + dictWord{136, 0, 36}, + dictWord{132, 0, 649}, + dictWord{8, 11, 175}, + dictWord{10, 11, 168}, + dictWord{138, 11, 573}, + dictWord{133, 0, 767}, + dictWord{134, 0, 1018}, + dictWord{135, 11, 1305}, + dictWord{12, 10, 30}, + dictWord{ + 13, + 10, + 148, + }, + dictWord{14, 10, 87}, + dictWord{14, 10, 182}, + dictWord{16, 10, 42}, + dictWord{148, 10, 70}, + dictWord{134, 11, 607}, + dictWord{4, 0, 273}, + dictWord{ + 5, + 0, + 658, + }, + dictWord{133, 0, 995}, + dictWord{6, 0, 72}, + dictWord{139, 11, 174}, + dictWord{10, 0, 483}, + dictWord{12, 0, 368}, + dictWord{7, 10, 56}, + dictWord{ + 7, + 10, + 1989, + }, + dictWord{8, 10, 337}, + dictWord{8, 10, 738}, + dictWord{9, 10, 600}, + dictWord{13, 10, 447}, + dictWord{142, 10, 92}, + dictWord{5, 11, 784}, + dictWord{ + 138, + 10, + 666, + }, + dictWord{135, 0, 1345}, + dictWord{139, 11, 882}, + dictWord{134, 0, 1293}, + dictWord{133, 0, 589}, + dictWord{134, 0, 1988}, + dictWord{5, 0, 117}, + dictWord{6, 0, 514}, + dictWord{6, 0, 541}, + dictWord{7, 0, 1164}, + dictWord{7, 0, 1436}, + dictWord{8, 0, 220}, + dictWord{8, 0, 648}, + dictWord{10, 0, 688}, + dictWord{ + 139, + 0, + 560, + }, + dictWord{136, 0, 379}, + dictWord{5, 0, 686}, + dictWord{7, 10, 866}, + dictWord{135, 10, 1163}, + dictWord{132, 10, 328}, + dictWord{9, 11, 14}, + dictWord{ + 9, + 11, + 441, + }, + dictWord{10, 11, 306}, + dictWord{139, 11, 9}, + dictWord{4, 10, 101}, + dictWord{135, 10, 1171}, + dictWord{5, 10, 833}, + dictWord{136, 10, 744}, + dictWord{5, 11, 161}, + dictWord{7, 11, 839}, + dictWord{135, 11, 887}, + dictWord{7, 0, 196}, + dictWord{10, 0, 765}, + dictWord{11, 0, 347}, + dictWord{11, 0, 552}, + dictWord{11, 0, 790}, + dictWord{12, 0, 263}, + dictWord{13, 0, 246}, + dictWord{13, 0, 270}, + dictWord{13, 0, 395}, + dictWord{14, 0, 176}, + dictWord{14, 0, 190}, + dictWord{ + 14, + 0, + 398, + }, + dictWord{14, 0, 412}, + dictWord{15, 0, 32}, + dictWord{15, 0, 63}, + dictWord{16, 0, 88}, + dictWord{147, 0, 105}, + dictWord{6, 10, 9}, + dictWord{6, 10, 397}, + dictWord{7, 10, 53}, + dictWord{7, 10, 1742}, + dictWord{10, 10, 632}, + dictWord{11, 10, 828}, + dictWord{140, 10, 146}, + dictWord{5, 0, 381}, + dictWord{135, 0, 1792}, + dictWord{134, 0, 1452}, + dictWord{135, 11, 429}, + dictWord{8, 0, 367}, + dictWord{10, 0, 760}, + dictWord{14, 0, 79}, + dictWord{20, 0, 17}, + dictWord{152, 0, 0}, + dictWord{7, 0, 616}, + dictWord{138, 0, 413}, + dictWord{11, 10, 417}, + dictWord{12, 10, 223}, + dictWord{140, 10, 265}, + dictWord{7, 11, 1611}, + dictWord{13, 11, 14}, + dictWord{15, 11, 44}, + dictWord{19, 11, 13}, + dictWord{148, 11, 76}, + dictWord{135, 0, 1229}, + dictWord{6, 0, 120}, + dictWord{7, 0, 1188}, + dictWord{7, 0, 1710}, + dictWord{8, 0, 286}, + dictWord{9, 0, 667}, + dictWord{11, 0, 592}, + dictWord{139, 0, 730}, + dictWord{135, 11, 1814}, + dictWord{135, 0, 1146}, + dictWord{4, 10, 186}, + dictWord{5, 10, 157}, + dictWord{8, 10, 168}, + dictWord{138, 10, 6}, + dictWord{4, 0, 352}, + dictWord{135, 0, 687}, + dictWord{4, 0, 192}, + dictWord{5, 0, 49}, + dictWord{ + 6, + 0, + 200, + }, + dictWord{6, 0, 293}, + dictWord{6, 0, 1696}, + dictWord{135, 0, 1151}, + dictWord{133, 10, 875}, + dictWord{5, 10, 773}, + dictWord{5, 10, 991}, + dictWord{ + 6, + 10, + 1635, + }, + dictWord{134, 10, 1788}, + dictWord{7, 10, 111}, + dictWord{136, 10, 581}, + dictWord{6, 0, 935}, + dictWord{134, 0, 1151}, + dictWord{134, 0, 1050}, + dictWord{132, 0, 650}, + dictWord{132, 0, 147}, + dictWord{11, 0, 194}, + dictWord{12, 0, 62}, + dictWord{12, 0, 88}, + dictWord{11, 11, 194}, + dictWord{12, 11, 62}, + dictWord{140, 11, 88}, + dictWord{6, 0, 339}, + dictWord{135, 0, 923}, + dictWord{134, 10, 1747}, + dictWord{7, 11, 643}, + dictWord{136, 11, 236}, + dictWord{ + 133, + 0, + 934, + }, + dictWord{7, 10, 1364}, + dictWord{7, 10, 1907}, + dictWord{141, 10, 158}, + dictWord{132, 10, 659}, + dictWord{4, 10, 404}, + dictWord{135, 10, 675}, + dictWord{7, 11, 581}, + dictWord{9, 11, 644}, + dictWord{137, 11, 699}, + dictWord{13, 0, 211}, + dictWord{14, 0, 133}, + dictWord{14, 0, 204}, + dictWord{15, 0, 64}, + dictWord{ + 15, + 0, + 69, + }, + dictWord{15, 0, 114}, + dictWord{16, 0, 10}, + dictWord{19, 0, 23}, + dictWord{19, 0, 35}, + dictWord{19, 0, 39}, + dictWord{19, 0, 51}, + dictWord{19, 0, 71}, + dictWord{19, 0, 75}, + dictWord{152, 0, 15}, + dictWord{133, 10, 391}, + dictWord{5, 11, 54}, + dictWord{135, 11, 1513}, + dictWord{7, 0, 222}, + dictWord{8, 0, 341}, + dictWord{ + 5, + 10, + 540, + }, + dictWord{134, 10, 1697}, + dictWord{134, 10, 78}, + dictWord{132, 11, 744}, + dictWord{136, 0, 293}, + dictWord{137, 11, 701}, + dictWord{ + 7, + 11, + 930, + }, + dictWord{10, 11, 402}, + dictWord{10, 11, 476}, + dictWord{13, 11, 452}, + dictWord{18, 11, 55}, + dictWord{147, 11, 104}, + dictWord{132, 0, 637}, + dictWord{133, 10, 460}, + dictWord{8, 11, 50}, + dictWord{137, 11, 624}, + dictWord{132, 11, 572}, + dictWord{134, 0, 1159}, + dictWord{4, 10, 199}, + dictWord{ + 139, + 10, + 34, + }, + dictWord{134, 0, 847}, + dictWord{134, 10, 388}, + dictWord{6, 11, 43}, + dictWord{7, 11, 38}, + dictWord{8, 11, 248}, + dictWord{9, 11, 504}, + dictWord{ + 138, + 11, + 513, + }, + dictWord{9, 0, 683}, + dictWord{4, 10, 511}, + dictWord{6, 10, 608}, + dictWord{9, 10, 333}, + dictWord{10, 10, 602}, + dictWord{11, 10, 441}, + dictWord{ + 11, + 10, + 723, + }, + dictWord{11, 10, 976}, + dictWord{140, 10, 357}, + dictWord{9, 0, 867}, + dictWord{138, 0, 837}, + dictWord{6, 0, 944}, + dictWord{135, 11, 326}, + dictWord{ + 135, + 0, + 1809, + }, + dictWord{5, 10, 938}, + dictWord{7, 11, 783}, + dictWord{136, 10, 707}, + dictWord{133, 11, 766}, + dictWord{133, 11, 363}, + dictWord{6, 0, 170}, + dictWord{7, 0, 1080}, + dictWord{8, 0, 395}, + dictWord{8, 0, 487}, + dictWord{141, 0, 147}, + dictWord{6, 11, 258}, + dictWord{140, 11, 409}, + dictWord{4, 0, 535}, + dictWord{ + 8, + 0, + 618, + }, + dictWord{5, 11, 249}, + dictWord{148, 11, 82}, + dictWord{6, 0, 1379}, + dictWord{149, 11, 15}, + dictWord{135, 0, 1625}, + dictWord{150, 0, 23}, + dictWord{ + 5, + 11, + 393, + }, + dictWord{6, 11, 378}, + dictWord{7, 11, 1981}, + dictWord{9, 11, 32}, + dictWord{9, 11, 591}, + dictWord{10, 11, 685}, + dictWord{10, 11, 741}, + dictWord{ + 142, + 11, + 382, + }, + dictWord{133, 11, 788}, + dictWord{7, 11, 1968}, + dictWord{10, 11, 19}, + dictWord{139, 11, 911}, + dictWord{7, 11, 1401}, + dictWord{ + 135, + 11, + 1476, + }, + dictWord{4, 11, 61}, + dictWord{5, 11, 58}, + dictWord{5, 11, 171}, + dictWord{5, 11, 635}, + dictWord{5, 11, 683}, + dictWord{5, 11, 700}, + dictWord{6, 11, 291}, + dictWord{6, 11, 566}, + dictWord{7, 11, 1650}, + dictWord{11, 11, 523}, + dictWord{12, 11, 273}, + dictWord{12, 11, 303}, + dictWord{15, 11, 39}, + dictWord{ + 143, + 11, + 111, + }, + dictWord{6, 10, 469}, + dictWord{7, 10, 1709}, + dictWord{138, 10, 515}, + dictWord{4, 0, 778}, + dictWord{134, 11, 589}, + dictWord{132, 0, 46}, + dictWord{ + 5, + 0, + 811, + }, + dictWord{6, 0, 1679}, + dictWord{6, 0, 1714}, + dictWord{135, 0, 2032}, + dictWord{7, 0, 1458}, + dictWord{9, 0, 407}, + dictWord{11, 0, 15}, + dictWord{12, 0, 651}, + dictWord{149, 0, 37}, + dictWord{7, 0, 938}, + dictWord{132, 10, 500}, + dictWord{6, 0, 34}, + dictWord{7, 0, 69}, + dictWord{7, 0, 1089}, + dictWord{7, 0, 1281}, + dictWord{ + 8, + 0, + 708, + }, + dictWord{8, 0, 721}, + dictWord{9, 0, 363}, + dictWord{148, 0, 98}, + dictWord{10, 11, 231}, + dictWord{147, 11, 124}, + dictWord{7, 11, 726}, + dictWord{ + 152, + 11, + 9, + }, + dictWord{5, 10, 68}, + dictWord{134, 10, 383}, + dictWord{136, 11, 583}, + dictWord{4, 11, 917}, + dictWord{133, 11, 1005}, + dictWord{11, 10, 216}, + dictWord{139, 10, 340}, + dictWord{135, 11, 1675}, + dictWord{8, 0, 441}, + dictWord{10, 0, 314}, + dictWord{143, 0, 3}, + dictWord{132, 11, 919}, + dictWord{4, 10, 337}, + dictWord{6, 10, 353}, + dictWord{7, 10, 1934}, + dictWord{8, 10, 488}, + dictWord{137, 10, 429}, + dictWord{7, 0, 889}, + dictWord{7, 10, 1795}, + dictWord{8, 10, 259}, + dictWord{9, 10, 135}, + dictWord{9, 10, 177}, + dictWord{9, 10, 860}, + dictWord{10, 10, 825}, + dictWord{11, 10, 115}, + dictWord{11, 10, 370}, + dictWord{11, 10, 405}, + dictWord{11, 10, 604}, + dictWord{12, 10, 10}, + dictWord{12, 10, 667}, + dictWord{12, 10, 669}, + dictWord{13, 10, 76}, + dictWord{14, 10, 310}, + dictWord{ + 15, + 10, + 76, + }, + dictWord{15, 10, 147}, + dictWord{148, 10, 23}, + dictWord{4, 10, 15}, + dictWord{4, 11, 255}, + dictWord{5, 10, 22}, + dictWord{5, 11, 302}, + dictWord{6, 11, 132}, + dictWord{6, 10, 244}, + dictWord{7, 10, 40}, + dictWord{7, 11, 128}, + dictWord{7, 10, 200}, + dictWord{7, 11, 283}, + dictWord{7, 10, 906}, + dictWord{7, 10, 1199}, + dictWord{ + 7, + 11, + 1299, + }, + dictWord{9, 10, 616}, + dictWord{10, 11, 52}, + dictWord{10, 11, 514}, + dictWord{10, 10, 716}, + dictWord{11, 10, 635}, + dictWord{11, 10, 801}, + dictWord{11, 11, 925}, + dictWord{12, 10, 458}, + dictWord{13, 11, 92}, + dictWord{142, 11, 309}, + dictWord{132, 0, 462}, + dictWord{137, 11, 173}, + dictWord{ + 135, + 10, + 1735, + }, + dictWord{8, 0, 525}, + dictWord{5, 10, 598}, + dictWord{7, 10, 791}, + dictWord{8, 10, 108}, + dictWord{137, 10, 123}, + dictWord{5, 0, 73}, + dictWord{6, 0, 23}, + dictWord{134, 0, 338}, + dictWord{132, 0, 676}, + dictWord{132, 10, 683}, + dictWord{7, 0, 725}, + dictWord{8, 0, 498}, + dictWord{139, 0, 268}, + dictWord{12, 0, 21}, + dictWord{151, 0, 7}, + dictWord{135, 0, 773}, + dictWord{4, 10, 155}, + dictWord{135, 10, 1689}, + dictWord{4, 0, 164}, + dictWord{5, 0, 730}, + dictWord{5, 10, 151}, + dictWord{ + 5, + 10, + 741, + }, + dictWord{6, 11, 210}, + dictWord{7, 10, 498}, + dictWord{7, 10, 870}, + dictWord{7, 10, 1542}, + dictWord{12, 10, 213}, + dictWord{14, 10, 36}, + dictWord{ + 14, + 10, + 391, + }, + dictWord{17, 10, 111}, + dictWord{18, 10, 6}, + dictWord{18, 10, 46}, + dictWord{18, 10, 151}, + dictWord{19, 10, 36}, + dictWord{20, 10, 32}, + dictWord{ + 20, + 10, + 56, + }, + dictWord{20, 10, 69}, + dictWord{20, 10, 102}, + dictWord{21, 10, 4}, + dictWord{22, 10, 8}, + dictWord{22, 10, 10}, + dictWord{22, 10, 14}, + dictWord{ + 150, + 10, + 31, + }, + dictWord{4, 10, 624}, + dictWord{135, 10, 1752}, + dictWord{4, 0, 583}, + dictWord{9, 0, 936}, + dictWord{15, 0, 214}, + dictWord{18, 0, 199}, + dictWord{24, 0, 26}, + dictWord{134, 11, 588}, + dictWord{7, 0, 1462}, + dictWord{11, 0, 659}, + dictWord{4, 11, 284}, + dictWord{134, 11, 223}, + dictWord{133, 0, 220}, + dictWord{ + 139, + 0, + 803, + }, + dictWord{132, 0, 544}, + dictWord{4, 10, 492}, + dictWord{133, 10, 451}, + dictWord{16, 0, 98}, + dictWord{148, 0, 119}, + dictWord{4, 11, 218}, + dictWord{ + 7, + 11, + 526, + }, + dictWord{143, 11, 137}, + dictWord{135, 10, 835}, + dictWord{4, 11, 270}, + dictWord{5, 11, 192}, + dictWord{6, 11, 332}, + dictWord{7, 11, 1322}, + dictWord{ + 13, + 11, + 9, + }, + dictWord{13, 10, 70}, + dictWord{14, 11, 104}, + dictWord{142, 11, 311}, + dictWord{132, 10, 539}, + dictWord{140, 11, 661}, + dictWord{5, 0, 176}, + dictWord{ + 6, + 0, + 437, + }, + dictWord{6, 0, 564}, + dictWord{11, 0, 181}, + dictWord{141, 0, 183}, + dictWord{135, 0, 1192}, + dictWord{6, 10, 113}, + dictWord{135, 10, 436}, + dictWord{136, 10, 718}, + dictWord{135, 10, 520}, + dictWord{135, 0, 1878}, + dictWord{140, 11, 196}, + dictWord{7, 11, 379}, + dictWord{8, 11, 481}, + dictWord{ + 137, + 11, + 377, + }, + dictWord{5, 11, 1003}, + dictWord{6, 11, 149}, + dictWord{137, 11, 746}, + dictWord{8, 11, 262}, + dictWord{9, 11, 627}, + dictWord{10, 11, 18}, + dictWord{ + 11, + 11, + 214, + }, + dictWord{11, 11, 404}, + dictWord{11, 11, 457}, + dictWord{11, 11, 780}, + dictWord{11, 11, 849}, + dictWord{11, 11, 913}, + dictWord{13, 11, 330}, + dictWord{13, 11, 401}, + dictWord{142, 11, 200}, + dictWord{149, 0, 26}, + dictWord{136, 11, 304}, + dictWord{132, 11, 142}, + dictWord{135, 0, 944}, + dictWord{ + 4, + 0, + 790, + }, + dictWord{5, 0, 273}, + dictWord{134, 0, 394}, + dictWord{134, 0, 855}, + dictWord{4, 0, 135}, + dictWord{6, 0, 127}, + dictWord{7, 0, 1185}, + dictWord{7, 0, 1511}, + dictWord{8, 0, 613}, + dictWord{11, 0, 5}, + dictWord{12, 0, 336}, + dictWord{12, 0, 495}, + dictWord{12, 0, 586}, + dictWord{12, 0, 660}, + dictWord{12, 0, 668}, + dictWord{ + 14, + 0, + 385, + }, + dictWord{15, 0, 118}, + dictWord{17, 0, 20}, + dictWord{146, 0, 98}, + dictWord{6, 0, 230}, + dictWord{9, 0, 752}, + dictWord{18, 0, 109}, + dictWord{12, 10, 610}, + dictWord{13, 10, 431}, + dictWord{144, 10, 59}, + dictWord{7, 0, 1954}, + dictWord{135, 11, 925}, + dictWord{4, 11, 471}, + dictWord{5, 11, 51}, + dictWord{6, 11, 602}, + dictWord{8, 11, 484}, + dictWord{10, 11, 195}, + dictWord{140, 11, 159}, + dictWord{132, 10, 307}, + dictWord{136, 11, 688}, + dictWord{132, 11, 697}, + dictWord{ + 7, + 11, + 812, + }, + dictWord{7, 11, 1261}, + dictWord{7, 11, 1360}, + dictWord{9, 11, 632}, + dictWord{140, 11, 352}, + dictWord{5, 0, 162}, + dictWord{8, 0, 68}, + dictWord{ + 133, + 10, + 964, + }, + dictWord{4, 0, 654}, + dictWord{136, 11, 212}, + dictWord{4, 0, 156}, + dictWord{7, 0, 998}, + dictWord{7, 0, 1045}, + dictWord{7, 0, 1860}, + dictWord{9, 0, 48}, + dictWord{9, 0, 692}, + dictWord{11, 0, 419}, + dictWord{139, 0, 602}, + dictWord{133, 11, 221}, + dictWord{4, 11, 373}, + dictWord{5, 11, 283}, + dictWord{6, 11, 480}, + dictWord{135, 11, 609}, + dictWord{142, 11, 216}, + dictWord{132, 0, 240}, + dictWord{6, 11, 192}, + dictWord{9, 11, 793}, + dictWord{145, 11, 55}, + dictWord{ + 4, + 10, + 75, + }, + dictWord{5, 10, 180}, + dictWord{6, 10, 500}, + dictWord{7, 10, 58}, + dictWord{7, 10, 710}, + dictWord{138, 10, 645}, + dictWord{4, 11, 132}, + dictWord{5, 11, 69}, + dictWord{5, 10, 649}, + dictWord{135, 11, 1242}, + dictWord{6, 10, 276}, + dictWord{7, 10, 282}, + dictWord{7, 10, 879}, + dictWord{7, 10, 924}, + dictWord{8, 10, 459}, + dictWord{9, 10, 599}, + dictWord{9, 10, 754}, + dictWord{11, 10, 574}, + dictWord{12, 10, 128}, + dictWord{12, 10, 494}, + dictWord{13, 10, 52}, + dictWord{13, 10, 301}, + dictWord{15, 10, 30}, + dictWord{143, 10, 132}, + dictWord{132, 10, 200}, + dictWord{4, 11, 111}, + dictWord{135, 11, 302}, + dictWord{9, 0, 197}, + dictWord{ + 10, + 0, + 300, + }, + dictWord{12, 0, 473}, + dictWord{13, 0, 90}, + dictWord{141, 0, 405}, + dictWord{132, 11, 767}, + dictWord{6, 11, 42}, + dictWord{7, 11, 1416}, + dictWord{ + 7, + 11, + 1590, + }, + dictWord{7, 11, 2005}, + dictWord{8, 11, 131}, + dictWord{8, 11, 466}, + dictWord{9, 11, 672}, + dictWord{13, 11, 252}, + dictWord{148, 11, 103}, + dictWord{ + 8, + 0, + 958, + }, + dictWord{8, 0, 999}, + dictWord{10, 0, 963}, + dictWord{138, 0, 1001}, + dictWord{135, 10, 1621}, + dictWord{135, 0, 858}, + dictWord{4, 0, 606}, + dictWord{ + 137, + 11, + 444, + }, + dictWord{6, 11, 44}, + dictWord{136, 11, 368}, + dictWord{139, 11, 172}, + dictWord{4, 11, 570}, + dictWord{133, 11, 120}, + dictWord{139, 11, 624}, + dictWord{7, 0, 1978}, + dictWord{8, 0, 676}, + dictWord{6, 10, 225}, + dictWord{137, 10, 211}, + dictWord{7, 0, 972}, + dictWord{11, 0, 102}, + dictWord{136, 10, 687}, + dictWord{6, 11, 227}, + dictWord{135, 11, 1589}, + dictWord{8, 10, 58}, + dictWord{9, 10, 724}, + dictWord{11, 10, 809}, + dictWord{13, 10, 113}, + dictWord{ + 145, + 10, + 72, + }, + dictWord{4, 0, 361}, + dictWord{133, 0, 315}, + dictWord{132, 0, 461}, + dictWord{6, 10, 345}, + dictWord{135, 10, 1247}, + dictWord{132, 0, 472}, + dictWord{ + 8, + 10, + 767, + }, + dictWord{8, 10, 803}, + dictWord{9, 10, 301}, + dictWord{137, 10, 903}, + dictWord{135, 11, 1333}, + dictWord{135, 11, 477}, + dictWord{7, 10, 1949}, + dictWord{136, 10, 674}, + dictWord{6, 0, 905}, + dictWord{138, 0, 747}, + dictWord{133, 0, 155}, + dictWord{134, 10, 259}, + dictWord{7, 0, 163}, + dictWord{8, 0, 319}, + dictWord{9, 0, 402}, + dictWord{10, 0, 24}, + dictWord{10, 0, 681}, + dictWord{11, 0, 200}, + dictWord{12, 0, 253}, + dictWord{12, 0, 410}, + dictWord{142, 0, 219}, + dictWord{ + 5, + 0, + 475, + }, + dictWord{7, 0, 1780}, + dictWord{9, 0, 230}, + dictWord{11, 0, 297}, + dictWord{11, 0, 558}, + dictWord{14, 0, 322}, + dictWord{19, 0, 76}, + dictWord{6, 11, 1667}, + dictWord{7, 11, 2036}, + dictWord{138, 11, 600}, + dictWord{136, 10, 254}, + dictWord{6, 0, 848}, + dictWord{135, 0, 1956}, + dictWord{6, 11, 511}, + dictWord{ + 140, + 11, + 132, + }, + dictWord{5, 11, 568}, + dictWord{6, 11, 138}, + dictWord{135, 11, 1293}, + dictWord{6, 0, 631}, + dictWord{137, 0, 838}, + dictWord{149, 0, 36}, + dictWord{ + 4, + 11, + 565, + }, + dictWord{8, 11, 23}, + dictWord{136, 11, 827}, + dictWord{5, 0, 944}, + dictWord{134, 0, 1769}, + dictWord{4, 0, 144}, + dictWord{6, 0, 842}, + dictWord{ + 6, + 0, + 1400, + }, + dictWord{4, 11, 922}, + dictWord{133, 11, 1023}, + dictWord{133, 10, 248}, + dictWord{9, 10, 800}, + dictWord{10, 10, 693}, + dictWord{11, 10, 482}, + dictWord{11, 10, 734}, + dictWord{139, 10, 789}, + dictWord{7, 11, 1002}, + dictWord{139, 11, 145}, + dictWord{4, 10, 116}, + dictWord{5, 10, 95}, + dictWord{5, 10, 445}, + dictWord{7, 10, 1688}, + dictWord{8, 10, 29}, + dictWord{9, 10, 272}, + dictWord{11, 10, 509}, + dictWord{139, 10, 915}, + dictWord{14, 0, 369}, + dictWord{146, 0, 72}, + dictWord{135, 10, 1641}, + dictWord{132, 11, 740}, + dictWord{133, 10, 543}, + dictWord{140, 11, 116}, + dictWord{6, 0, 247}, + dictWord{9, 0, 555}, + dictWord{ + 5, + 10, + 181, + }, + dictWord{136, 10, 41}, + dictWord{133, 10, 657}, + dictWord{136, 0, 996}, + dictWord{138, 10, 709}, + dictWord{7, 0, 189}, + dictWord{8, 10, 202}, + dictWord{ + 138, + 10, + 536, + }, + dictWord{136, 11, 402}, + dictWord{4, 11, 716}, + dictWord{141, 11, 31}, + dictWord{10, 0, 280}, + dictWord{138, 0, 797}, + dictWord{9, 10, 423}, + dictWord{140, 10, 89}, + dictWord{8, 10, 113}, + dictWord{9, 10, 877}, + dictWord{10, 10, 554}, + dictWord{11, 10, 83}, + dictWord{12, 10, 136}, + dictWord{147, 10, 109}, + dictWord{133, 10, 976}, + dictWord{7, 0, 746}, + dictWord{132, 10, 206}, + dictWord{136, 0, 526}, + dictWord{139, 0, 345}, + dictWord{136, 0, 1017}, + dictWord{ + 8, + 11, + 152, + }, + dictWord{9, 11, 53}, + dictWord{9, 11, 268}, + dictWord{9, 11, 901}, + dictWord{10, 11, 518}, + dictWord{10, 11, 829}, + dictWord{11, 11, 188}, + dictWord{ + 13, + 11, + 74, + }, + dictWord{14, 11, 46}, + dictWord{15, 11, 17}, + dictWord{15, 11, 33}, + dictWord{17, 11, 40}, + dictWord{18, 11, 36}, + dictWord{19, 11, 20}, + dictWord{22, 11, 1}, + dictWord{152, 11, 2}, + dictWord{133, 11, 736}, + dictWord{136, 11, 532}, + dictWord{5, 0, 428}, + dictWord{138, 0, 651}, + dictWord{135, 11, 681}, + dictWord{ + 135, + 0, + 1162, + }, + dictWord{7, 0, 327}, + dictWord{13, 0, 230}, + dictWord{17, 0, 113}, + dictWord{8, 10, 226}, + dictWord{10, 10, 537}, + dictWord{11, 10, 570}, + dictWord{ + 11, + 10, + 605, + }, + dictWord{11, 10, 799}, + dictWord{11, 10, 804}, + dictWord{12, 10, 85}, + dictWord{12, 10, 516}, + dictWord{12, 10, 623}, + dictWord{12, 11, 677}, + dictWord{ + 13, + 10, + 361, + }, + dictWord{14, 10, 77}, + dictWord{14, 10, 78}, + dictWord{147, 10, 110}, + dictWord{4, 0, 792}, + dictWord{7, 0, 1717}, + dictWord{10, 0, 546}, + dictWord{ + 132, + 10, + 769, + }, + dictWord{4, 11, 684}, + dictWord{136, 11, 384}, + dictWord{132, 10, 551}, + dictWord{134, 0, 1203}, + dictWord{9, 10, 57}, + dictWord{9, 10, 459}, + dictWord{10, 10, 425}, + dictWord{11, 10, 119}, + dictWord{12, 10, 184}, + dictWord{12, 10, 371}, + dictWord{13, 10, 358}, + dictWord{145, 10, 51}, + dictWord{5, 0, 672}, + dictWord{5, 10, 814}, + dictWord{8, 10, 10}, + dictWord{9, 10, 421}, + dictWord{9, 10, 729}, + dictWord{10, 10, 609}, + dictWord{139, 10, 689}, + dictWord{138, 0, 189}, + dictWord{134, 10, 624}, + dictWord{7, 11, 110}, + dictWord{7, 11, 188}, + dictWord{8, 11, 290}, + dictWord{8, 11, 591}, + dictWord{9, 11, 382}, + dictWord{9, 11, 649}, + dictWord{11, 11, 71}, + dictWord{11, 11, 155}, + dictWord{11, 11, 313}, + dictWord{12, 11, 5}, + dictWord{13, 11, 325}, + dictWord{142, 11, 287}, + dictWord{133, 0, 99}, + dictWord{6, 0, 1053}, + dictWord{135, 0, 298}, + dictWord{7, 11, 360}, + dictWord{7, 11, 425}, + dictWord{9, 11, 66}, + dictWord{9, 11, 278}, + dictWord{138, 11, 644}, + dictWord{4, 0, 397}, + dictWord{136, 0, 555}, + dictWord{137, 10, 269}, + dictWord{132, 10, 528}, + dictWord{4, 11, 900}, + dictWord{133, 11, 861}, + dictWord{ + 6, + 0, + 1157, + }, + dictWord{5, 11, 254}, + dictWord{7, 11, 985}, + dictWord{136, 11, 73}, + dictWord{7, 11, 1959}, + dictWord{136, 11, 683}, + dictWord{12, 0, 398}, + dictWord{ + 20, + 0, + 39, + }, + dictWord{21, 0, 11}, + dictWord{150, 0, 41}, + dictWord{4, 0, 485}, + dictWord{7, 0, 353}, + dictWord{135, 0, 1523}, + dictWord{6, 0, 366}, + dictWord{7, 0, 1384}, + dictWord{135, 0, 1601}, + dictWord{138, 0, 787}, + dictWord{137, 0, 282}, + dictWord{5, 10, 104}, + dictWord{6, 10, 173}, + dictWord{135, 10, 1631}, + dictWord{ + 139, + 11, + 146, + }, + dictWord{4, 0, 157}, + dictWord{133, 0, 471}, + dictWord{134, 0, 941}, + dictWord{132, 11, 725}, + dictWord{7, 0, 1336}, + dictWord{8, 10, 138}, + dictWord{ + 8, + 10, + 342, + }, + dictWord{9, 10, 84}, + dictWord{10, 10, 193}, + dictWord{11, 10, 883}, + dictWord{140, 10, 359}, + dictWord{134, 11, 196}, + dictWord{136, 0, 116}, + dictWord{133, 11, 831}, + dictWord{134, 0, 787}, + dictWord{134, 10, 95}, + dictWord{6, 10, 406}, + dictWord{10, 10, 409}, + dictWord{10, 10, 447}, + dictWord{ + 11, + 10, + 44, + }, + dictWord{140, 10, 100}, + dictWord{5, 0, 160}, + dictWord{7, 0, 363}, + dictWord{7, 0, 589}, + dictWord{10, 0, 170}, + dictWord{141, 0, 55}, + dictWord{134, 0, 1815}, + dictWord{132, 0, 866}, + dictWord{6, 0, 889}, + dictWord{6, 0, 1067}, + dictWord{6, 0, 1183}, + dictWord{4, 11, 321}, + dictWord{134, 11, 569}, + dictWord{5, 11, 848}, + dictWord{134, 11, 66}, + dictWord{4, 11, 36}, + dictWord{6, 10, 1636}, + dictWord{7, 11, 1387}, + dictWord{10, 11, 205}, + dictWord{11, 11, 755}, + dictWord{ + 141, + 11, + 271, + }, + dictWord{132, 0, 689}, + dictWord{9, 0, 820}, + dictWord{4, 10, 282}, + dictWord{7, 10, 1034}, + dictWord{11, 10, 398}, + dictWord{11, 10, 634}, + dictWord{ + 12, + 10, + 1, + }, + dictWord{12, 10, 79}, + dictWord{12, 10, 544}, + dictWord{14, 10, 237}, + dictWord{17, 10, 10}, + dictWord{146, 10, 20}, + dictWord{4, 0, 108}, + dictWord{7, 0, 804}, + dictWord{139, 0, 498}, + dictWord{132, 11, 887}, + dictWord{6, 0, 1119}, + dictWord{135, 11, 620}, + dictWord{6, 11, 165}, + dictWord{138, 11, 388}, + dictWord{ + 5, + 0, + 244, + }, + dictWord{5, 10, 499}, + dictWord{6, 10, 476}, + dictWord{7, 10, 600}, + dictWord{7, 10, 888}, + dictWord{135, 10, 1096}, + dictWord{140, 0, 609}, + dictWord{ + 135, + 0, + 1005, + }, + dictWord{4, 0, 412}, + dictWord{133, 0, 581}, + dictWord{4, 11, 719}, + dictWord{135, 11, 155}, + dictWord{7, 10, 296}, + dictWord{7, 10, 596}, + dictWord{ + 8, + 10, + 560, + }, + dictWord{8, 10, 586}, + dictWord{9, 10, 612}, + dictWord{11, 10, 304}, + dictWord{12, 10, 46}, + dictWord{13, 10, 89}, + dictWord{14, 10, 112}, + dictWord{ + 145, + 10, + 122, + }, + dictWord{4, 0, 895}, + dictWord{133, 0, 772}, + dictWord{142, 11, 307}, + dictWord{135, 0, 1898}, + dictWord{4, 0, 926}, + dictWord{133, 0, 983}, + dictWord{4, 11, 353}, + dictWord{6, 11, 146}, + dictWord{6, 11, 1789}, + dictWord{7, 11, 288}, + dictWord{7, 11, 990}, + dictWord{7, 11, 1348}, + dictWord{9, 11, 665}, + dictWord{ + 9, + 11, + 898, + }, + dictWord{11, 11, 893}, + dictWord{142, 11, 212}, + dictWord{132, 0, 538}, + dictWord{133, 11, 532}, + dictWord{6, 0, 294}, + dictWord{7, 0, 1267}, + dictWord{8, 0, 624}, + dictWord{141, 0, 496}, + dictWord{7, 0, 1325}, + dictWord{4, 11, 45}, + dictWord{135, 11, 1257}, + dictWord{138, 0, 301}, + dictWord{9, 0, 298}, + dictWord{12, 0, 291}, + dictWord{13, 0, 276}, + dictWord{14, 0, 6}, + dictWord{17, 0, 18}, + dictWord{21, 0, 32}, + dictWord{7, 10, 1599}, + dictWord{7, 10, 1723}, + dictWord{ + 8, + 10, + 79, + }, + dictWord{8, 10, 106}, + dictWord{8, 10, 190}, + dictWord{8, 10, 302}, + dictWord{8, 10, 383}, + dictWord{8, 10, 713}, + dictWord{9, 10, 119}, + dictWord{9, 10, 233}, + dictWord{9, 10, 419}, + dictWord{9, 10, 471}, + dictWord{10, 10, 181}, + dictWord{10, 10, 406}, + dictWord{11, 10, 57}, + dictWord{11, 10, 85}, + dictWord{11, 10, 120}, + dictWord{11, 10, 177}, + dictWord{11, 10, 296}, + dictWord{11, 10, 382}, + dictWord{11, 10, 454}, + dictWord{11, 10, 758}, + dictWord{11, 10, 999}, + dictWord{ + 12, + 10, + 27, + }, + dictWord{12, 10, 131}, + dictWord{12, 10, 245}, + dictWord{12, 10, 312}, + dictWord{12, 10, 446}, + dictWord{12, 10, 454}, + dictWord{13, 10, 98}, + dictWord{ + 13, + 10, + 426, + }, + dictWord{13, 10, 508}, + dictWord{14, 10, 163}, + dictWord{14, 10, 272}, + dictWord{14, 10, 277}, + dictWord{14, 10, 370}, + dictWord{15, 10, 95}, + dictWord{15, 10, 138}, + dictWord{15, 10, 167}, + dictWord{17, 10, 38}, + dictWord{148, 10, 96}, + dictWord{132, 0, 757}, + dictWord{134, 0, 1263}, + dictWord{4, 0, 820}, + dictWord{134, 10, 1759}, + dictWord{133, 0, 722}, + dictWord{136, 11, 816}, + dictWord{138, 10, 372}, + dictWord{145, 10, 16}, + dictWord{134, 0, 1039}, + dictWord{ + 4, + 0, + 991, + }, + dictWord{134, 0, 2028}, + dictWord{133, 10, 258}, + dictWord{7, 0, 1875}, + dictWord{139, 0, 124}, + dictWord{6, 11, 559}, + dictWord{6, 11, 1691}, + dictWord{135, 11, 586}, + dictWord{5, 0, 324}, + dictWord{7, 0, 881}, + dictWord{8, 10, 134}, + dictWord{9, 10, 788}, + dictWord{140, 10, 438}, + dictWord{7, 11, 1823}, + dictWord{139, 11, 693}, + dictWord{6, 0, 1348}, + dictWord{134, 0, 1545}, + dictWord{134, 0, 911}, + dictWord{132, 0, 954}, + dictWord{8, 0, 329}, + dictWord{8, 0, 414}, + dictWord{7, 10, 1948}, + dictWord{135, 10, 2004}, + dictWord{5, 0, 517}, + dictWord{6, 10, 439}, + dictWord{7, 10, 780}, + dictWord{135, 10, 1040}, + dictWord{ + 132, + 0, + 816, + }, + dictWord{5, 10, 1}, + dictWord{6, 10, 81}, + dictWord{138, 10, 520}, + dictWord{9, 0, 713}, + dictWord{10, 0, 222}, + dictWord{5, 10, 482}, + dictWord{8, 10, 98}, + dictWord{10, 10, 700}, + dictWord{10, 10, 822}, + dictWord{11, 10, 302}, + dictWord{11, 10, 778}, + dictWord{12, 10, 50}, + dictWord{12, 10, 127}, + dictWord{12, 10, 396}, + dictWord{13, 10, 62}, + dictWord{13, 10, 328}, + dictWord{14, 10, 122}, + dictWord{147, 10, 72}, + dictWord{137, 0, 33}, + dictWord{5, 10, 2}, + dictWord{7, 10, 1494}, + dictWord{136, 10, 589}, + dictWord{6, 10, 512}, + dictWord{7, 10, 797}, + dictWord{8, 10, 253}, + dictWord{9, 10, 77}, + dictWord{10, 10, 1}, + dictWord{10, 11, 108}, + dictWord{10, 10, 129}, + dictWord{10, 10, 225}, + dictWord{11, 11, 116}, + dictWord{11, 10, 118}, + dictWord{11, 10, 226}, + dictWord{11, 10, 251}, + dictWord{ + 11, + 10, + 430, + }, + dictWord{11, 10, 701}, + dictWord{11, 10, 974}, + dictWord{11, 10, 982}, + dictWord{12, 10, 64}, + dictWord{12, 10, 260}, + dictWord{12, 10, 488}, + dictWord{ + 140, + 10, + 690, + }, + dictWord{134, 11, 456}, + dictWord{133, 11, 925}, + dictWord{5, 0, 150}, + dictWord{7, 0, 106}, + dictWord{7, 0, 774}, + dictWord{8, 0, 603}, + dictWord{ + 9, + 0, + 593, + }, + dictWord{9, 0, 634}, + dictWord{10, 0, 44}, + dictWord{10, 0, 173}, + dictWord{11, 0, 462}, + dictWord{11, 0, 515}, + dictWord{13, 0, 216}, + dictWord{13, 0, 288}, + dictWord{142, 0, 400}, + dictWord{137, 10, 347}, + dictWord{5, 0, 748}, + dictWord{134, 0, 553}, + dictWord{12, 0, 108}, + dictWord{141, 0, 291}, + dictWord{7, 0, 420}, + dictWord{4, 10, 12}, + dictWord{7, 10, 522}, + dictWord{7, 10, 809}, + dictWord{8, 10, 797}, + dictWord{141, 10, 88}, + dictWord{6, 11, 193}, + dictWord{7, 11, 240}, + dictWord{ + 7, + 11, + 1682, + }, + dictWord{10, 11, 51}, + dictWord{10, 11, 640}, + dictWord{11, 11, 410}, + dictWord{13, 11, 82}, + dictWord{14, 11, 247}, + dictWord{14, 11, 331}, + dictWord{142, 11, 377}, + dictWord{133, 10, 528}, + dictWord{135, 0, 1777}, + dictWord{4, 0, 493}, + dictWord{144, 0, 55}, + dictWord{136, 11, 633}, + dictWord{ + 139, + 0, + 81, + }, + dictWord{6, 0, 980}, + dictWord{136, 0, 321}, + dictWord{148, 10, 109}, + dictWord{5, 10, 266}, + dictWord{9, 10, 290}, + dictWord{9, 10, 364}, + dictWord{ + 10, + 10, + 293, + }, + dictWord{11, 10, 606}, + dictWord{142, 10, 45}, + dictWord{6, 0, 568}, + dictWord{7, 0, 112}, + dictWord{7, 0, 1804}, + dictWord{8, 0, 362}, + dictWord{8, 0, 410}, + dictWord{8, 0, 830}, + dictWord{9, 0, 514}, + dictWord{11, 0, 649}, + dictWord{142, 0, 157}, + dictWord{4, 0, 74}, + dictWord{6, 0, 510}, + dictWord{6, 10, 594}, + dictWord{ + 9, + 10, + 121, + }, + dictWord{10, 10, 49}, + dictWord{10, 10, 412}, + dictWord{139, 10, 834}, + dictWord{134, 0, 838}, + dictWord{136, 10, 748}, + dictWord{132, 10, 466}, + dictWord{132, 0, 625}, + dictWord{135, 11, 1443}, + dictWord{4, 11, 237}, + dictWord{135, 11, 514}, + dictWord{9, 10, 378}, + dictWord{141, 10, 162}, + dictWord{6, 0, 16}, + dictWord{6, 0, 158}, + dictWord{7, 0, 43}, + dictWord{7, 0, 129}, + dictWord{7, 0, 181}, + dictWord{8, 0, 276}, + dictWord{8, 0, 377}, + dictWord{10, 0, 523}, + dictWord{ + 11, + 0, + 816, + }, + dictWord{12, 0, 455}, + dictWord{13, 0, 303}, + dictWord{142, 0, 135}, + dictWord{135, 0, 281}, + dictWord{4, 0, 1}, + dictWord{7, 0, 1143}, + dictWord{7, 0, 1463}, + dictWord{8, 0, 61}, + dictWord{9, 0, 207}, + dictWord{9, 0, 390}, + dictWord{9, 0, 467}, + dictWord{139, 0, 836}, + dictWord{6, 11, 392}, + dictWord{7, 11, 65}, + dictWord{ + 135, + 11, + 2019, + }, + dictWord{132, 10, 667}, + dictWord{4, 0, 723}, + dictWord{5, 0, 895}, + dictWord{7, 0, 1031}, + dictWord{8, 0, 199}, + dictWord{8, 0, 340}, + dictWord{9, 0, 153}, + dictWord{9, 0, 215}, + dictWord{10, 0, 21}, + dictWord{10, 0, 59}, + dictWord{10, 0, 80}, + dictWord{10, 0, 224}, + dictWord{10, 0, 838}, + dictWord{11, 0, 229}, + dictWord{ + 11, + 0, + 652, + }, + dictWord{12, 0, 192}, + dictWord{13, 0, 146}, + dictWord{142, 0, 91}, + dictWord{132, 0, 295}, + dictWord{137, 0, 51}, + dictWord{9, 11, 222}, + dictWord{ + 10, + 11, + 43, + }, + dictWord{139, 11, 900}, + dictWord{5, 0, 309}, + dictWord{140, 0, 211}, + dictWord{5, 0, 125}, + dictWord{8, 0, 77}, + dictWord{138, 0, 15}, + dictWord{136, 11, 604}, + dictWord{138, 0, 789}, + dictWord{5, 0, 173}, + dictWord{4, 10, 39}, + dictWord{7, 10, 1843}, + dictWord{8, 10, 407}, + dictWord{11, 10, 144}, + dictWord{140, 10, 523}, + dictWord{138, 11, 265}, + dictWord{133, 0, 439}, + dictWord{132, 10, 510}, + dictWord{7, 0, 648}, + dictWord{7, 0, 874}, + dictWord{11, 0, 164}, + dictWord{12, 0, 76}, + dictWord{18, 0, 9}, + dictWord{7, 10, 1980}, + dictWord{10, 10, 487}, + dictWord{138, 10, 809}, + dictWord{12, 0, 111}, + dictWord{14, 0, 294}, + dictWord{19, 0, 45}, + dictWord{13, 10, 260}, + dictWord{146, 10, 63}, + dictWord{133, 11, 549}, + dictWord{134, 10, 570}, + dictWord{4, 0, 8}, + dictWord{7, 0, 1152}, + dictWord{7, 0, 1153}, + dictWord{7, 0, 1715}, + dictWord{9, 0, 374}, + dictWord{10, 0, 478}, + dictWord{139, 0, 648}, + dictWord{135, 0, 1099}, + dictWord{5, 0, 575}, + dictWord{6, 0, 354}, + dictWord{ + 135, + 0, + 701, + }, + dictWord{7, 11, 36}, + dictWord{8, 11, 201}, + dictWord{136, 11, 605}, + dictWord{4, 10, 787}, + dictWord{136, 11, 156}, + dictWord{6, 0, 518}, + dictWord{ + 149, + 11, + 13, + }, + dictWord{140, 11, 224}, + dictWord{134, 0, 702}, + dictWord{132, 10, 516}, + dictWord{5, 11, 724}, + dictWord{10, 11, 305}, + dictWord{11, 11, 151}, + dictWord{12, 11, 33}, + dictWord{12, 11, 121}, + dictWord{12, 11, 381}, + dictWord{17, 11, 3}, + dictWord{17, 11, 27}, + dictWord{17, 11, 78}, + dictWord{18, 11, 18}, + dictWord{19, 11, 54}, + dictWord{149, 11, 5}, + dictWord{8, 0, 87}, + dictWord{4, 11, 523}, + dictWord{5, 11, 638}, + dictWord{11, 10, 887}, + dictWord{14, 10, 365}, + dictWord{ + 142, + 10, + 375, + }, + dictWord{138, 0, 438}, + dictWord{136, 10, 821}, + dictWord{135, 11, 1908}, + dictWord{6, 11, 242}, + dictWord{7, 11, 227}, + dictWord{7, 11, 1581}, + dictWord{8, 11, 104}, + dictWord{9, 11, 113}, + dictWord{9, 11, 220}, + dictWord{9, 11, 427}, + dictWord{10, 11, 74}, + dictWord{10, 11, 239}, + dictWord{11, 11, 579}, + dictWord{11, 11, 1023}, + dictWord{13, 11, 4}, + dictWord{13, 11, 204}, + dictWord{13, 11, 316}, + dictWord{18, 11, 95}, + dictWord{148, 11, 86}, + dictWord{4, 0, 69}, + dictWord{5, 0, 122}, + dictWord{5, 0, 849}, + dictWord{6, 0, 1633}, + dictWord{9, 0, 656}, + dictWord{138, 0, 464}, + dictWord{7, 0, 1802}, + dictWord{4, 10, 10}, + dictWord{ + 139, + 10, + 786, + }, + dictWord{135, 11, 861}, + dictWord{139, 0, 499}, + dictWord{7, 0, 476}, + dictWord{7, 0, 1592}, + dictWord{138, 0, 87}, + dictWord{133, 10, 684}, + dictWord{ + 4, + 0, + 840, + }, + dictWord{134, 10, 27}, + dictWord{142, 0, 283}, + dictWord{6, 0, 1620}, + dictWord{7, 11, 1328}, + dictWord{136, 11, 494}, + dictWord{5, 0, 859}, + dictWord{ + 7, + 0, + 1160, + }, + dictWord{8, 0, 107}, + dictWord{9, 0, 291}, + dictWord{9, 0, 439}, + dictWord{10, 0, 663}, + dictWord{11, 0, 609}, + dictWord{140, 0, 197}, + dictWord{ + 7, + 11, + 1306, + }, + dictWord{8, 11, 505}, + dictWord{9, 11, 482}, + dictWord{10, 11, 126}, + dictWord{11, 11, 225}, + dictWord{12, 11, 347}, + dictWord{12, 11, 449}, + dictWord{ + 13, + 11, + 19, + }, + dictWord{142, 11, 218}, + dictWord{5, 11, 268}, + dictWord{10, 11, 764}, + dictWord{12, 11, 120}, + dictWord{13, 11, 39}, + dictWord{145, 11, 127}, + dictWord{145, 10, 56}, + dictWord{7, 11, 1672}, + dictWord{10, 11, 472}, + dictWord{11, 11, 189}, + dictWord{143, 11, 51}, + dictWord{6, 10, 342}, + dictWord{6, 10, 496}, + dictWord{8, 10, 275}, + dictWord{137, 10, 206}, + dictWord{133, 0, 600}, + dictWord{4, 0, 117}, + dictWord{6, 0, 372}, + dictWord{7, 0, 1905}, + dictWord{142, 0, 323}, + dictWord{4, 10, 909}, + dictWord{5, 10, 940}, + dictWord{135, 11, 1471}, + dictWord{132, 10, 891}, + dictWord{4, 0, 722}, + dictWord{139, 0, 471}, + dictWord{4, 11, 384}, + dictWord{135, 11, 1022}, + dictWord{132, 10, 687}, + dictWord{9, 0, 5}, + dictWord{12, 0, 216}, + dictWord{12, 0, 294}, + dictWord{12, 0, 298}, + dictWord{12, 0, 400}, + dictWord{12, 0, 518}, + dictWord{13, 0, 229}, + dictWord{143, 0, 139}, + dictWord{135, 11, 1703}, + dictWord{7, 11, 1602}, + dictWord{10, 11, 698}, + dictWord{ + 12, + 11, + 212, + }, + dictWord{141, 11, 307}, + dictWord{6, 10, 41}, + dictWord{141, 10, 160}, + dictWord{135, 11, 1077}, + dictWord{9, 11, 159}, + dictWord{11, 11, 28}, + dictWord{140, 11, 603}, + dictWord{4, 0, 514}, + dictWord{7, 0, 1304}, + dictWord{138, 0, 477}, + dictWord{134, 0, 1774}, + dictWord{9, 0, 88}, + dictWord{139, 0, 270}, + dictWord{5, 0, 12}, + dictWord{7, 0, 375}, + dictWord{9, 0, 438}, + dictWord{134, 10, 1718}, + dictWord{132, 11, 515}, + dictWord{136, 10, 778}, + dictWord{8, 11, 632}, + dictWord{8, 11, 697}, + dictWord{137, 11, 854}, + dictWord{6, 0, 362}, + dictWord{6, 0, 997}, + dictWord{146, 0, 51}, + dictWord{7, 0, 816}, + dictWord{7, 0, 1241}, + dictWord{ + 9, + 0, + 283, + }, + dictWord{9, 0, 520}, + dictWord{10, 0, 213}, + dictWord{10, 0, 307}, + dictWord{10, 0, 463}, + dictWord{10, 0, 671}, + dictWord{10, 0, 746}, + dictWord{11, 0, 401}, + dictWord{11, 0, 794}, + dictWord{12, 0, 517}, + dictWord{18, 0, 107}, + dictWord{147, 0, 115}, + dictWord{133, 10, 115}, + dictWord{150, 11, 28}, + dictWord{4, 11, 136}, + dictWord{133, 11, 551}, + dictWord{142, 10, 314}, + dictWord{132, 0, 258}, + dictWord{6, 0, 22}, + dictWord{7, 0, 903}, + dictWord{7, 0, 1963}, + dictWord{8, 0, 639}, + dictWord{138, 0, 577}, + dictWord{5, 0, 681}, + dictWord{8, 0, 782}, + dictWord{13, 0, 130}, + dictWord{17, 0, 84}, + dictWord{5, 10, 193}, + dictWord{140, 10, 178}, + dictWord{ + 9, + 11, + 17, + }, + dictWord{138, 11, 291}, + dictWord{7, 11, 1287}, + dictWord{9, 11, 44}, + dictWord{10, 11, 552}, + dictWord{10, 11, 642}, + dictWord{11, 11, 839}, + dictWord{12, 11, 274}, + dictWord{12, 11, 275}, + dictWord{12, 11, 372}, + dictWord{13, 11, 91}, + dictWord{142, 11, 125}, + dictWord{135, 10, 174}, + dictWord{4, 0, 664}, + dictWord{5, 0, 804}, + dictWord{139, 0, 1013}, + dictWord{134, 0, 942}, + dictWord{6, 0, 1349}, + dictWord{6, 0, 1353}, + dictWord{6, 0, 1450}, + dictWord{7, 11, 1518}, + dictWord{139, 11, 694}, + dictWord{11, 0, 356}, + dictWord{4, 10, 122}, + dictWord{5, 10, 796}, + dictWord{5, 10, 952}, + dictWord{6, 10, 1660}, + dictWord{ + 6, + 10, + 1671, + }, + dictWord{8, 10, 567}, + dictWord{9, 10, 687}, + dictWord{9, 10, 742}, + dictWord{10, 10, 686}, + dictWord{11, 10, 682}, + dictWord{140, 10, 281}, + dictWord{ + 5, + 0, + 32, + }, + dictWord{6, 11, 147}, + dictWord{7, 11, 886}, + dictWord{9, 11, 753}, + dictWord{138, 11, 268}, + dictWord{5, 10, 179}, + dictWord{7, 10, 1095}, + dictWord{ + 135, + 10, + 1213, + }, + dictWord{4, 10, 66}, + dictWord{7, 10, 722}, + dictWord{135, 10, 904}, + dictWord{135, 10, 352}, + dictWord{9, 11, 245}, + dictWord{138, 11, 137}, + dictWord{4, 0, 289}, + dictWord{7, 0, 629}, + dictWord{7, 0, 1698}, + dictWord{7, 0, 1711}, + dictWord{12, 0, 215}, + dictWord{133, 11, 414}, + dictWord{6, 0, 1975}, + dictWord{135, 11, 1762}, + dictWord{6, 0, 450}, + dictWord{136, 0, 109}, + dictWord{141, 10, 35}, + dictWord{134, 11, 599}, + dictWord{136, 0, 705}, + dictWord{ + 133, + 0, + 664, + }, + dictWord{134, 11, 1749}, + dictWord{11, 11, 402}, + dictWord{12, 11, 109}, + dictWord{12, 11, 431}, + dictWord{13, 11, 179}, + dictWord{13, 11, 206}, + dictWord{14, 11, 175}, + dictWord{14, 11, 217}, + dictWord{16, 11, 3}, + dictWord{148, 11, 53}, + dictWord{135, 0, 1238}, + dictWord{134, 11, 1627}, + dictWord{ + 132, + 11, + 488, + }, + dictWord{13, 0, 318}, + dictWord{10, 10, 592}, + dictWord{10, 10, 753}, + dictWord{12, 10, 317}, + dictWord{12, 10, 355}, + dictWord{12, 10, 465}, + dictWord{ + 12, + 10, + 469, + }, + dictWord{12, 10, 560}, + dictWord{140, 10, 578}, + dictWord{133, 10, 564}, + dictWord{132, 11, 83}, + dictWord{140, 11, 676}, + dictWord{6, 0, 1872}, + dictWord{6, 0, 1906}, + dictWord{6, 0, 1907}, + dictWord{9, 0, 934}, + dictWord{9, 0, 956}, + dictWord{9, 0, 960}, + dictWord{9, 0, 996}, + dictWord{12, 0, 794}, + dictWord{ + 12, + 0, + 876, + }, + dictWord{12, 0, 880}, + dictWord{12, 0, 918}, + dictWord{15, 0, 230}, + dictWord{18, 0, 234}, + dictWord{18, 0, 238}, + dictWord{21, 0, 38}, + dictWord{149, 0, 62}, + dictWord{134, 10, 556}, + dictWord{134, 11, 278}, + dictWord{137, 0, 103}, + dictWord{7, 10, 544}, + dictWord{8, 10, 719}, + dictWord{138, 10, 61}, + dictWord{ + 4, + 10, + 5, + }, + dictWord{5, 10, 498}, + dictWord{8, 10, 637}, + dictWord{137, 10, 521}, + dictWord{7, 0, 777}, + dictWord{12, 0, 229}, + dictWord{12, 0, 239}, + dictWord{15, 0, 12}, + dictWord{12, 11, 229}, + dictWord{12, 11, 239}, + dictWord{143, 11, 12}, + dictWord{6, 0, 26}, + dictWord{7, 11, 388}, + dictWord{7, 11, 644}, + dictWord{139, 11, 781}, + dictWord{7, 11, 229}, + dictWord{8, 11, 59}, + dictWord{9, 11, 190}, + dictWord{9, 11, 257}, + dictWord{10, 11, 378}, + dictWord{140, 11, 191}, + dictWord{133, 10, 927}, + dictWord{135, 10, 1441}, + dictWord{4, 10, 893}, + dictWord{5, 10, 780}, + dictWord{133, 10, 893}, + dictWord{4, 0, 414}, + dictWord{5, 0, 467}, + dictWord{9, 0, 654}, + dictWord{10, 0, 451}, + dictWord{12, 0, 59}, + dictWord{141, 0, 375}, + dictWord{142, 0, 173}, + dictWord{135, 0, 17}, + dictWord{7, 0, 1350}, + dictWord{133, 10, 238}, + dictWord{135, 0, 955}, + dictWord{4, 0, 960}, + dictWord{10, 0, 887}, + dictWord{12, 0, 753}, + dictWord{18, 0, 161}, + dictWord{18, 0, 162}, + dictWord{152, 0, 19}, + dictWord{136, 11, 344}, + dictWord{6, 10, 1729}, + dictWord{137, 11, 288}, + dictWord{132, 11, 660}, + dictWord{4, 0, 217}, + dictWord{5, 0, 710}, + dictWord{7, 0, 760}, + dictWord{7, 0, 1926}, + dictWord{9, 0, 428}, + dictWord{9, 0, 708}, + dictWord{10, 0, 254}, + dictWord{10, 0, 296}, + dictWord{10, 0, 720}, + dictWord{11, 0, 109}, + dictWord{ + 11, + 0, + 255, + }, + dictWord{12, 0, 165}, + dictWord{12, 0, 315}, + dictWord{13, 0, 107}, + dictWord{13, 0, 203}, + dictWord{14, 0, 54}, + dictWord{14, 0, 99}, + dictWord{14, 0, 114}, + dictWord{14, 0, 388}, + dictWord{16, 0, 85}, + dictWord{17, 0, 9}, + dictWord{17, 0, 33}, + dictWord{20, 0, 25}, + dictWord{20, 0, 28}, + dictWord{20, 0, 29}, + dictWord{21, 0, 9}, + dictWord{21, 0, 10}, + dictWord{21, 0, 34}, + dictWord{22, 0, 17}, + dictWord{4, 10, 60}, + dictWord{7, 10, 1800}, + dictWord{8, 10, 314}, + dictWord{9, 10, 700}, + dictWord{ + 139, + 10, + 487, + }, + dictWord{7, 11, 1035}, + dictWord{138, 11, 737}, + dictWord{7, 11, 690}, + dictWord{9, 11, 217}, + dictWord{9, 11, 587}, + dictWord{140, 11, 521}, + dictWord{6, 0, 919}, + dictWord{7, 11, 706}, + dictWord{7, 11, 1058}, + dictWord{138, 11, 538}, + dictWord{7, 10, 1853}, + dictWord{138, 10, 437}, + dictWord{ + 136, + 10, + 419, + }, + dictWord{6, 0, 280}, + dictWord{10, 0, 502}, + dictWord{11, 0, 344}, + dictWord{140, 0, 38}, + dictWord{5, 0, 45}, + dictWord{7, 0, 1161}, + dictWord{11, 0, 448}, + dictWord{11, 0, 880}, + dictWord{13, 0, 139}, + dictWord{13, 0, 407}, + dictWord{15, 0, 16}, + dictWord{17, 0, 95}, + dictWord{18, 0, 66}, + dictWord{18, 0, 88}, + dictWord{ + 18, + 0, + 123, + }, + dictWord{149, 0, 7}, + dictWord{11, 11, 92}, + dictWord{11, 11, 196}, + dictWord{11, 11, 409}, + dictWord{11, 11, 450}, + dictWord{11, 11, 666}, + dictWord{ + 11, + 11, + 777, + }, + dictWord{12, 11, 262}, + dictWord{13, 11, 385}, + dictWord{13, 11, 393}, + dictWord{15, 11, 115}, + dictWord{16, 11, 45}, + dictWord{145, 11, 82}, + dictWord{136, 0, 777}, + dictWord{134, 11, 1744}, + dictWord{4, 0, 410}, + dictWord{7, 0, 521}, + dictWord{133, 10, 828}, + dictWord{134, 0, 673}, + dictWord{7, 0, 1110}, + dictWord{7, 0, 1778}, + dictWord{7, 10, 176}, + dictWord{135, 10, 178}, + dictWord{5, 10, 806}, + dictWord{7, 11, 268}, + dictWord{7, 10, 1976}, + dictWord{ + 136, + 11, + 569, + }, + dictWord{4, 11, 733}, + dictWord{9, 11, 194}, + dictWord{10, 11, 92}, + dictWord{11, 11, 198}, + dictWord{12, 11, 84}, + dictWord{12, 11, 87}, + dictWord{ + 13, + 11, + 128, + }, + dictWord{144, 11, 74}, + dictWord{5, 0, 341}, + dictWord{7, 0, 1129}, + dictWord{11, 0, 414}, + dictWord{4, 10, 51}, + dictWord{6, 10, 4}, + dictWord{7, 10, 591}, + dictWord{7, 10, 849}, + dictWord{7, 10, 951}, + dictWord{7, 10, 1613}, + dictWord{7, 10, 1760}, + dictWord{7, 10, 1988}, + dictWord{9, 10, 434}, + dictWord{10, 10, 754}, + dictWord{11, 10, 25}, + dictWord{139, 10, 37}, + dictWord{133, 10, 902}, + dictWord{135, 10, 928}, + dictWord{135, 0, 787}, + dictWord{132, 0, 436}, + dictWord{ + 134, + 10, + 270, + }, + dictWord{7, 0, 1587}, + dictWord{135, 0, 1707}, + dictWord{6, 0, 377}, + dictWord{7, 0, 1025}, + dictWord{9, 0, 613}, + dictWord{145, 0, 104}, + dictWord{ + 7, + 11, + 982, + }, + dictWord{7, 11, 1361}, + dictWord{10, 11, 32}, + dictWord{143, 11, 56}, + dictWord{139, 0, 96}, + dictWord{132, 0, 451}, + dictWord{132, 10, 416}, + dictWord{ + 142, + 10, + 372, + }, + dictWord{5, 10, 152}, + dictWord{5, 10, 197}, + dictWord{7, 11, 306}, + dictWord{7, 10, 340}, + dictWord{7, 10, 867}, + dictWord{10, 10, 548}, + dictWord{ + 10, + 10, + 581, + }, + dictWord{11, 10, 6}, + dictWord{12, 10, 3}, + dictWord{12, 10, 19}, + dictWord{14, 10, 110}, + dictWord{142, 10, 289}, + dictWord{134, 0, 680}, + dictWord{ + 134, + 11, + 609, + }, + dictWord{7, 0, 483}, + dictWord{7, 10, 190}, + dictWord{8, 10, 28}, + dictWord{8, 10, 141}, + dictWord{8, 10, 444}, + dictWord{8, 10, 811}, + dictWord{ + 9, + 10, + 468, + }, + dictWord{11, 10, 334}, + dictWord{12, 10, 24}, + dictWord{12, 10, 386}, + dictWord{140, 10, 576}, + dictWord{10, 0, 916}, + dictWord{133, 10, 757}, + dictWord{ + 5, + 10, + 721, + }, + dictWord{135, 10, 1553}, + dictWord{133, 11, 178}, + dictWord{134, 0, 937}, + dictWord{132, 10, 898}, + dictWord{133, 0, 739}, + dictWord{ + 147, + 0, + 82, + }, + dictWord{135, 0, 663}, + dictWord{146, 0, 128}, + dictWord{5, 10, 277}, + dictWord{141, 10, 247}, + dictWord{134, 0, 1087}, + dictWord{132, 10, 435}, + dictWord{ + 6, + 11, + 381, + }, + dictWord{7, 11, 645}, + dictWord{7, 11, 694}, + dictWord{136, 11, 546}, + dictWord{7, 0, 503}, + dictWord{135, 0, 1885}, + dictWord{6, 0, 1965}, + dictWord{ + 8, + 0, + 925, + }, + dictWord{138, 0, 955}, + dictWord{4, 0, 113}, + dictWord{5, 0, 163}, + dictWord{5, 0, 735}, + dictWord{7, 0, 1009}, + dictWord{9, 0, 9}, + dictWord{9, 0, 771}, + dictWord{12, 0, 90}, + dictWord{13, 0, 138}, + dictWord{13, 0, 410}, + dictWord{143, 0, 128}, + dictWord{4, 0, 324}, + dictWord{138, 0, 104}, + dictWord{7, 0, 460}, + dictWord{ + 5, + 10, + 265, + }, + dictWord{134, 10, 212}, + dictWord{133, 11, 105}, + dictWord{7, 11, 261}, + dictWord{7, 11, 1107}, + dictWord{7, 11, 1115}, + dictWord{7, 11, 1354}, + dictWord{7, 11, 1588}, + dictWord{7, 11, 1705}, + dictWord{7, 11, 1902}, + dictWord{9, 11, 465}, + dictWord{10, 11, 248}, + dictWord{10, 11, 349}, + dictWord{10, 11, 647}, + dictWord{11, 11, 527}, + dictWord{11, 11, 660}, + dictWord{11, 11, 669}, + dictWord{12, 11, 529}, + dictWord{141, 11, 305}, + dictWord{5, 11, 438}, + dictWord{ + 9, + 11, + 694, + }, + dictWord{12, 11, 627}, + dictWord{141, 11, 210}, + dictWord{152, 11, 11}, + dictWord{4, 0, 935}, + dictWord{133, 0, 823}, + dictWord{132, 10, 702}, + dictWord{ + 5, + 0, + 269, + }, + dictWord{7, 0, 434}, + dictWord{7, 0, 891}, + dictWord{8, 0, 339}, + dictWord{9, 0, 702}, + dictWord{11, 0, 594}, + dictWord{11, 0, 718}, + dictWord{17, 0, 100}, + dictWord{5, 10, 808}, + dictWord{135, 10, 2045}, + dictWord{7, 0, 1014}, + dictWord{9, 0, 485}, + dictWord{141, 0, 264}, + dictWord{134, 0, 1713}, + dictWord{7, 0, 1810}, + dictWord{11, 0, 866}, + dictWord{12, 0, 103}, + dictWord{13, 0, 495}, + dictWord{140, 11, 233}, + dictWord{4, 0, 423}, + dictWord{10, 0, 949}, + dictWord{138, 0, 1013}, + dictWord{135, 0, 900}, + dictWord{8, 11, 25}, + dictWord{138, 11, 826}, + dictWord{5, 10, 166}, + dictWord{8, 10, 739}, + dictWord{140, 10, 511}, + dictWord{ + 134, + 0, + 2018, + }, + dictWord{7, 11, 1270}, + dictWord{139, 11, 612}, + dictWord{4, 10, 119}, + dictWord{5, 10, 170}, + dictWord{5, 10, 447}, + dictWord{7, 10, 1708}, + dictWord{ + 7, + 10, + 1889, + }, + dictWord{9, 10, 357}, + dictWord{9, 10, 719}, + dictWord{12, 10, 486}, + dictWord{140, 10, 596}, + dictWord{12, 0, 574}, + dictWord{140, 11, 574}, + dictWord{132, 11, 308}, + dictWord{6, 0, 964}, + dictWord{6, 0, 1206}, + dictWord{134, 0, 1302}, + dictWord{4, 10, 450}, + dictWord{135, 10, 1158}, + dictWord{ + 135, + 11, + 150, + }, + dictWord{136, 11, 649}, + dictWord{14, 0, 213}, + dictWord{148, 0, 38}, + dictWord{9, 11, 45}, + dictWord{9, 11, 311}, + dictWord{141, 11, 42}, + dictWord{ + 134, + 11, + 521, + }, + dictWord{7, 10, 1375}, + dictWord{7, 10, 1466}, + dictWord{138, 10, 331}, + dictWord{132, 10, 754}, + dictWord{5, 11, 339}, + dictWord{7, 11, 1442}, + dictWord{14, 11, 3}, + dictWord{15, 11, 41}, + dictWord{147, 11, 66}, + dictWord{136, 11, 378}, + dictWord{134, 0, 1022}, + dictWord{5, 10, 850}, + dictWord{136, 10, 799}, + dictWord{142, 0, 143}, + dictWord{135, 0, 2029}, + dictWord{134, 11, 1628}, + dictWord{8, 0, 523}, + dictWord{150, 0, 34}, + dictWord{5, 0, 625}, + dictWord{ + 135, + 0, + 1617, + }, + dictWord{7, 0, 275}, + dictWord{7, 10, 238}, + dictWord{7, 10, 2033}, + dictWord{8, 10, 120}, + dictWord{8, 10, 188}, + dictWord{8, 10, 659}, + dictWord{ + 9, + 10, + 598, + }, + dictWord{10, 10, 466}, + dictWord{12, 10, 342}, + dictWord{12, 10, 588}, + dictWord{13, 10, 503}, + dictWord{14, 10, 246}, + dictWord{143, 10, 92}, + dictWord{ + 7, + 0, + 37, + }, + dictWord{8, 0, 425}, + dictWord{8, 0, 693}, + dictWord{9, 0, 720}, + dictWord{10, 0, 380}, + dictWord{10, 0, 638}, + dictWord{11, 0, 273}, + dictWord{11, 0, 473}, + dictWord{12, 0, 61}, + dictWord{143, 0, 43}, + dictWord{135, 11, 829}, + dictWord{135, 0, 1943}, + dictWord{132, 0, 765}, + dictWord{5, 11, 486}, + dictWord{ + 135, + 11, + 1349, + }, + dictWord{7, 11, 1635}, + dictWord{8, 11, 17}, + dictWord{10, 11, 217}, + dictWord{138, 11, 295}, + dictWord{4, 10, 201}, + dictWord{7, 10, 1744}, + dictWord{ + 8, + 10, + 602, + }, + dictWord{11, 10, 247}, + dictWord{11, 10, 826}, + dictWord{145, 10, 65}, + dictWord{138, 11, 558}, + dictWord{11, 0, 551}, + dictWord{142, 0, 159}, + dictWord{8, 10, 164}, + dictWord{146, 10, 62}, + dictWord{139, 11, 176}, + dictWord{132, 0, 168}, + dictWord{136, 0, 1010}, + dictWord{134, 0, 1994}, + dictWord{ + 135, + 0, + 91, + }, + dictWord{138, 0, 532}, + dictWord{135, 10, 1243}, + dictWord{135, 0, 1884}, + dictWord{132, 10, 907}, + dictWord{5, 10, 100}, + dictWord{10, 10, 329}, + dictWord{12, 10, 416}, + dictWord{149, 10, 29}, + dictWord{134, 11, 447}, + dictWord{132, 10, 176}, + dictWord{5, 10, 636}, + dictWord{5, 10, 998}, + dictWord{7, 10, 9}, + dictWord{7, 10, 1508}, + dictWord{8, 10, 26}, + dictWord{9, 10, 317}, + dictWord{9, 10, 358}, + dictWord{10, 10, 210}, + dictWord{10, 10, 292}, + dictWord{10, 10, 533}, + dictWord{11, 10, 555}, + dictWord{12, 10, 526}, + dictWord{12, 10, 607}, + dictWord{13, 10, 263}, + dictWord{13, 10, 459}, + dictWord{142, 10, 271}, + dictWord{ + 4, + 11, + 609, + }, + dictWord{135, 11, 756}, + dictWord{6, 0, 15}, + dictWord{7, 0, 70}, + dictWord{10, 0, 240}, + dictWord{147, 0, 93}, + dictWord{4, 11, 930}, + dictWord{133, 11, 947}, + dictWord{134, 0, 1227}, + dictWord{134, 0, 1534}, + dictWord{133, 11, 939}, + dictWord{133, 11, 962}, + dictWord{5, 11, 651}, + dictWord{8, 11, 170}, + dictWord{ + 9, + 11, + 61, + }, + dictWord{9, 11, 63}, + dictWord{10, 11, 23}, + dictWord{10, 11, 37}, + dictWord{10, 11, 834}, + dictWord{11, 11, 4}, + dictWord{11, 11, 187}, + dictWord{ + 11, + 11, + 281, + }, + dictWord{11, 11, 503}, + dictWord{11, 11, 677}, + dictWord{12, 11, 96}, + dictWord{12, 11, 130}, + dictWord{12, 11, 244}, + dictWord{14, 11, 5}, + dictWord{ + 14, + 11, + 40, + }, + dictWord{14, 11, 162}, + dictWord{14, 11, 202}, + dictWord{146, 11, 133}, + dictWord{4, 11, 406}, + dictWord{5, 11, 579}, + dictWord{12, 11, 492}, + dictWord{ + 150, + 11, + 15, + }, + dictWord{139, 0, 392}, + dictWord{6, 10, 610}, + dictWord{10, 10, 127}, + dictWord{141, 10, 27}, + dictWord{7, 0, 655}, + dictWord{7, 0, 1844}, + dictWord{ + 136, + 10, + 119, + }, + dictWord{4, 0, 145}, + dictWord{6, 0, 176}, + dictWord{7, 0, 395}, + dictWord{137, 0, 562}, + dictWord{132, 0, 501}, + dictWord{140, 11, 145}, + dictWord{ + 136, + 0, + 1019, + }, + dictWord{134, 0, 509}, + dictWord{139, 0, 267}, + dictWord{6, 11, 17}, + dictWord{7, 11, 16}, + dictWord{7, 11, 1001}, + dictWord{7, 11, 1982}, + dictWord{ + 9, + 11, + 886, + }, + dictWord{10, 11, 489}, + dictWord{10, 11, 800}, + dictWord{11, 11, 782}, + dictWord{12, 11, 320}, + dictWord{13, 11, 467}, + dictWord{14, 11, 145}, + dictWord{14, 11, 387}, + dictWord{143, 11, 119}, + dictWord{145, 11, 17}, + dictWord{6, 0, 1099}, + dictWord{133, 11, 458}, + dictWord{7, 11, 1983}, + dictWord{8, 11, 0}, + dictWord{8, 11, 171}, + dictWord{9, 11, 120}, + dictWord{9, 11, 732}, + dictWord{10, 11, 473}, + dictWord{11, 11, 656}, + dictWord{11, 11, 998}, + dictWord{18, 11, 0}, + dictWord{18, 11, 2}, + dictWord{147, 11, 21}, + dictWord{12, 11, 427}, + dictWord{146, 11, 38}, + dictWord{10, 0, 948}, + dictWord{138, 0, 968}, + dictWord{7, 10, 126}, + dictWord{136, 10, 84}, + dictWord{136, 10, 790}, + dictWord{4, 0, 114}, + dictWord{9, 0, 492}, + dictWord{13, 0, 462}, + dictWord{142, 0, 215}, + dictWord{6, 10, 64}, + dictWord{12, 10, 377}, + dictWord{141, 10, 309}, + dictWord{4, 0, 77}, + dictWord{5, 0, 361}, + dictWord{6, 0, 139}, + dictWord{6, 0, 401}, + dictWord{6, 0, 404}, + dictWord{ + 7, + 0, + 413, + }, + dictWord{7, 0, 715}, + dictWord{7, 0, 1716}, + dictWord{11, 0, 279}, + dictWord{12, 0, 179}, + dictWord{12, 0, 258}, + dictWord{13, 0, 244}, + dictWord{142, 0, 358}, + dictWord{134, 0, 1717}, + dictWord{7, 0, 772}, + dictWord{7, 0, 1061}, + dictWord{7, 0, 1647}, + dictWord{8, 0, 82}, + dictWord{11, 0, 250}, + dictWord{11, 0, 607}, + dictWord{12, 0, 311}, + dictWord{12, 0, 420}, + dictWord{13, 0, 184}, + dictWord{13, 0, 367}, + dictWord{7, 10, 1104}, + dictWord{11, 10, 269}, + dictWord{11, 10, 539}, + dictWord{11, 10, 627}, + dictWord{11, 10, 706}, + dictWord{11, 10, 975}, + dictWord{12, 10, 248}, + dictWord{12, 10, 434}, + dictWord{12, 10, 600}, + dictWord{ + 12, + 10, + 622, + }, + dictWord{13, 10, 297}, + dictWord{13, 10, 485}, + dictWord{14, 10, 69}, + dictWord{14, 10, 409}, + dictWord{143, 10, 108}, + dictWord{135, 0, 724}, + dictWord{ + 4, + 11, + 512, + }, + dictWord{4, 11, 519}, + dictWord{133, 11, 342}, + dictWord{134, 0, 1133}, + dictWord{145, 11, 29}, + dictWord{11, 10, 977}, + dictWord{141, 10, 507}, + dictWord{6, 0, 841}, + dictWord{6, 0, 1042}, + dictWord{6, 0, 1194}, + dictWord{10, 0, 993}, + dictWord{140, 0, 1021}, + dictWord{6, 11, 31}, + dictWord{7, 11, 491}, + dictWord{7, 11, 530}, + dictWord{8, 11, 592}, + dictWord{9, 10, 34}, + dictWord{11, 11, 53}, + dictWord{11, 10, 484}, + dictWord{11, 11, 779}, + dictWord{12, 11, 167}, + dictWord{12, 11, 411}, + dictWord{14, 11, 14}, + dictWord{14, 11, 136}, + dictWord{15, 11, 72}, + dictWord{16, 11, 17}, + dictWord{144, 11, 72}, + dictWord{4, 0, 1021}, + dictWord{6, 0, 2037}, + dictWord{133, 11, 907}, + dictWord{7, 0, 373}, + dictWord{8, 0, 335}, + dictWord{8, 0, 596}, + dictWord{9, 0, 488}, + dictWord{6, 10, 1700}, + dictWord{ + 7, + 10, + 293, + }, + dictWord{7, 10, 382}, + dictWord{7, 10, 1026}, + dictWord{7, 10, 1087}, + dictWord{7, 10, 2027}, + dictWord{8, 10, 252}, + dictWord{8, 10, 727}, + dictWord{ + 8, + 10, + 729, + }, + dictWord{9, 10, 30}, + dictWord{9, 10, 199}, + dictWord{9, 10, 231}, + dictWord{9, 10, 251}, + dictWord{9, 10, 334}, + dictWord{9, 10, 361}, + dictWord{9, 10, 712}, + dictWord{10, 10, 55}, + dictWord{10, 10, 60}, + dictWord{10, 10, 232}, + dictWord{10, 10, 332}, + dictWord{10, 10, 384}, + dictWord{10, 10, 396}, + dictWord{ + 10, + 10, + 504, + }, + dictWord{10, 10, 542}, + dictWord{10, 10, 652}, + dictWord{11, 10, 20}, + dictWord{11, 10, 48}, + dictWord{11, 10, 207}, + dictWord{11, 10, 291}, + dictWord{ + 11, + 10, + 298, + }, + dictWord{11, 10, 342}, + dictWord{11, 10, 365}, + dictWord{11, 10, 394}, + dictWord{11, 10, 620}, + dictWord{11, 10, 705}, + dictWord{11, 10, 1017}, + dictWord{12, 10, 123}, + dictWord{12, 10, 340}, + dictWord{12, 10, 406}, + dictWord{12, 10, 643}, + dictWord{13, 10, 61}, + dictWord{13, 10, 269}, + dictWord{ + 13, + 10, + 311, + }, + dictWord{13, 10, 319}, + dictWord{13, 10, 486}, + dictWord{14, 10, 234}, + dictWord{15, 10, 62}, + dictWord{15, 10, 85}, + dictWord{16, 10, 71}, + dictWord{ + 18, + 10, + 119, + }, + dictWord{148, 10, 105}, + dictWord{150, 0, 37}, + dictWord{4, 11, 208}, + dictWord{5, 11, 106}, + dictWord{6, 11, 531}, + dictWord{8, 11, 408}, + dictWord{ + 9, + 11, + 188, + }, + dictWord{138, 11, 572}, + dictWord{132, 0, 564}, + dictWord{6, 0, 513}, + dictWord{135, 0, 1052}, + dictWord{132, 0, 825}, + dictWord{9, 0, 899}, + dictWord{ + 140, + 11, + 441, + }, + dictWord{134, 0, 778}, + dictWord{133, 11, 379}, + dictWord{7, 0, 1417}, + dictWord{12, 0, 382}, + dictWord{17, 0, 48}, + dictWord{152, 0, 12}, + dictWord{ + 132, + 11, + 241, + }, + dictWord{7, 0, 1116}, + dictWord{6, 10, 379}, + dictWord{7, 10, 270}, + dictWord{8, 10, 176}, + dictWord{8, 10, 183}, + dictWord{9, 10, 432}, + dictWord{ + 9, + 10, + 661, + }, + dictWord{12, 10, 247}, + dictWord{12, 10, 617}, + dictWord{146, 10, 125}, + dictWord{5, 10, 792}, + dictWord{133, 10, 900}, + dictWord{6, 0, 545}, + dictWord{ + 7, + 0, + 565, + }, + dictWord{7, 0, 1669}, + dictWord{10, 0, 114}, + dictWord{11, 0, 642}, + dictWord{140, 0, 618}, + dictWord{133, 0, 5}, + dictWord{138, 11, 7}, + dictWord{ + 132, + 11, + 259, + }, + dictWord{135, 0, 192}, + dictWord{134, 0, 701}, + dictWord{136, 0, 763}, + dictWord{135, 10, 1979}, + dictWord{4, 10, 901}, + dictWord{133, 10, 776}, + dictWord{10, 0, 755}, + dictWord{147, 0, 29}, + dictWord{133, 0, 759}, + dictWord{4, 11, 173}, + dictWord{5, 11, 312}, + dictWord{5, 11, 512}, + dictWord{135, 11, 1285}, + dictWord{7, 11, 1603}, + dictWord{7, 11, 1691}, + dictWord{9, 11, 464}, + dictWord{11, 11, 195}, + dictWord{12, 11, 279}, + dictWord{12, 11, 448}, + dictWord{ + 14, + 11, + 11, + }, + dictWord{147, 11, 102}, + dictWord{7, 0, 370}, + dictWord{7, 0, 1007}, + dictWord{7, 0, 1177}, + dictWord{135, 0, 1565}, + dictWord{135, 0, 1237}, + dictWord{ + 4, + 0, + 87, + }, + dictWord{5, 0, 250}, + dictWord{141, 0, 298}, + dictWord{4, 11, 452}, + dictWord{5, 11, 583}, + dictWord{5, 11, 817}, + dictWord{6, 11, 433}, + dictWord{7, 11, 593}, + dictWord{7, 11, 720}, + dictWord{7, 11, 1378}, + dictWord{8, 11, 161}, + dictWord{9, 11, 284}, + dictWord{10, 11, 313}, + dictWord{139, 11, 886}, + dictWord{4, 11, 547}, + dictWord{135, 11, 1409}, + dictWord{136, 11, 722}, + dictWord{4, 10, 37}, + dictWord{5, 10, 334}, + dictWord{135, 10, 1253}, + dictWord{132, 10, 508}, + dictWord{ + 12, + 0, + 107, + }, + dictWord{146, 0, 31}, + dictWord{8, 11, 420}, + dictWord{139, 11, 193}, + dictWord{135, 0, 814}, + dictWord{135, 11, 409}, + dictWord{140, 0, 991}, + dictWord{4, 0, 57}, + dictWord{7, 0, 1195}, + dictWord{7, 0, 1438}, + dictWord{7, 0, 1548}, + dictWord{7, 0, 1835}, + dictWord{7, 0, 1904}, + dictWord{9, 0, 757}, + dictWord{ + 10, + 0, + 604, + }, + dictWord{139, 0, 519}, + dictWord{132, 0, 540}, + dictWord{138, 11, 308}, + dictWord{132, 10, 533}, + dictWord{136, 0, 608}, + dictWord{144, 11, 65}, + dictWord{4, 0, 1014}, + dictWord{134, 0, 2029}, + dictWord{4, 0, 209}, + dictWord{7, 0, 902}, + dictWord{5, 11, 1002}, + dictWord{136, 11, 745}, + dictWord{134, 0, 2030}, + dictWord{6, 0, 303}, + dictWord{7, 0, 335}, + dictWord{7, 0, 1437}, + dictWord{7, 0, 1668}, + dictWord{8, 0, 553}, + dictWord{8, 0, 652}, + dictWord{8, 0, 656}, + dictWord{ + 9, + 0, + 558, + }, + dictWord{11, 0, 743}, + dictWord{149, 0, 18}, + dictWord{5, 11, 575}, + dictWord{6, 11, 354}, + dictWord{135, 11, 701}, + dictWord{4, 11, 239}, + dictWord{ + 6, + 11, + 477, + }, + dictWord{7, 11, 1607}, + dictWord{11, 11, 68}, + dictWord{139, 11, 617}, + dictWord{132, 0, 559}, + dictWord{8, 0, 527}, + dictWord{18, 0, 60}, + dictWord{ + 147, + 0, + 24, + }, + dictWord{133, 10, 920}, + dictWord{138, 0, 511}, + dictWord{133, 0, 1017}, + dictWord{133, 0, 675}, + dictWord{138, 10, 391}, + dictWord{11, 0, 156}, + dictWord{135, 10, 1952}, + dictWord{138, 11, 369}, + dictWord{132, 11, 367}, + dictWord{133, 0, 709}, + dictWord{6, 0, 698}, + dictWord{134, 0, 887}, + dictWord{ + 142, + 10, + 126, + }, + dictWord{134, 0, 1745}, + dictWord{132, 10, 483}, + dictWord{13, 11, 299}, + dictWord{142, 11, 75}, + dictWord{133, 0, 714}, + dictWord{7, 0, 8}, + dictWord{ + 136, + 0, + 206, + }, + dictWord{138, 10, 480}, + dictWord{4, 11, 694}, + dictWord{9, 10, 495}, + dictWord{146, 10, 104}, + dictWord{7, 11, 1248}, + dictWord{11, 11, 621}, + dictWord{139, 11, 702}, + dictWord{140, 11, 687}, + dictWord{132, 0, 776}, + dictWord{139, 10, 1009}, + dictWord{135, 0, 1272}, + dictWord{134, 0, 1059}, + dictWord{ + 8, + 10, + 653, + }, + dictWord{13, 10, 93}, + dictWord{147, 10, 14}, + dictWord{135, 11, 213}, + dictWord{136, 0, 406}, + dictWord{133, 10, 172}, + dictWord{132, 0, 947}, + dictWord{8, 0, 175}, + dictWord{10, 0, 168}, + dictWord{138, 0, 573}, + dictWord{132, 0, 870}, + dictWord{6, 0, 1567}, + dictWord{151, 11, 28}, + dictWord{ + 134, + 11, + 472, + }, + dictWord{5, 10, 260}, + dictWord{136, 11, 132}, + dictWord{4, 11, 751}, + dictWord{11, 11, 390}, + dictWord{140, 11, 32}, + dictWord{4, 11, 409}, + dictWord{ + 133, + 11, + 78, + }, + dictWord{12, 0, 554}, + dictWord{6, 11, 473}, + dictWord{145, 11, 105}, + dictWord{133, 0, 784}, + dictWord{8, 0, 908}, + dictWord{136, 11, 306}, + dictWord{139, 0, 882}, + dictWord{6, 0, 358}, + dictWord{7, 0, 1393}, + dictWord{8, 0, 396}, + dictWord{10, 0, 263}, + dictWord{14, 0, 154}, + dictWord{16, 0, 48}, + dictWord{ + 17, + 0, + 8, + }, + dictWord{7, 11, 1759}, + dictWord{8, 11, 396}, + dictWord{10, 11, 263}, + dictWord{14, 11, 154}, + dictWord{16, 11, 48}, + dictWord{145, 11, 8}, + dictWord{ + 13, + 11, + 163, + }, + dictWord{13, 11, 180}, + dictWord{18, 11, 78}, + dictWord{148, 11, 35}, + dictWord{14, 0, 32}, + dictWord{18, 0, 85}, + dictWord{20, 0, 2}, + dictWord{152, 0, 16}, + dictWord{7, 0, 228}, + dictWord{10, 0, 770}, + dictWord{8, 10, 167}, + dictWord{8, 10, 375}, + dictWord{9, 10, 82}, + dictWord{9, 10, 561}, + dictWord{138, 10, 620}, + dictWord{132, 0, 845}, + dictWord{9, 0, 14}, + dictWord{9, 0, 441}, + dictWord{10, 0, 306}, + dictWord{139, 0, 9}, + dictWord{11, 0, 966}, + dictWord{12, 0, 287}, + dictWord{ + 13, + 0, + 342, + }, + dictWord{13, 0, 402}, + dictWord{15, 0, 110}, + dictWord{15, 0, 163}, + dictWord{8, 10, 194}, + dictWord{136, 10, 756}, + dictWord{134, 0, 1578}, + dictWord{ + 4, + 0, + 967, + }, + dictWord{6, 0, 1820}, + dictWord{6, 0, 1847}, + dictWord{140, 0, 716}, + dictWord{136, 0, 594}, + dictWord{7, 0, 1428}, + dictWord{7, 0, 1640}, + dictWord{ + 7, + 0, + 1867, + }, + dictWord{9, 0, 169}, + dictWord{9, 0, 182}, + dictWord{9, 0, 367}, + dictWord{9, 0, 478}, + dictWord{9, 0, 506}, + dictWord{9, 0, 551}, + dictWord{9, 0, 557}, + dictWord{ + 9, + 0, + 648, + }, + dictWord{9, 0, 697}, + dictWord{9, 0, 705}, + dictWord{9, 0, 725}, + dictWord{9, 0, 787}, + dictWord{9, 0, 794}, + dictWord{10, 0, 198}, + dictWord{10, 0, 214}, + dictWord{10, 0, 267}, + dictWord{10, 0, 275}, + dictWord{10, 0, 456}, + dictWord{10, 0, 551}, + dictWord{10, 0, 561}, + dictWord{10, 0, 613}, + dictWord{10, 0, 627}, + dictWord{ + 10, + 0, + 668, + }, + dictWord{10, 0, 675}, + dictWord{10, 0, 691}, + dictWord{10, 0, 695}, + dictWord{10, 0, 707}, + dictWord{10, 0, 715}, + dictWord{11, 0, 183}, + dictWord{ + 11, + 0, + 201, + }, + dictWord{11, 0, 244}, + dictWord{11, 0, 262}, + dictWord{11, 0, 352}, + dictWord{11, 0, 439}, + dictWord{11, 0, 493}, + dictWord{11, 0, 572}, + dictWord{11, 0, 591}, + dictWord{11, 0, 608}, + dictWord{11, 0, 611}, + dictWord{11, 0, 646}, + dictWord{11, 0, 674}, + dictWord{11, 0, 711}, + dictWord{11, 0, 751}, + dictWord{11, 0, 761}, + dictWord{11, 0, 776}, + dictWord{11, 0, 785}, + dictWord{11, 0, 850}, + dictWord{11, 0, 853}, + dictWord{11, 0, 862}, + dictWord{11, 0, 865}, + dictWord{11, 0, 868}, + dictWord{ + 11, + 0, + 875, + }, + dictWord{11, 0, 898}, + dictWord{11, 0, 902}, + dictWord{11, 0, 903}, + dictWord{11, 0, 910}, + dictWord{11, 0, 932}, + dictWord{11, 0, 942}, + dictWord{ + 11, + 0, + 957, + }, + dictWord{11, 0, 967}, + dictWord{11, 0, 972}, + dictWord{12, 0, 148}, + dictWord{12, 0, 195}, + dictWord{12, 0, 220}, + dictWord{12, 0, 237}, + dictWord{12, 0, 318}, + dictWord{12, 0, 339}, + dictWord{12, 0, 393}, + dictWord{12, 0, 445}, + dictWord{12, 0, 450}, + dictWord{12, 0, 474}, + dictWord{12, 0, 505}, + dictWord{12, 0, 509}, + dictWord{12, 0, 533}, + dictWord{12, 0, 591}, + dictWord{12, 0, 594}, + dictWord{12, 0, 597}, + dictWord{12, 0, 621}, + dictWord{12, 0, 633}, + dictWord{12, 0, 642}, + dictWord{ + 13, + 0, + 59, + }, + dictWord{13, 0, 60}, + dictWord{13, 0, 145}, + dictWord{13, 0, 239}, + dictWord{13, 0, 250}, + dictWord{13, 0, 329}, + dictWord{13, 0, 344}, + dictWord{13, 0, 365}, + dictWord{13, 0, 372}, + dictWord{13, 0, 387}, + dictWord{13, 0, 403}, + dictWord{13, 0, 414}, + dictWord{13, 0, 456}, + dictWord{13, 0, 470}, + dictWord{13, 0, 478}, + dictWord{13, 0, 483}, + dictWord{13, 0, 489}, + dictWord{14, 0, 55}, + dictWord{14, 0, 57}, + dictWord{14, 0, 81}, + dictWord{14, 0, 90}, + dictWord{14, 0, 148}, + dictWord{ + 14, + 0, + 239, + }, + dictWord{14, 0, 266}, + dictWord{14, 0, 321}, + dictWord{14, 0, 326}, + dictWord{14, 0, 327}, + dictWord{14, 0, 330}, + dictWord{14, 0, 347}, + dictWord{14, 0, 355}, + dictWord{14, 0, 401}, + dictWord{14, 0, 404}, + dictWord{14, 0, 411}, + dictWord{14, 0, 414}, + dictWord{14, 0, 416}, + dictWord{14, 0, 420}, + dictWord{15, 0, 61}, + dictWord{15, 0, 74}, + dictWord{15, 0, 87}, + dictWord{15, 0, 88}, + dictWord{15, 0, 94}, + dictWord{15, 0, 96}, + dictWord{15, 0, 116}, + dictWord{15, 0, 149}, + dictWord{15, 0, 154}, + dictWord{16, 0, 50}, + dictWord{16, 0, 63}, + dictWord{16, 0, 73}, + dictWord{17, 0, 2}, + dictWord{17, 0, 66}, + dictWord{17, 0, 92}, + dictWord{17, 0, 103}, + dictWord{ + 17, + 0, + 112, + }, + dictWord{17, 0, 120}, + dictWord{18, 0, 50}, + dictWord{18, 0, 54}, + dictWord{18, 0, 82}, + dictWord{18, 0, 86}, + dictWord{18, 0, 90}, + dictWord{18, 0, 111}, + dictWord{ + 18, + 0, + 115, + }, + dictWord{18, 0, 156}, + dictWord{19, 0, 40}, + dictWord{19, 0, 79}, + dictWord{20, 0, 78}, + dictWord{21, 0, 22}, + dictWord{135, 11, 883}, + dictWord{5, 0, 161}, + dictWord{135, 0, 839}, + dictWord{4, 0, 782}, + dictWord{13, 11, 293}, + dictWord{142, 11, 56}, + dictWord{133, 11, 617}, + dictWord{139, 11, 50}, + dictWord{ + 135, + 10, + 22, + }, + dictWord{145, 0, 64}, + dictWord{5, 10, 639}, + dictWord{7, 10, 1249}, + dictWord{139, 10, 896}, + dictWord{138, 0, 998}, + dictWord{135, 11, 2042}, + dictWord{ + 4, + 11, + 546, + }, + dictWord{142, 11, 233}, + dictWord{6, 0, 1043}, + dictWord{134, 0, 1574}, + dictWord{134, 0, 1496}, + dictWord{4, 10, 102}, + dictWord{7, 10, 815}, + dictWord{7, 10, 1699}, + dictWord{139, 10, 964}, + dictWord{12, 0, 781}, + dictWord{142, 0, 461}, + dictWord{4, 11, 313}, + dictWord{133, 11, 577}, + dictWord{ + 6, + 0, + 639, + }, + dictWord{6, 0, 1114}, + dictWord{137, 0, 817}, + dictWord{8, 11, 184}, + dictWord{141, 11, 433}, + dictWord{7, 0, 1814}, + dictWord{135, 11, 935}, + dictWord{ + 10, + 0, + 997, + }, + dictWord{140, 0, 958}, + dictWord{4, 0, 812}, + dictWord{137, 11, 625}, + dictWord{132, 10, 899}, + dictWord{136, 10, 795}, + dictWord{5, 11, 886}, + dictWord{6, 11, 46}, + dictWord{6, 11, 1790}, + dictWord{7, 11, 14}, + dictWord{7, 11, 732}, + dictWord{7, 11, 1654}, + dictWord{8, 11, 95}, + dictWord{8, 11, 327}, + dictWord{ + 8, + 11, + 616, + }, + dictWord{10, 11, 598}, + dictWord{10, 11, 769}, + dictWord{11, 11, 134}, + dictWord{11, 11, 747}, + dictWord{12, 11, 378}, + dictWord{142, 11, 97}, + dictWord{136, 0, 139}, + dictWord{6, 10, 52}, + dictWord{9, 10, 104}, + dictWord{9, 10, 559}, + dictWord{12, 10, 308}, + dictWord{147, 10, 87}, + dictWord{133, 11, 1021}, + dictWord{132, 10, 604}, + dictWord{132, 10, 301}, + dictWord{136, 10, 779}, + dictWord{7, 0, 643}, + dictWord{136, 0, 236}, + dictWord{132, 11, 153}, + dictWord{ + 134, + 0, + 1172, + }, + dictWord{147, 10, 32}, + dictWord{133, 11, 798}, + dictWord{6, 0, 1338}, + dictWord{132, 11, 587}, + dictWord{6, 11, 598}, + dictWord{7, 11, 42}, + dictWord{ + 8, + 11, + 695, + }, + dictWord{10, 11, 212}, + dictWord{11, 11, 158}, + dictWord{14, 11, 196}, + dictWord{145, 11, 85}, + dictWord{135, 10, 508}, + dictWord{5, 11, 957}, + dictWord{5, 11, 1008}, + dictWord{135, 11, 249}, + dictWord{4, 11, 129}, + dictWord{135, 11, 465}, + dictWord{5, 0, 54}, + dictWord{7, 11, 470}, + dictWord{7, 11, 1057}, + dictWord{7, 11, 1201}, + dictWord{9, 11, 755}, + dictWord{11, 11, 906}, + dictWord{140, 11, 527}, + dictWord{7, 11, 908}, + dictWord{146, 11, 7}, + dictWord{ + 5, + 11, + 148, + }, + dictWord{136, 11, 450}, + dictWord{144, 11, 1}, + dictWord{4, 0, 256}, + dictWord{135, 0, 1488}, + dictWord{9, 0, 351}, + dictWord{6, 10, 310}, + dictWord{ + 7, + 10, + 1849, + }, + dictWord{8, 10, 72}, + dictWord{8, 10, 272}, + dictWord{8, 10, 431}, + dictWord{9, 10, 12}, + dictWord{10, 10, 563}, + dictWord{10, 10, 630}, + dictWord{ + 10, + 10, + 796, + }, + dictWord{10, 10, 810}, + dictWord{11, 10, 367}, + dictWord{11, 10, 599}, + dictWord{11, 10, 686}, + dictWord{140, 10, 672}, + dictWord{6, 0, 1885}, + dictWord{ + 6, + 0, + 1898, + }, + dictWord{6, 0, 1899}, + dictWord{140, 0, 955}, + dictWord{4, 0, 714}, + dictWord{133, 0, 469}, + dictWord{6, 0, 1270}, + dictWord{134, 0, 1456}, + dictWord{132, 0, 744}, + dictWord{6, 0, 313}, + dictWord{7, 10, 537}, + dictWord{8, 10, 64}, + dictWord{9, 10, 127}, + dictWord{10, 10, 496}, + dictWord{12, 10, 510}, + dictWord{141, 10, 384}, + dictWord{4, 11, 217}, + dictWord{4, 10, 244}, + dictWord{5, 11, 710}, + dictWord{7, 10, 233}, + dictWord{7, 11, 1926}, + dictWord{9, 11, 428}, + dictWord{9, 11, 708}, + dictWord{10, 11, 254}, + dictWord{10, 11, 296}, + dictWord{10, 11, 720}, + dictWord{11, 11, 109}, + dictWord{11, 11, 255}, + dictWord{12, 11, 165}, + dictWord{12, 11, 315}, + dictWord{13, 11, 107}, + dictWord{13, 11, 203}, + dictWord{14, 11, 54}, + dictWord{14, 11, 99}, + dictWord{14, 11, 114}, + dictWord{ + 14, + 11, + 388, + }, + dictWord{16, 11, 85}, + dictWord{17, 11, 9}, + dictWord{17, 11, 33}, + dictWord{20, 11, 25}, + dictWord{20, 11, 28}, + dictWord{20, 11, 29}, + dictWord{21, 11, 9}, + dictWord{21, 11, 10}, + dictWord{21, 11, 34}, + dictWord{150, 11, 17}, + dictWord{138, 0, 402}, + dictWord{7, 0, 969}, + dictWord{146, 0, 55}, + dictWord{8, 0, 50}, + dictWord{ + 137, + 0, + 624, + }, + dictWord{134, 0, 1355}, + dictWord{132, 0, 572}, + dictWord{134, 10, 1650}, + dictWord{10, 10, 702}, + dictWord{139, 10, 245}, + dictWord{ + 10, + 0, + 847, + }, + dictWord{142, 0, 445}, + dictWord{6, 0, 43}, + dictWord{7, 0, 38}, + dictWord{8, 0, 248}, + dictWord{138, 0, 513}, + dictWord{133, 0, 369}, + dictWord{137, 10, 338}, + dictWord{133, 0, 766}, + dictWord{133, 0, 363}, + dictWord{133, 10, 896}, + dictWord{8, 11, 392}, + dictWord{11, 11, 54}, + dictWord{13, 11, 173}, + dictWord{ + 13, + 11, + 294, + }, + dictWord{148, 11, 7}, + dictWord{134, 0, 678}, + dictWord{7, 11, 1230}, + dictWord{136, 11, 531}, + dictWord{6, 0, 258}, + dictWord{140, 0, 409}, + dictWord{ + 5, + 0, + 249, + }, + dictWord{148, 0, 82}, + dictWord{7, 10, 1117}, + dictWord{136, 10, 539}, + dictWord{5, 0, 393}, + dictWord{6, 0, 378}, + dictWord{7, 0, 1981}, + dictWord{9, 0, 32}, + dictWord{9, 0, 591}, + dictWord{10, 0, 685}, + dictWord{10, 0, 741}, + dictWord{142, 0, 382}, + dictWord{133, 0, 788}, + dictWord{134, 0, 1281}, + dictWord{ + 134, + 0, + 1295, + }, + dictWord{7, 0, 1968}, + dictWord{141, 0, 509}, + dictWord{4, 0, 61}, + dictWord{5, 0, 58}, + dictWord{5, 0, 171}, + dictWord{5, 0, 683}, + dictWord{6, 0, 291}, + dictWord{ + 6, + 0, + 566, + }, + dictWord{7, 0, 1650}, + dictWord{11, 0, 523}, + dictWord{12, 0, 273}, + dictWord{12, 0, 303}, + dictWord{15, 0, 39}, + dictWord{143, 0, 111}, + dictWord{ + 6, + 0, + 706, + }, + dictWord{134, 0, 1283}, + dictWord{134, 0, 589}, + dictWord{135, 11, 1433}, + dictWord{133, 11, 435}, + dictWord{7, 0, 1059}, + dictWord{13, 0, 54}, + dictWord{ + 5, + 10, + 4, + }, + dictWord{5, 10, 810}, + dictWord{6, 10, 13}, + dictWord{6, 10, 538}, + dictWord{6, 10, 1690}, + dictWord{6, 10, 1726}, + dictWord{7, 10, 1819}, + dictWord{ + 8, + 10, + 148, + }, + dictWord{8, 10, 696}, + dictWord{8, 10, 791}, + dictWord{12, 10, 125}, + dictWord{143, 10, 9}, + dictWord{135, 10, 1268}, + dictWord{5, 11, 85}, + dictWord{ + 6, + 11, + 419, + }, + dictWord{7, 11, 134}, + dictWord{7, 11, 305}, + dictWord{7, 11, 361}, + dictWord{7, 11, 1337}, + dictWord{8, 11, 71}, + dictWord{140, 11, 519}, + dictWord{ + 137, + 0, + 824, + }, + dictWord{140, 11, 688}, + dictWord{5, 11, 691}, + dictWord{7, 11, 345}, + dictWord{7, 10, 1385}, + dictWord{9, 11, 94}, + dictWord{11, 10, 582}, + dictWord{ + 11, + 10, + 650, + }, + dictWord{11, 10, 901}, + dictWord{11, 10, 949}, + dictWord{12, 11, 169}, + dictWord{12, 10, 232}, + dictWord{12, 10, 236}, + dictWord{13, 10, 413}, + dictWord{13, 10, 501}, + dictWord{146, 10, 116}, + dictWord{4, 0, 917}, + dictWord{133, 0, 1005}, + dictWord{7, 0, 1598}, + dictWord{5, 11, 183}, + dictWord{6, 11, 582}, + dictWord{9, 11, 344}, + dictWord{10, 11, 679}, + dictWord{140, 11, 435}, + dictWord{4, 10, 925}, + dictWord{5, 10, 803}, + dictWord{8, 10, 698}, + dictWord{ + 138, + 10, + 828, + }, + dictWord{132, 0, 919}, + dictWord{135, 11, 511}, + dictWord{139, 10, 992}, + dictWord{4, 0, 255}, + dictWord{5, 0, 302}, + dictWord{6, 0, 132}, + dictWord{ + 7, + 0, + 128, + }, + dictWord{7, 0, 283}, + dictWord{7, 0, 1299}, + dictWord{10, 0, 52}, + dictWord{10, 0, 514}, + dictWord{11, 0, 925}, + dictWord{13, 0, 92}, + dictWord{142, 0, 309}, + dictWord{134, 0, 1369}, + dictWord{135, 10, 1847}, + dictWord{134, 0, 328}, + dictWord{7, 11, 1993}, + dictWord{136, 11, 684}, + dictWord{133, 10, 383}, + dictWord{137, 0, 173}, + dictWord{134, 11, 583}, + dictWord{134, 0, 1411}, + dictWord{19, 0, 65}, + dictWord{5, 11, 704}, + dictWord{8, 11, 357}, + dictWord{10, 11, 745}, + dictWord{14, 11, 426}, + dictWord{17, 11, 94}, + dictWord{147, 11, 57}, + dictWord{9, 10, 660}, + dictWord{138, 10, 347}, + dictWord{4, 11, 179}, + dictWord{5, 11, 198}, + dictWord{133, 11, 697}, + dictWord{7, 11, 347}, + dictWord{7, 11, 971}, + dictWord{8, 11, 181}, + dictWord{138, 11, 711}, + dictWord{141, 0, 442}, + dictWord{ + 11, + 0, + 842, + }, + dictWord{11, 0, 924}, + dictWord{13, 0, 317}, + dictWord{13, 0, 370}, + dictWord{13, 0, 469}, + dictWord{13, 0, 471}, + dictWord{14, 0, 397}, + dictWord{18, 0, 69}, + dictWord{18, 0, 145}, + dictWord{7, 10, 572}, + dictWord{9, 10, 592}, + dictWord{11, 10, 680}, + dictWord{12, 10, 356}, + dictWord{140, 10, 550}, + dictWord{14, 11, 19}, + dictWord{14, 11, 28}, + dictWord{144, 11, 29}, + dictWord{136, 0, 534}, + dictWord{4, 11, 243}, + dictWord{5, 11, 203}, + dictWord{7, 11, 19}, + dictWord{7, 11, 71}, + dictWord{7, 11, 113}, + dictWord{10, 11, 405}, + dictWord{11, 11, 357}, + dictWord{142, 11, 240}, + dictWord{6, 0, 210}, + dictWord{10, 0, 845}, + dictWord{138, 0, 862}, + dictWord{7, 11, 1351}, + dictWord{9, 11, 581}, + dictWord{10, 11, 639}, + dictWord{11, 11, 453}, + dictWord{140, 11, 584}, + dictWord{7, 11, 1450}, + dictWord{ + 139, + 11, + 99, + }, + dictWord{10, 0, 892}, + dictWord{12, 0, 719}, + dictWord{144, 0, 105}, + dictWord{4, 0, 284}, + dictWord{6, 0, 223}, + dictWord{134, 11, 492}, + dictWord{5, 11, 134}, + dictWord{6, 11, 408}, + dictWord{6, 11, 495}, + dictWord{135, 11, 1593}, + dictWord{136, 0, 529}, + dictWord{137, 0, 807}, + dictWord{4, 0, 218}, + dictWord{7, 0, 526}, + dictWord{143, 0, 137}, + dictWord{6, 0, 1444}, + dictWord{142, 11, 4}, + dictWord{132, 11, 665}, + dictWord{4, 0, 270}, + dictWord{5, 0, 192}, + dictWord{6, 0, 332}, + dictWord{7, 0, 1322}, + dictWord{4, 11, 248}, + dictWord{7, 11, 137}, + dictWord{137, 11, 349}, + dictWord{140, 0, 661}, + dictWord{7, 0, 1517}, + dictWord{11, 0, 597}, + dictWord{14, 0, 76}, + dictWord{14, 0, 335}, + dictWord{20, 0, 33}, + dictWord{7, 10, 748}, + dictWord{139, 10, 700}, + dictWord{5, 11, 371}, + dictWord{135, 11, 563}, + dictWord{146, 11, 57}, + dictWord{133, 10, 127}, + dictWord{133, 0, 418}, + dictWord{4, 11, 374}, + dictWord{7, 11, 547}, + dictWord{7, 11, 1700}, + dictWord{7, 11, 1833}, + dictWord{139, 11, 858}, + dictWord{6, 10, 198}, + dictWord{140, 10, 83}, + dictWord{7, 11, 1812}, + dictWord{13, 11, 259}, + dictWord{13, 11, 356}, + dictWord{ + 14, + 11, + 242, + }, + dictWord{147, 11, 114}, + dictWord{7, 0, 379}, + dictWord{8, 0, 481}, + dictWord{9, 0, 377}, + dictWord{5, 10, 276}, + dictWord{6, 10, 55}, + dictWord{ + 135, + 10, + 1369, + }, + dictWord{138, 11, 286}, + dictWord{5, 0, 1003}, + dictWord{6, 0, 149}, + dictWord{6, 10, 1752}, + dictWord{136, 10, 726}, + dictWord{8, 0, 262}, + dictWord{ + 9, + 0, + 627, + }, + dictWord{10, 0, 18}, + dictWord{11, 0, 214}, + dictWord{11, 0, 404}, + dictWord{11, 0, 457}, + dictWord{11, 0, 780}, + dictWord{11, 0, 913}, + dictWord{13, 0, 401}, + dictWord{14, 0, 200}, + dictWord{6, 11, 1647}, + dictWord{7, 11, 1552}, + dictWord{7, 11, 2010}, + dictWord{9, 11, 494}, + dictWord{137, 11, 509}, + dictWord{ + 135, + 0, + 742, + }, + dictWord{136, 0, 304}, + dictWord{132, 0, 142}, + dictWord{133, 10, 764}, + dictWord{6, 10, 309}, + dictWord{7, 10, 331}, + dictWord{138, 10, 550}, + dictWord{135, 10, 1062}, + dictWord{6, 11, 123}, + dictWord{7, 11, 214}, + dictWord{7, 10, 986}, + dictWord{9, 11, 728}, + dictWord{10, 11, 157}, + dictWord{11, 11, 346}, + dictWord{11, 11, 662}, + dictWord{143, 11, 106}, + dictWord{135, 10, 1573}, + dictWord{7, 0, 925}, + dictWord{137, 0, 799}, + dictWord{4, 0, 471}, + dictWord{5, 0, 51}, + dictWord{6, 0, 602}, + dictWord{8, 0, 484}, + dictWord{138, 0, 195}, + dictWord{136, 0, 688}, + dictWord{132, 0, 697}, + dictWord{6, 0, 1169}, + dictWord{6, 0, 1241}, + dictWord{6, 10, 194}, + dictWord{7, 10, 133}, + dictWord{10, 10, 493}, + dictWord{10, 10, 570}, + dictWord{139, 10, 664}, + dictWord{140, 0, 751}, + dictWord{7, 0, 929}, + dictWord{10, 0, 452}, + dictWord{11, 0, 878}, + dictWord{16, 0, 33}, + dictWord{5, 10, 24}, + dictWord{5, 10, 569}, + dictWord{6, 10, 3}, + dictWord{6, 10, 119}, + dictWord{ + 6, + 10, + 143, + }, + dictWord{6, 10, 440}, + dictWord{7, 10, 599}, + dictWord{7, 10, 1686}, + dictWord{7, 10, 1854}, + dictWord{8, 10, 424}, + dictWord{9, 10, 43}, + dictWord{ + 9, + 10, + 584, + }, + dictWord{9, 10, 760}, + dictWord{10, 10, 328}, + dictWord{11, 10, 159}, + dictWord{11, 10, 253}, + dictWord{12, 10, 487}, + dictWord{140, 10, 531}, + dictWord{ + 4, + 11, + 707, + }, + dictWord{13, 11, 106}, + dictWord{18, 11, 49}, + dictWord{147, 11, 41}, + dictWord{5, 0, 221}, + dictWord{5, 11, 588}, + dictWord{134, 11, 393}, + dictWord{134, 0, 1437}, + dictWord{6, 11, 211}, + dictWord{7, 11, 1690}, + dictWord{11, 11, 486}, + dictWord{140, 11, 369}, + dictWord{5, 10, 14}, + dictWord{5, 10, 892}, + dictWord{6, 10, 283}, + dictWord{7, 10, 234}, + dictWord{136, 10, 537}, + dictWord{4, 0, 988}, + dictWord{136, 0, 955}, + dictWord{135, 0, 1251}, + dictWord{4, 10, 126}, + dictWord{8, 10, 635}, + dictWord{147, 10, 34}, + dictWord{4, 10, 316}, + dictWord{135, 10, 1561}, + dictWord{137, 10, 861}, + dictWord{4, 10, 64}, + dictWord{ + 5, + 10, + 352, + }, + dictWord{5, 10, 720}, + dictWord{6, 10, 368}, + dictWord{139, 10, 359}, + dictWord{134, 0, 192}, + dictWord{4, 0, 132}, + dictWord{5, 0, 69}, + dictWord{ + 135, + 0, + 1242, + }, + dictWord{7, 10, 1577}, + dictWord{10, 10, 304}, + dictWord{10, 10, 549}, + dictWord{12, 10, 365}, + dictWord{13, 10, 220}, + dictWord{13, 10, 240}, + dictWord{142, 10, 33}, + dictWord{4, 0, 111}, + dictWord{7, 0, 865}, + dictWord{134, 11, 219}, + dictWord{5, 11, 582}, + dictWord{6, 11, 1646}, + dictWord{7, 11, 99}, + dictWord{ + 7, + 11, + 1962, + }, + dictWord{7, 11, 1986}, + dictWord{8, 11, 515}, + dictWord{8, 11, 773}, + dictWord{9, 11, 23}, + dictWord{9, 11, 491}, + dictWord{12, 11, 620}, + dictWord{ + 14, + 11, + 52, + }, + dictWord{145, 11, 50}, + dictWord{132, 0, 767}, + dictWord{7, 11, 568}, + dictWord{148, 11, 21}, + dictWord{6, 0, 42}, + dictWord{7, 0, 1416}, + dictWord{ + 7, + 0, + 2005, + }, + dictWord{8, 0, 131}, + dictWord{8, 0, 466}, + dictWord{9, 0, 672}, + dictWord{13, 0, 252}, + dictWord{20, 0, 103}, + dictWord{133, 11, 851}, + dictWord{ + 135, + 0, + 1050, + }, + dictWord{6, 10, 175}, + dictWord{137, 10, 289}, + dictWord{5, 10, 432}, + dictWord{133, 10, 913}, + dictWord{6, 0, 44}, + dictWord{136, 0, 368}, + dictWord{ + 135, + 11, + 784, + }, + dictWord{132, 0, 570}, + dictWord{133, 0, 120}, + dictWord{139, 10, 595}, + dictWord{140, 0, 29}, + dictWord{6, 0, 227}, + dictWord{135, 0, 1589}, + dictWord{4, 11, 98}, + dictWord{7, 11, 1365}, + dictWord{9, 11, 422}, + dictWord{9, 11, 670}, + dictWord{10, 11, 775}, + dictWord{11, 11, 210}, + dictWord{13, 11, 26}, + dictWord{13, 11, 457}, + dictWord{141, 11, 476}, + dictWord{140, 10, 80}, + dictWord{5, 10, 931}, + dictWord{134, 10, 1698}, + dictWord{133, 0, 522}, + dictWord{ + 134, + 0, + 1120, + }, + dictWord{135, 0, 1529}, + dictWord{12, 0, 739}, + dictWord{14, 0, 448}, + dictWord{142, 0, 467}, + dictWord{11, 10, 526}, + dictWord{11, 10, 939}, + dictWord{141, 10, 290}, + dictWord{5, 10, 774}, + dictWord{6, 10, 1637}, + dictWord{6, 10, 1686}, + dictWord{134, 10, 1751}, + dictWord{6, 0, 1667}, + dictWord{ + 135, + 0, + 2036, + }, + dictWord{7, 10, 1167}, + dictWord{11, 10, 934}, + dictWord{13, 10, 391}, + dictWord{145, 10, 76}, + dictWord{137, 11, 147}, + dictWord{6, 10, 260}, + dictWord{ + 7, + 10, + 1484, + }, + dictWord{11, 11, 821}, + dictWord{12, 11, 110}, + dictWord{12, 11, 153}, + dictWord{18, 11, 41}, + dictWord{150, 11, 19}, + dictWord{6, 0, 511}, + dictWord{12, 0, 132}, + dictWord{134, 10, 573}, + dictWord{5, 0, 568}, + dictWord{6, 0, 138}, + dictWord{135, 0, 1293}, + dictWord{132, 0, 1020}, + dictWord{8, 0, 258}, + dictWord{9, 0, 208}, + dictWord{137, 0, 359}, + dictWord{4, 0, 565}, + dictWord{8, 0, 23}, + dictWord{136, 0, 827}, + dictWord{134, 0, 344}, + dictWord{4, 0, 922}, + dictWord{ + 5, + 0, + 1023, + }, + dictWord{13, 11, 477}, + dictWord{14, 11, 120}, + dictWord{148, 11, 61}, + dictWord{134, 0, 240}, + dictWord{5, 11, 209}, + dictWord{6, 11, 30}, + dictWord{ + 11, + 11, + 56, + }, + dictWord{139, 11, 305}, + dictWord{6, 0, 171}, + dictWord{7, 0, 1002}, + dictWord{7, 0, 1324}, + dictWord{9, 0, 415}, + dictWord{14, 0, 230}, + dictWord{ + 18, + 0, + 68, + }, + dictWord{4, 10, 292}, + dictWord{4, 10, 736}, + dictWord{5, 10, 871}, + dictWord{6, 10, 1689}, + dictWord{7, 10, 1944}, + dictWord{137, 10, 580}, + dictWord{ + 9, + 11, + 635, + }, + dictWord{139, 11, 559}, + dictWord{4, 11, 150}, + dictWord{5, 11, 303}, + dictWord{134, 11, 327}, + dictWord{6, 10, 63}, + dictWord{135, 10, 920}, + dictWord{ + 133, + 10, + 793, + }, + dictWord{8, 11, 192}, + dictWord{10, 11, 78}, + dictWord{10, 11, 555}, + dictWord{11, 11, 308}, + dictWord{13, 11, 359}, + dictWord{147, 11, 95}, + dictWord{135, 11, 786}, + dictWord{135, 11, 1712}, + dictWord{136, 0, 402}, + dictWord{6, 0, 754}, + dictWord{6, 11, 1638}, + dictWord{7, 11, 79}, + dictWord{7, 11, 496}, + dictWord{9, 11, 138}, + dictWord{10, 11, 336}, + dictWord{11, 11, 12}, + dictWord{12, 11, 412}, + dictWord{12, 11, 440}, + dictWord{142, 11, 305}, + dictWord{4, 0, 716}, + dictWord{141, 0, 31}, + dictWord{133, 0, 982}, + dictWord{8, 0, 691}, + dictWord{8, 0, 731}, + dictWord{5, 10, 67}, + dictWord{6, 10, 62}, + dictWord{6, 10, 374}, + dictWord{ + 135, + 10, + 1391, + }, + dictWord{9, 10, 790}, + dictWord{140, 10, 47}, + dictWord{139, 11, 556}, + dictWord{151, 11, 1}, + dictWord{7, 11, 204}, + dictWord{7, 11, 415}, + dictWord{8, 11, 42}, + dictWord{10, 11, 85}, + dictWord{11, 11, 33}, + dictWord{11, 11, 564}, + dictWord{12, 11, 571}, + dictWord{149, 11, 1}, + dictWord{8, 0, 888}, + dictWord{ + 7, + 11, + 610, + }, + dictWord{135, 11, 1501}, + dictWord{4, 10, 391}, + dictWord{135, 10, 1169}, + dictWord{5, 0, 847}, + dictWord{9, 0, 840}, + dictWord{138, 0, 803}, + dictWord{137, 0, 823}, + dictWord{134, 0, 785}, + dictWord{8, 0, 152}, + dictWord{9, 0, 53}, + dictWord{9, 0, 268}, + dictWord{9, 0, 901}, + dictWord{10, 0, 518}, + dictWord{ + 10, + 0, + 829, + }, + dictWord{11, 0, 188}, + dictWord{13, 0, 74}, + dictWord{14, 0, 46}, + dictWord{15, 0, 17}, + dictWord{15, 0, 33}, + dictWord{17, 0, 40}, + dictWord{18, 0, 36}, + dictWord{ + 19, + 0, + 20, + }, + dictWord{22, 0, 1}, + dictWord{152, 0, 2}, + dictWord{4, 11, 3}, + dictWord{5, 11, 247}, + dictWord{5, 11, 644}, + dictWord{7, 11, 744}, + dictWord{7, 11, 1207}, + dictWord{7, 11, 1225}, + dictWord{7, 11, 1909}, + dictWord{146, 11, 147}, + dictWord{136, 0, 532}, + dictWord{135, 0, 681}, + dictWord{132, 10, 271}, + dictWord{ + 140, + 0, + 314, + }, + dictWord{140, 0, 677}, + dictWord{4, 0, 684}, + dictWord{136, 0, 384}, + dictWord{5, 11, 285}, + dictWord{9, 11, 67}, + dictWord{13, 11, 473}, + dictWord{ + 143, + 11, + 82, + }, + dictWord{4, 10, 253}, + dictWord{5, 10, 544}, + dictWord{7, 10, 300}, + dictWord{137, 10, 340}, + dictWord{7, 0, 110}, + dictWord{7, 0, 447}, + dictWord{8, 0, 290}, + dictWord{8, 0, 591}, + dictWord{9, 0, 382}, + dictWord{9, 0, 649}, + dictWord{11, 0, 71}, + dictWord{11, 0, 155}, + dictWord{11, 0, 313}, + dictWord{12, 0, 5}, + dictWord{13, 0, 325}, + dictWord{142, 0, 287}, + dictWord{134, 0, 1818}, + dictWord{136, 0, 1007}, + dictWord{138, 0, 321}, + dictWord{7, 0, 360}, + dictWord{7, 0, 425}, + dictWord{9, 0, 66}, + dictWord{9, 0, 278}, + dictWord{138, 0, 644}, + dictWord{133, 10, 818}, + dictWord{5, 0, 385}, + dictWord{5, 10, 541}, + dictWord{6, 10, 94}, + dictWord{6, 10, 499}, + dictWord{ + 7, + 10, + 230, + }, + dictWord{139, 10, 321}, + dictWord{4, 10, 920}, + dictWord{5, 10, 25}, + dictWord{5, 10, 790}, + dictWord{6, 10, 457}, + dictWord{7, 10, 853}, + dictWord{ + 136, + 10, + 788, + }, + dictWord{4, 0, 900}, + dictWord{133, 0, 861}, + dictWord{5, 0, 254}, + dictWord{7, 0, 985}, + dictWord{136, 0, 73}, + dictWord{7, 0, 1959}, + dictWord{ + 136, + 0, + 683, + }, + dictWord{134, 10, 1765}, + dictWord{133, 10, 822}, + dictWord{132, 10, 634}, + dictWord{4, 11, 29}, + dictWord{6, 11, 532}, + dictWord{7, 11, 1628}, + dictWord{ + 7, + 11, + 1648, + }, + dictWord{9, 11, 303}, + dictWord{9, 11, 350}, + dictWord{10, 11, 433}, + dictWord{11, 11, 97}, + dictWord{11, 11, 557}, + dictWord{11, 11, 745}, + dictWord{12, 11, 289}, + dictWord{12, 11, 335}, + dictWord{12, 11, 348}, + dictWord{12, 11, 606}, + dictWord{13, 11, 116}, + dictWord{13, 11, 233}, + dictWord{ + 13, + 11, + 466, + }, + dictWord{14, 11, 181}, + dictWord{14, 11, 209}, + dictWord{14, 11, 232}, + dictWord{14, 11, 236}, + dictWord{14, 11, 300}, + dictWord{16, 11, 41}, + dictWord{ + 148, + 11, + 97, + }, + dictWord{19, 0, 86}, + dictWord{6, 10, 36}, + dictWord{7, 10, 658}, + dictWord{136, 10, 454}, + dictWord{135, 11, 1692}, + dictWord{132, 0, 725}, + dictWord{ + 5, + 11, + 501, + }, + dictWord{7, 11, 1704}, + dictWord{9, 11, 553}, + dictWord{11, 11, 520}, + dictWord{12, 11, 557}, + dictWord{141, 11, 249}, + dictWord{134, 0, 196}, + dictWord{133, 0, 831}, + dictWord{136, 0, 723}, + dictWord{7, 0, 1897}, + dictWord{13, 0, 80}, + dictWord{13, 0, 437}, + dictWord{145, 0, 74}, + dictWord{4, 0, 992}, + dictWord{ + 6, + 0, + 627, + }, + dictWord{136, 0, 994}, + dictWord{135, 11, 1294}, + dictWord{132, 10, 104}, + dictWord{5, 0, 848}, + dictWord{6, 0, 66}, + dictWord{136, 0, 764}, + dictWord{ + 4, + 0, + 36, + }, + dictWord{7, 0, 1387}, + dictWord{10, 0, 205}, + dictWord{139, 0, 755}, + dictWord{6, 0, 1046}, + dictWord{134, 0, 1485}, + dictWord{134, 0, 950}, + dictWord{132, 0, 887}, + dictWord{14, 0, 450}, + dictWord{148, 0, 111}, + dictWord{7, 0, 620}, + dictWord{7, 0, 831}, + dictWord{9, 10, 542}, + dictWord{9, 10, 566}, + dictWord{ + 138, + 10, + 728, + }, + dictWord{6, 0, 165}, + dictWord{138, 0, 388}, + dictWord{139, 10, 263}, + dictWord{4, 0, 719}, + dictWord{135, 0, 155}, + dictWord{138, 10, 468}, + dictWord{6, 11, 453}, + dictWord{144, 11, 36}, + dictWord{134, 11, 129}, + dictWord{5, 0, 533}, + dictWord{7, 0, 755}, + dictWord{138, 0, 780}, + dictWord{134, 0, 1465}, + dictWord{4, 0, 353}, + dictWord{6, 0, 146}, + dictWord{6, 0, 1789}, + dictWord{7, 0, 427}, + dictWord{7, 0, 990}, + dictWord{7, 0, 1348}, + dictWord{9, 0, 665}, + dictWord{9, 0, 898}, + dictWord{11, 0, 893}, + dictWord{142, 0, 212}, + dictWord{7, 10, 87}, + dictWord{142, 10, 288}, + dictWord{4, 0, 45}, + dictWord{135, 0, 1257}, + dictWord{12, 0, 7}, + dictWord{7, 10, 988}, + dictWord{7, 10, 1939}, + dictWord{9, 10, 64}, + dictWord{9, 10, 502}, + dictWord{12, 10, 34}, + dictWord{13, 10, 12}, + dictWord{13, 10, 234}, + dictWord{147, 10, 77}, + dictWord{4, 0, 607}, + dictWord{5, 11, 60}, + dictWord{6, 11, 504}, + dictWord{7, 11, 614}, + dictWord{7, 11, 1155}, + dictWord{140, 11, 0}, + dictWord{ + 135, + 10, + 141, + }, + dictWord{8, 11, 198}, + dictWord{11, 11, 29}, + dictWord{140, 11, 534}, + dictWord{140, 0, 65}, + dictWord{136, 0, 816}, + dictWord{132, 10, 619}, + dictWord{139, 0, 88}, + dictWord{5, 10, 246}, + dictWord{8, 10, 189}, + dictWord{9, 10, 355}, + dictWord{9, 10, 512}, + dictWord{10, 10, 124}, + dictWord{10, 10, 453}, + dictWord{11, 10, 143}, + dictWord{11, 10, 416}, + dictWord{11, 10, 859}, + dictWord{141, 10, 341}, + dictWord{4, 11, 379}, + dictWord{135, 11, 1397}, + dictWord{ + 4, + 0, + 600, + }, + dictWord{137, 0, 621}, + dictWord{133, 0, 367}, + dictWord{134, 0, 561}, + dictWord{6, 0, 559}, + dictWord{134, 0, 1691}, + dictWord{6, 0, 585}, + dictWord{ + 134, + 11, + 585, + }, + dictWord{135, 11, 1228}, + dictWord{4, 11, 118}, + dictWord{5, 10, 678}, + dictWord{6, 11, 274}, + dictWord{6, 11, 361}, + dictWord{7, 11, 75}, + dictWord{ + 141, + 11, + 441, + }, + dictWord{135, 11, 1818}, + dictWord{137, 11, 841}, + dictWord{5, 0, 573}, + dictWord{6, 0, 287}, + dictWord{7, 10, 862}, + dictWord{7, 10, 1886}, + dictWord{138, 10, 179}, + dictWord{132, 10, 517}, + dictWord{140, 11, 693}, + dictWord{5, 11, 314}, + dictWord{6, 11, 221}, + dictWord{7, 11, 419}, + dictWord{ + 10, + 11, + 650, + }, + dictWord{11, 11, 396}, + dictWord{12, 11, 156}, + dictWord{13, 11, 369}, + dictWord{14, 11, 333}, + dictWord{145, 11, 47}, + dictWord{140, 10, 540}, + dictWord{136, 10, 667}, + dictWord{11, 10, 403}, + dictWord{146, 10, 83}, + dictWord{6, 0, 672}, + dictWord{133, 10, 761}, + dictWord{9, 0, 157}, + dictWord{10, 10, 131}, + dictWord{140, 10, 72}, + dictWord{7, 0, 714}, + dictWord{134, 11, 460}, + dictWord{134, 0, 456}, + dictWord{133, 0, 925}, + dictWord{5, 11, 682}, + dictWord{ + 135, + 11, + 1887, + }, + dictWord{136, 11, 510}, + dictWord{136, 11, 475}, + dictWord{133, 11, 1016}, + dictWord{9, 0, 19}, + dictWord{7, 11, 602}, + dictWord{8, 11, 179}, + dictWord{ + 10, + 11, + 781, + }, + dictWord{140, 11, 126}, + dictWord{6, 11, 329}, + dictWord{138, 11, 111}, + dictWord{6, 0, 822}, + dictWord{134, 0, 1473}, + dictWord{144, 11, 86}, + dictWord{11, 0, 113}, + dictWord{139, 11, 113}, + dictWord{5, 11, 821}, + dictWord{134, 11, 1687}, + dictWord{133, 10, 449}, + dictWord{7, 0, 463}, + dictWord{ + 17, + 0, + 69, + }, + dictWord{136, 10, 103}, + dictWord{7, 10, 2028}, + dictWord{138, 10, 641}, + dictWord{6, 0, 193}, + dictWord{7, 0, 240}, + dictWord{7, 0, 1682}, + dictWord{ + 10, + 0, + 51, + }, + dictWord{10, 0, 640}, + dictWord{11, 0, 410}, + dictWord{13, 0, 82}, + dictWord{14, 0, 247}, + dictWord{14, 0, 331}, + dictWord{142, 0, 377}, + dictWord{6, 0, 471}, + dictWord{11, 0, 411}, + dictWord{142, 0, 2}, + dictWord{5, 11, 71}, + dictWord{7, 11, 1407}, + dictWord{9, 11, 388}, + dictWord{9, 11, 704}, + dictWord{10, 11, 261}, + dictWord{ + 10, + 11, + 619, + }, + dictWord{11, 11, 547}, + dictWord{11, 11, 619}, + dictWord{143, 11, 157}, + dictWord{136, 0, 633}, + dictWord{135, 0, 1148}, + dictWord{6, 0, 554}, + dictWord{7, 0, 1392}, + dictWord{12, 0, 129}, + dictWord{7, 10, 1274}, + dictWord{7, 10, 1386}, + dictWord{7, 11, 2008}, + dictWord{9, 11, 337}, + dictWord{10, 11, 517}, + dictWord{146, 10, 87}, + dictWord{7, 0, 803}, + dictWord{8, 0, 542}, + dictWord{6, 10, 187}, + dictWord{7, 10, 1203}, + dictWord{8, 10, 380}, + dictWord{14, 10, 117}, + dictWord{149, 10, 28}, + dictWord{6, 10, 297}, + dictWord{7, 10, 793}, + dictWord{139, 10, 938}, + dictWord{8, 0, 438}, + dictWord{11, 0, 363}, + dictWord{7, 10, 464}, + dictWord{11, 10, 105}, + dictWord{12, 10, 231}, + dictWord{14, 10, 386}, + dictWord{15, 10, 102}, + dictWord{148, 10, 75}, + dictWord{5, 11, 16}, + dictWord{6, 11, 86}, + dictWord{6, 11, 603}, + dictWord{7, 11, 292}, + dictWord{7, 11, 561}, + dictWord{8, 11, 257}, + dictWord{8, 11, 382}, + dictWord{9, 11, 721}, + dictWord{9, 11, 778}, + dictWord{ + 11, + 11, + 581, + }, + dictWord{140, 11, 466}, + dictWord{6, 0, 717}, + dictWord{4, 11, 486}, + dictWord{133, 11, 491}, + dictWord{132, 0, 875}, + dictWord{132, 11, 72}, + dictWord{6, 11, 265}, + dictWord{135, 11, 847}, + dictWord{4, 0, 237}, + dictWord{135, 0, 514}, + dictWord{6, 0, 392}, + dictWord{7, 0, 65}, + dictWord{135, 0, 2019}, + dictWord{140, 11, 261}, + dictWord{135, 11, 922}, + dictWord{137, 11, 404}, + dictWord{12, 0, 563}, + dictWord{14, 0, 101}, + dictWord{18, 0, 129}, + dictWord{ + 7, + 10, + 1010, + }, + dictWord{11, 10, 733}, + dictWord{11, 10, 759}, + dictWord{13, 10, 34}, + dictWord{146, 10, 45}, + dictWord{7, 10, 1656}, + dictWord{9, 10, 369}, + dictWord{ + 10, + 10, + 338, + }, + dictWord{10, 10, 490}, + dictWord{11, 10, 154}, + dictWord{11, 10, 545}, + dictWord{11, 10, 775}, + dictWord{13, 10, 77}, + dictWord{141, 10, 274}, + dictWord{4, 0, 444}, + dictWord{10, 0, 146}, + dictWord{140, 0, 9}, + dictWord{139, 11, 163}, + dictWord{7, 0, 1260}, + dictWord{135, 0, 1790}, + dictWord{9, 0, 222}, + dictWord{10, 0, 43}, + dictWord{139, 0, 900}, + dictWord{137, 11, 234}, + dictWord{138, 0, 971}, + dictWord{137, 0, 761}, + dictWord{134, 0, 699}, + dictWord{ + 136, + 11, + 434, + }, + dictWord{6, 0, 1116}, + dictWord{7, 0, 1366}, + dictWord{5, 10, 20}, + dictWord{6, 11, 197}, + dictWord{6, 10, 298}, + dictWord{7, 10, 659}, + dictWord{8, 11, 205}, + dictWord{137, 10, 219}, + dictWord{132, 11, 490}, + dictWord{11, 11, 820}, + dictWord{150, 11, 51}, + dictWord{7, 10, 1440}, + dictWord{11, 10, 854}, + dictWord{ + 11, + 10, + 872, + }, + dictWord{11, 10, 921}, + dictWord{12, 10, 551}, + dictWord{13, 10, 472}, + dictWord{142, 10, 367}, + dictWord{140, 11, 13}, + dictWord{132, 0, 829}, + dictWord{12, 0, 242}, + dictWord{132, 10, 439}, + dictWord{136, 10, 669}, + dictWord{6, 0, 593}, + dictWord{6, 11, 452}, + dictWord{7, 11, 312}, + dictWord{ + 138, + 11, + 219, + }, + dictWord{4, 11, 333}, + dictWord{9, 11, 176}, + dictWord{12, 11, 353}, + dictWord{141, 11, 187}, + dictWord{7, 0, 36}, + dictWord{8, 0, 201}, + dictWord{ + 136, + 0, + 605, + }, + dictWord{140, 0, 224}, + dictWord{132, 10, 233}, + dictWord{134, 0, 1430}, + dictWord{134, 0, 1806}, + dictWord{4, 0, 523}, + dictWord{133, 0, 638}, + dictWord{ + 6, + 0, + 1889, + }, + dictWord{9, 0, 958}, + dictWord{9, 0, 971}, + dictWord{9, 0, 976}, + dictWord{12, 0, 796}, + dictWord{12, 0, 799}, + dictWord{12, 0, 808}, + dictWord{ + 12, + 0, + 835, + }, + dictWord{12, 0, 836}, + dictWord{12, 0, 914}, + dictWord{12, 0, 946}, + dictWord{15, 0, 216}, + dictWord{15, 0, 232}, + dictWord{18, 0, 183}, + dictWord{18, 0, 187}, + dictWord{18, 0, 194}, + dictWord{18, 0, 212}, + dictWord{18, 0, 232}, + dictWord{149, 0, 49}, + dictWord{132, 10, 482}, + dictWord{6, 0, 827}, + dictWord{134, 0, 1434}, + dictWord{135, 10, 346}, + dictWord{134, 0, 2043}, + dictWord{6, 0, 242}, + dictWord{7, 0, 227}, + dictWord{7, 0, 1581}, + dictWord{8, 0, 104}, + dictWord{9, 0, 113}, + dictWord{9, 0, 220}, + dictWord{9, 0, 427}, + dictWord{10, 0, 136}, + dictWord{10, 0, 239}, + dictWord{11, 0, 579}, + dictWord{11, 0, 1023}, + dictWord{13, 0, 4}, + dictWord{ + 13, + 0, + 204, + }, + dictWord{13, 0, 316}, + dictWord{148, 0, 86}, + dictWord{134, 11, 1685}, + dictWord{7, 0, 148}, + dictWord{8, 0, 284}, + dictWord{141, 0, 63}, + dictWord{ + 142, + 0, + 10, + }, + dictWord{135, 11, 584}, + dictWord{134, 0, 1249}, + dictWord{7, 0, 861}, + dictWord{135, 10, 334}, + dictWord{5, 10, 795}, + dictWord{6, 10, 1741}, + dictWord{ + 137, + 11, + 70, + }, + dictWord{132, 0, 807}, + dictWord{7, 11, 135}, + dictWord{8, 11, 7}, + dictWord{8, 11, 62}, + dictWord{9, 11, 243}, + dictWord{10, 11, 658}, + dictWord{ + 10, + 11, + 697, + }, + dictWord{11, 11, 456}, + dictWord{139, 11, 756}, + dictWord{9, 11, 395}, + dictWord{138, 11, 79}, + dictWord{137, 11, 108}, + dictWord{147, 0, 94}, + dictWord{136, 0, 494}, + dictWord{135, 11, 631}, + dictWord{135, 10, 622}, + dictWord{7, 0, 1510}, + dictWord{135, 10, 1750}, + dictWord{4, 10, 203}, + dictWord{ + 135, + 10, + 1936, + }, + dictWord{7, 11, 406}, + dictWord{7, 11, 459}, + dictWord{8, 11, 606}, + dictWord{139, 11, 726}, + dictWord{7, 0, 1306}, + dictWord{8, 0, 505}, + dictWord{ + 9, + 0, + 482, + }, + dictWord{10, 0, 126}, + dictWord{11, 0, 225}, + dictWord{12, 0, 347}, + dictWord{12, 0, 449}, + dictWord{13, 0, 19}, + dictWord{14, 0, 218}, + dictWord{142, 0, 435}, + dictWord{5, 0, 268}, + dictWord{10, 0, 764}, + dictWord{12, 0, 120}, + dictWord{13, 0, 39}, + dictWord{145, 0, 127}, + dictWord{142, 11, 68}, + dictWord{11, 10, 678}, + dictWord{140, 10, 307}, + dictWord{12, 11, 268}, + dictWord{12, 11, 640}, + dictWord{142, 11, 119}, + dictWord{135, 10, 2044}, + dictWord{133, 11, 612}, + dictWord{ + 4, + 11, + 372, + }, + dictWord{7, 11, 482}, + dictWord{8, 11, 158}, + dictWord{9, 11, 602}, + dictWord{9, 11, 615}, + dictWord{10, 11, 245}, + dictWord{10, 11, 678}, + dictWord{ + 10, + 11, + 744, + }, + dictWord{11, 11, 248}, + dictWord{139, 11, 806}, + dictWord{7, 10, 311}, + dictWord{9, 10, 308}, + dictWord{140, 10, 255}, + dictWord{4, 0, 384}, + dictWord{135, 0, 1022}, + dictWord{5, 11, 854}, + dictWord{135, 11, 1991}, + dictWord{135, 10, 1266}, + dictWord{4, 10, 400}, + dictWord{5, 10, 267}, + dictWord{ + 135, + 10, + 232, + }, + dictWord{135, 0, 1703}, + dictWord{9, 0, 159}, + dictWord{11, 0, 661}, + dictWord{140, 0, 603}, + dictWord{4, 0, 964}, + dictWord{14, 0, 438}, + dictWord{ + 14, + 0, + 444, + }, + dictWord{14, 0, 456}, + dictWord{22, 0, 60}, + dictWord{22, 0, 63}, + dictWord{9, 11, 106}, + dictWord{9, 11, 163}, + dictWord{9, 11, 296}, + dictWord{10, 11, 167}, + dictWord{10, 11, 172}, + dictWord{10, 11, 777}, + dictWord{139, 11, 16}, + dictWord{136, 0, 583}, + dictWord{132, 0, 515}, + dictWord{8, 0, 632}, + dictWord{8, 0, 697}, + dictWord{137, 0, 854}, + dictWord{5, 11, 195}, + dictWord{135, 11, 1685}, + dictWord{6, 0, 1123}, + dictWord{134, 0, 1365}, + dictWord{134, 11, 328}, + dictWord{ + 7, + 11, + 1997, + }, + dictWord{8, 11, 730}, + dictWord{139, 11, 1006}, + dictWord{4, 0, 136}, + dictWord{133, 0, 551}, + dictWord{134, 0, 1782}, + dictWord{7, 0, 1287}, + dictWord{ + 9, + 0, + 44, + }, + dictWord{10, 0, 552}, + dictWord{10, 0, 642}, + dictWord{11, 0, 839}, + dictWord{12, 0, 274}, + dictWord{12, 0, 275}, + dictWord{12, 0, 372}, + dictWord{ + 13, + 0, + 91, + }, + dictWord{142, 0, 125}, + dictWord{5, 11, 751}, + dictWord{11, 11, 797}, + dictWord{140, 11, 203}, + dictWord{133, 0, 732}, + dictWord{7, 0, 679}, + dictWord{ + 8, + 0, + 313, + }, + dictWord{4, 10, 100}, + dictWord{135, 11, 821}, + dictWord{10, 0, 361}, + dictWord{142, 0, 316}, + dictWord{134, 0, 595}, + dictWord{6, 0, 147}, + dictWord{ + 7, + 0, + 886, + }, + dictWord{9, 0, 753}, + dictWord{138, 0, 268}, + dictWord{5, 10, 362}, + dictWord{5, 10, 443}, + dictWord{6, 10, 318}, + dictWord{7, 10, 1019}, + dictWord{ + 139, + 10, + 623, + }, + dictWord{5, 10, 463}, + dictWord{136, 10, 296}, + dictWord{4, 10, 454}, + dictWord{5, 11, 950}, + dictWord{5, 11, 994}, + dictWord{134, 11, 351}, + dictWord{ + 138, + 0, + 137, + }, + dictWord{5, 10, 48}, + dictWord{5, 10, 404}, + dictWord{6, 10, 557}, + dictWord{7, 10, 458}, + dictWord{8, 10, 597}, + dictWord{10, 10, 455}, + dictWord{ + 10, + 10, + 606, + }, + dictWord{11, 10, 49}, + dictWord{11, 10, 548}, + dictWord{12, 10, 476}, + dictWord{13, 10, 18}, + dictWord{141, 10, 450}, + dictWord{133, 0, 414}, + dictWord{ + 135, + 0, + 1762, + }, + dictWord{5, 11, 421}, + dictWord{135, 11, 47}, + dictWord{5, 10, 442}, + dictWord{135, 10, 1984}, + dictWord{134, 0, 599}, + dictWord{134, 0, 1749}, + dictWord{134, 0, 1627}, + dictWord{4, 0, 488}, + dictWord{132, 11, 350}, + dictWord{137, 11, 751}, + dictWord{132, 0, 83}, + dictWord{140, 0, 676}, + dictWord{ + 133, + 11, + 967, + }, + dictWord{7, 0, 1639}, + dictWord{5, 10, 55}, + dictWord{140, 10, 161}, + dictWord{4, 11, 473}, + dictWord{7, 11, 623}, + dictWord{8, 11, 808}, + dictWord{ + 9, + 11, + 871, + }, + dictWord{9, 11, 893}, + dictWord{11, 11, 38}, + dictWord{11, 11, 431}, + dictWord{12, 11, 112}, + dictWord{12, 11, 217}, + dictWord{12, 11, 243}, + dictWord{ + 12, + 11, + 562, + }, + dictWord{12, 11, 683}, + dictWord{13, 11, 141}, + dictWord{13, 11, 197}, + dictWord{13, 11, 227}, + dictWord{13, 11, 406}, + dictWord{13, 11, 487}, + dictWord{14, 11, 156}, + dictWord{14, 11, 203}, + dictWord{14, 11, 224}, + dictWord{14, 11, 256}, + dictWord{18, 11, 58}, + dictWord{150, 11, 0}, + dictWord{ + 133, + 10, + 450, + }, + dictWord{7, 11, 736}, + dictWord{139, 11, 264}, + dictWord{134, 0, 278}, + dictWord{4, 11, 222}, + dictWord{7, 11, 286}, + dictWord{136, 11, 629}, + dictWord{ + 135, + 10, + 869, + }, + dictWord{140, 0, 97}, + dictWord{144, 0, 14}, + dictWord{134, 0, 1085}, + dictWord{4, 10, 213}, + dictWord{7, 10, 223}, + dictWord{136, 10, 80}, + dictWord{ + 7, + 0, + 388, + }, + dictWord{7, 0, 644}, + dictWord{139, 0, 781}, + dictWord{132, 0, 849}, + dictWord{7, 0, 229}, + dictWord{8, 0, 59}, + dictWord{9, 0, 190}, + dictWord{10, 0, 378}, + dictWord{140, 0, 191}, + dictWord{7, 10, 381}, + dictWord{7, 10, 806}, + dictWord{7, 10, 820}, + dictWord{8, 10, 354}, + dictWord{8, 10, 437}, + dictWord{8, 10, 787}, + dictWord{9, 10, 657}, + dictWord{10, 10, 58}, + dictWord{10, 10, 339}, + dictWord{10, 10, 749}, + dictWord{11, 10, 914}, + dictWord{12, 10, 162}, + dictWord{13, 10, 75}, + dictWord{14, 10, 106}, + dictWord{14, 10, 198}, + dictWord{14, 10, 320}, + dictWord{14, 10, 413}, + dictWord{146, 10, 43}, + dictWord{141, 11, 306}, + dictWord{ + 136, + 10, + 747, + }, + dictWord{134, 0, 1115}, + dictWord{16, 0, 94}, + dictWord{16, 0, 108}, + dictWord{136, 11, 146}, + dictWord{6, 0, 700}, + dictWord{6, 0, 817}, + dictWord{ + 134, + 0, + 1002, + }, + dictWord{133, 10, 692}, + dictWord{4, 11, 465}, + dictWord{135, 11, 1663}, + dictWord{134, 10, 191}, + dictWord{6, 0, 1414}, + dictWord{ + 135, + 11, + 913, + }, + dictWord{132, 0, 660}, + dictWord{7, 0, 1035}, + dictWord{138, 0, 737}, + dictWord{6, 10, 162}, + dictWord{7, 10, 1960}, + dictWord{136, 10, 831}, + dictWord{ + 132, + 10, + 706, + }, + dictWord{7, 0, 690}, + dictWord{9, 0, 217}, + dictWord{9, 0, 587}, + dictWord{140, 0, 521}, + dictWord{138, 10, 426}, + dictWord{135, 10, 1235}, + dictWord{ + 6, + 11, + 82, + }, + dictWord{7, 11, 138}, + dictWord{7, 11, 517}, + dictWord{9, 11, 673}, + dictWord{139, 11, 238}, + dictWord{138, 0, 272}, + dictWord{5, 11, 495}, + dictWord{ + 7, + 11, + 834, + }, + dictWord{9, 11, 733}, + dictWord{139, 11, 378}, + dictWord{134, 0, 1744}, + dictWord{132, 0, 1011}, + dictWord{7, 11, 828}, + dictWord{142, 11, 116}, + dictWord{4, 0, 733}, + dictWord{9, 0, 194}, + dictWord{10, 0, 92}, + dictWord{11, 0, 198}, + dictWord{12, 0, 84}, + dictWord{13, 0, 128}, + dictWord{133, 11, 559}, + dictWord{ + 10, + 0, + 57, + }, + dictWord{10, 0, 277}, + dictWord{6, 11, 21}, + dictWord{6, 11, 1737}, + dictWord{7, 11, 1444}, + dictWord{136, 11, 224}, + dictWord{4, 10, 204}, + dictWord{ + 137, + 10, + 902, + }, + dictWord{136, 10, 833}, + dictWord{11, 0, 348}, + dictWord{12, 0, 99}, + dictWord{18, 0, 1}, + dictWord{18, 0, 11}, + dictWord{19, 0, 4}, + dictWord{7, 10, 366}, + dictWord{9, 10, 287}, + dictWord{12, 10, 199}, + dictWord{12, 10, 556}, + dictWord{140, 10, 577}, + dictWord{6, 0, 1981}, + dictWord{136, 0, 936}, + dictWord{ + 21, + 0, + 33, + }, + dictWord{150, 0, 40}, + dictWord{5, 11, 519}, + dictWord{138, 11, 204}, + dictWord{5, 10, 356}, + dictWord{135, 10, 224}, + dictWord{134, 0, 775}, + dictWord{ + 135, + 0, + 306, + }, + dictWord{7, 10, 630}, + dictWord{9, 10, 567}, + dictWord{11, 10, 150}, + dictWord{11, 10, 444}, + dictWord{141, 10, 119}, + dictWord{5, 0, 979}, + dictWord{ + 134, + 10, + 539, + }, + dictWord{133, 0, 611}, + dictWord{4, 11, 402}, + dictWord{135, 11, 1679}, + dictWord{5, 0, 178}, + dictWord{7, 11, 2}, + dictWord{8, 11, 323}, + dictWord{ + 136, + 11, + 479, + }, + dictWord{5, 11, 59}, + dictWord{135, 11, 672}, + dictWord{4, 0, 1010}, + dictWord{6, 0, 1969}, + dictWord{138, 11, 237}, + dictWord{133, 11, 412}, + dictWord{146, 11, 34}, + dictWord{7, 11, 1740}, + dictWord{146, 11, 48}, + dictWord{134, 0, 664}, + dictWord{139, 10, 814}, + dictWord{4, 11, 85}, + dictWord{ + 135, + 11, + 549, + }, + dictWord{133, 11, 94}, + dictWord{133, 11, 457}, + dictWord{132, 0, 390}, + dictWord{134, 0, 1510}, + dictWord{4, 10, 235}, + dictWord{135, 10, 255}, + dictWord{4, 10, 194}, + dictWord{5, 10, 584}, + dictWord{6, 11, 11}, + dictWord{6, 10, 384}, + dictWord{7, 11, 187}, + dictWord{7, 10, 583}, + dictWord{10, 10, 761}, + dictWord{ + 11, + 10, + 760, + }, + dictWord{139, 10, 851}, + dictWord{4, 11, 522}, + dictWord{139, 11, 802}, + dictWord{135, 0, 493}, + dictWord{10, 11, 776}, + dictWord{13, 11, 345}, + dictWord{142, 11, 425}, + dictWord{146, 0, 37}, + dictWord{4, 11, 52}, + dictWord{135, 11, 661}, + dictWord{134, 0, 724}, + dictWord{134, 0, 829}, + dictWord{ + 133, + 11, + 520, + }, + dictWord{133, 10, 562}, + dictWord{4, 11, 281}, + dictWord{5, 11, 38}, + dictWord{7, 11, 194}, + dictWord{7, 11, 668}, + dictWord{7, 11, 1893}, + dictWord{ + 137, + 11, + 397, + }, + dictWord{5, 10, 191}, + dictWord{137, 10, 271}, + dictWord{7, 0, 1537}, + dictWord{14, 0, 96}, + dictWord{143, 0, 73}, + dictWord{5, 0, 473}, + dictWord{ + 11, + 0, + 168, + }, + dictWord{4, 10, 470}, + dictWord{6, 10, 153}, + dictWord{7, 10, 1503}, + dictWord{7, 10, 1923}, + dictWord{10, 10, 701}, + dictWord{11, 10, 132}, + dictWord{ + 11, + 10, + 227, + }, + dictWord{11, 10, 320}, + dictWord{11, 10, 436}, + dictWord{11, 10, 525}, + dictWord{11, 10, 855}, + dictWord{12, 10, 41}, + dictWord{12, 10, 286}, + dictWord{13, 10, 103}, + dictWord{13, 10, 284}, + dictWord{14, 10, 255}, + dictWord{14, 10, 262}, + dictWord{15, 10, 117}, + dictWord{143, 10, 127}, + dictWord{ + 133, + 0, + 105, + }, + dictWord{5, 0, 438}, + dictWord{9, 0, 694}, + dictWord{12, 0, 627}, + dictWord{141, 0, 210}, + dictWord{133, 10, 327}, + dictWord{6, 10, 552}, + dictWord{ + 7, + 10, + 1754, + }, + dictWord{137, 10, 604}, + dictWord{134, 0, 1256}, + dictWord{152, 0, 11}, + dictWord{5, 11, 448}, + dictWord{11, 11, 98}, + dictWord{139, 11, 524}, + dictWord{ + 7, + 0, + 1626, + }, + dictWord{5, 10, 80}, + dictWord{6, 10, 405}, + dictWord{7, 10, 403}, + dictWord{7, 10, 1502}, + dictWord{8, 10, 456}, + dictWord{9, 10, 487}, + dictWord{ + 9, + 10, + 853, + }, + dictWord{9, 10, 889}, + dictWord{10, 10, 309}, + dictWord{11, 10, 721}, + dictWord{11, 10, 994}, + dictWord{12, 10, 430}, + dictWord{13, 10, 165}, + dictWord{ + 14, + 11, + 16, + }, + dictWord{146, 11, 44}, + dictWord{132, 0, 779}, + dictWord{8, 0, 25}, + dictWord{138, 0, 826}, + dictWord{4, 10, 453}, + dictWord{5, 10, 887}, + dictWord{ + 6, + 10, + 535, + }, + dictWord{8, 10, 6}, + dictWord{8, 10, 543}, + dictWord{136, 10, 826}, + dictWord{137, 11, 461}, + dictWord{140, 11, 632}, + dictWord{132, 0, 308}, + dictWord{135, 0, 741}, + dictWord{132, 0, 671}, + dictWord{7, 0, 150}, + dictWord{8, 0, 649}, + dictWord{136, 0, 1020}, + dictWord{9, 0, 99}, + dictWord{6, 11, 336}, + dictWord{ + 8, + 11, + 552, + }, + dictWord{9, 11, 285}, + dictWord{10, 11, 99}, + dictWord{139, 11, 568}, + dictWord{134, 0, 521}, + dictWord{5, 0, 339}, + dictWord{14, 0, 3}, + dictWord{ + 15, + 0, + 41, + }, + dictWord{15, 0, 166}, + dictWord{147, 0, 66}, + dictWord{6, 11, 423}, + dictWord{7, 11, 665}, + dictWord{7, 11, 1210}, + dictWord{9, 11, 218}, + dictWord{ + 141, + 11, + 222, + }, + dictWord{6, 0, 543}, + dictWord{5, 10, 101}, + dictWord{5, 11, 256}, + dictWord{6, 10, 88}, + dictWord{7, 10, 1677}, + dictWord{9, 10, 100}, + dictWord{10, 10, 677}, + dictWord{14, 10, 169}, + dictWord{14, 10, 302}, + dictWord{14, 10, 313}, + dictWord{15, 10, 48}, + dictWord{143, 10, 84}, + dictWord{4, 10, 310}, + dictWord{ + 7, + 10, + 708, + }, + dictWord{7, 10, 996}, + dictWord{9, 10, 795}, + dictWord{10, 10, 390}, + dictWord{10, 10, 733}, + dictWord{11, 10, 451}, + dictWord{12, 10, 249}, + dictWord{ + 14, + 10, + 115, + }, + dictWord{14, 10, 286}, + dictWord{143, 10, 100}, + dictWord{133, 10, 587}, + dictWord{13, 11, 417}, + dictWord{14, 11, 129}, + dictWord{143, 11, 15}, + dictWord{134, 0, 1358}, + dictWord{136, 11, 554}, + dictWord{132, 10, 498}, + dictWord{7, 10, 217}, + dictWord{8, 10, 140}, + dictWord{138, 10, 610}, + dictWord{ + 135, + 11, + 989, + }, + dictWord{135, 11, 634}, + dictWord{6, 0, 155}, + dictWord{140, 0, 234}, + dictWord{135, 11, 462}, + dictWord{132, 11, 618}, + dictWord{ + 134, + 0, + 1628, + }, + dictWord{132, 0, 766}, + dictWord{4, 11, 339}, + dictWord{5, 10, 905}, + dictWord{135, 11, 259}, + dictWord{135, 0, 829}, + dictWord{4, 11, 759}, + dictWord{ + 141, + 11, + 169, + }, + dictWord{7, 0, 1445}, + dictWord{4, 10, 456}, + dictWord{7, 10, 358}, + dictWord{7, 10, 1637}, + dictWord{8, 10, 643}, + dictWord{139, 10, 483}, + dictWord{ + 5, + 0, + 486, + }, + dictWord{135, 0, 1349}, + dictWord{5, 11, 688}, + dictWord{135, 11, 712}, + dictWord{7, 0, 1635}, + dictWord{8, 0, 17}, + dictWord{10, 0, 217}, + dictWord{ + 10, + 0, + 295, + }, + dictWord{12, 0, 2}, + dictWord{140, 11, 2}, + dictWord{138, 0, 558}, + dictWord{150, 10, 56}, + dictWord{4, 11, 278}, + dictWord{5, 11, 465}, + dictWord{ + 135, + 11, + 1367, + }, + dictWord{136, 11, 482}, + dictWord{133, 10, 535}, + dictWord{6, 0, 1362}, + dictWord{6, 0, 1461}, + dictWord{10, 11, 274}, + dictWord{10, 11, 625}, + dictWord{139, 11, 530}, + dictWord{5, 0, 599}, + dictWord{5, 11, 336}, + dictWord{6, 11, 341}, + dictWord{6, 11, 478}, + dictWord{6, 11, 1763}, + dictWord{136, 11, 386}, + dictWord{7, 10, 1748}, + dictWord{137, 11, 151}, + dictWord{134, 0, 1376}, + dictWord{133, 10, 539}, + dictWord{135, 11, 73}, + dictWord{135, 11, 1971}, + dictWord{139, 11, 283}, + dictWord{9, 0, 93}, + dictWord{139, 0, 474}, + dictWord{6, 10, 91}, + dictWord{135, 10, 435}, + dictWord{6, 0, 447}, + dictWord{5, 11, 396}, + dictWord{134, 11, 501}, + dictWord{4, 10, 16}, + dictWord{5, 10, 316}, + dictWord{5, 10, 842}, + dictWord{6, 10, 370}, + dictWord{6, 10, 1778}, + dictWord{8, 10, 166}, + dictWord{11, 10, 812}, + dictWord{12, 10, 206}, + dictWord{12, 10, 351}, + dictWord{14, 10, 418}, + dictWord{16, 10, 15}, + dictWord{16, 10, 34}, + dictWord{18, 10, 3}, + dictWord{19, 10, 3}, + dictWord{19, 10, 7}, + dictWord{20, 10, 4}, + dictWord{149, 10, 21}, + dictWord{7, 0, 577}, + dictWord{7, 0, 1432}, + dictWord{9, 0, 475}, + dictWord{9, 0, 505}, + dictWord{9, 0, 526}, + dictWord{9, 0, 609}, + dictWord{9, 0, 689}, + dictWord{9, 0, 726}, + dictWord{9, 0, 735}, + dictWord{9, 0, 738}, + dictWord{10, 0, 556}, + dictWord{ + 10, + 0, + 674, + }, + dictWord{10, 0, 684}, + dictWord{11, 0, 89}, + dictWord{11, 0, 202}, + dictWord{11, 0, 272}, + dictWord{11, 0, 380}, + dictWord{11, 0, 415}, + dictWord{11, 0, 505}, + dictWord{11, 0, 537}, + dictWord{11, 0, 550}, + dictWord{11, 0, 562}, + dictWord{11, 0, 640}, + dictWord{11, 0, 667}, + dictWord{11, 0, 688}, + dictWord{11, 0, 847}, + dictWord{11, 0, 927}, + dictWord{11, 0, 930}, + dictWord{11, 0, 940}, + dictWord{12, 0, 144}, + dictWord{12, 0, 325}, + dictWord{12, 0, 329}, + dictWord{12, 0, 389}, + dictWord{ + 12, + 0, + 403, + }, + dictWord{12, 0, 451}, + dictWord{12, 0, 515}, + dictWord{12, 0, 604}, + dictWord{12, 0, 616}, + dictWord{12, 0, 626}, + dictWord{13, 0, 66}, + dictWord{ + 13, + 0, + 131, + }, + dictWord{13, 0, 167}, + dictWord{13, 0, 236}, + dictWord{13, 0, 368}, + dictWord{13, 0, 411}, + dictWord{13, 0, 434}, + dictWord{13, 0, 453}, + dictWord{13, 0, 461}, + dictWord{13, 0, 474}, + dictWord{14, 0, 59}, + dictWord{14, 0, 60}, + dictWord{14, 0, 139}, + dictWord{14, 0, 152}, + dictWord{14, 0, 276}, + dictWord{14, 0, 353}, + dictWord{ + 14, + 0, + 402, + }, + dictWord{15, 0, 28}, + dictWord{15, 0, 81}, + dictWord{15, 0, 123}, + dictWord{15, 0, 152}, + dictWord{18, 0, 136}, + dictWord{148, 0, 88}, + dictWord{ + 4, + 11, + 929, + }, + dictWord{133, 11, 799}, + dictWord{136, 11, 46}, + dictWord{142, 0, 307}, + dictWord{4, 0, 609}, + dictWord{7, 0, 756}, + dictWord{9, 0, 544}, + dictWord{ + 11, + 0, + 413, + }, + dictWord{144, 0, 25}, + dictWord{10, 0, 687}, + dictWord{7, 10, 619}, + dictWord{10, 10, 547}, + dictWord{11, 10, 122}, + dictWord{140, 10, 601}, + dictWord{ + 4, + 0, + 930, + }, + dictWord{133, 0, 947}, + dictWord{133, 0, 939}, + dictWord{142, 0, 21}, + dictWord{4, 11, 892}, + dictWord{133, 11, 770}, + dictWord{133, 0, 962}, + dictWord{ + 5, + 0, + 651, + }, + dictWord{8, 0, 170}, + dictWord{9, 0, 61}, + dictWord{9, 0, 63}, + dictWord{10, 0, 23}, + dictWord{10, 0, 37}, + dictWord{10, 0, 834}, + dictWord{11, 0, 4}, + dictWord{ + 11, + 0, + 187, + }, + dictWord{11, 0, 281}, + dictWord{11, 0, 503}, + dictWord{11, 0, 677}, + dictWord{12, 0, 96}, + dictWord{12, 0, 130}, + dictWord{12, 0, 244}, + dictWord{14, 0, 5}, + dictWord{14, 0, 40}, + dictWord{14, 0, 162}, + dictWord{14, 0, 202}, + dictWord{146, 0, 133}, + dictWord{4, 0, 406}, + dictWord{5, 0, 579}, + dictWord{12, 0, 492}, + dictWord{ + 150, + 0, + 15, + }, + dictWord{135, 11, 158}, + dictWord{135, 0, 597}, + dictWord{132, 0, 981}, + dictWord{132, 10, 888}, + dictWord{4, 10, 149}, + dictWord{138, 10, 368}, + dictWord{132, 0, 545}, + dictWord{4, 10, 154}, + dictWord{7, 10, 1134}, + dictWord{136, 10, 105}, + dictWord{135, 11, 2001}, + dictWord{134, 0, 1558}, + dictWord{ + 4, + 10, + 31, + }, + dictWord{6, 10, 429}, + dictWord{7, 10, 962}, + dictWord{9, 10, 458}, + dictWord{139, 10, 691}, + dictWord{132, 10, 312}, + dictWord{135, 10, 1642}, + dictWord{ + 6, + 0, + 17, + }, + dictWord{6, 0, 1304}, + dictWord{7, 0, 16}, + dictWord{7, 0, 1001}, + dictWord{9, 0, 886}, + dictWord{10, 0, 489}, + dictWord{10, 0, 800}, + dictWord{11, 0, 782}, + dictWord{12, 0, 320}, + dictWord{13, 0, 467}, + dictWord{14, 0, 145}, + dictWord{14, 0, 387}, + dictWord{143, 0, 119}, + dictWord{135, 0, 1982}, + dictWord{17, 0, 17}, + dictWord{7, 11, 1461}, + dictWord{140, 11, 91}, + dictWord{4, 10, 236}, + dictWord{132, 11, 602}, + dictWord{138, 0, 907}, + dictWord{136, 0, 110}, + dictWord{7, 0, 272}, + dictWord{19, 0, 53}, + dictWord{5, 10, 836}, + dictWord{5, 10, 857}, + dictWord{134, 10, 1680}, + dictWord{5, 0, 458}, + dictWord{7, 11, 1218}, + dictWord{136, 11, 303}, + dictWord{7, 0, 1983}, + dictWord{8, 0, 0}, + dictWord{8, 0, 171}, + dictWord{9, 0, 120}, + dictWord{9, 0, 732}, + dictWord{10, 0, 473}, + dictWord{11, 0, 656}, + dictWord{ + 11, + 0, + 998, + }, + dictWord{18, 0, 0}, + dictWord{18, 0, 2}, + dictWord{19, 0, 21}, + dictWord{10, 10, 68}, + dictWord{139, 10, 494}, + dictWord{137, 11, 662}, + dictWord{4, 11, 13}, + dictWord{5, 11, 567}, + dictWord{7, 11, 1498}, + dictWord{9, 11, 124}, + dictWord{11, 11, 521}, + dictWord{140, 11, 405}, + dictWord{4, 10, 81}, + dictWord{139, 10, 867}, + dictWord{135, 11, 1006}, + dictWord{7, 11, 800}, + dictWord{7, 11, 1783}, + dictWord{138, 11, 12}, + dictWord{9, 0, 295}, + dictWord{10, 0, 443}, + dictWord{ + 5, + 10, + 282, + }, + dictWord{8, 10, 650}, + dictWord{137, 10, 907}, + dictWord{132, 11, 735}, + dictWord{4, 11, 170}, + dictWord{4, 10, 775}, + dictWord{135, 11, 323}, + dictWord{ + 6, + 0, + 1844, + }, + dictWord{10, 0, 924}, + dictWord{11, 11, 844}, + dictWord{12, 11, 104}, + dictWord{140, 11, 625}, + dictWord{5, 11, 304}, + dictWord{7, 11, 1403}, + dictWord{140, 11, 498}, + dictWord{134, 0, 1232}, + dictWord{4, 0, 519}, + dictWord{10, 0, 70}, + dictWord{12, 0, 26}, + dictWord{14, 0, 17}, + dictWord{14, 0, 178}, + dictWord{ + 15, + 0, + 34, + }, + dictWord{149, 0, 12}, + dictWord{132, 0, 993}, + dictWord{4, 11, 148}, + dictWord{133, 11, 742}, + dictWord{6, 0, 31}, + dictWord{7, 0, 491}, + dictWord{7, 0, 530}, + dictWord{8, 0, 592}, + dictWord{11, 0, 53}, + dictWord{11, 0, 779}, + dictWord{12, 0, 167}, + dictWord{12, 0, 411}, + dictWord{14, 0, 14}, + dictWord{14, 0, 136}, + dictWord{ + 15, + 0, + 72, + }, + dictWord{16, 0, 17}, + dictWord{144, 0, 72}, + dictWord{133, 0, 907}, + dictWord{134, 0, 733}, + dictWord{133, 11, 111}, + dictWord{4, 10, 71}, + dictWord{ + 5, + 10, + 376, + }, + dictWord{7, 10, 119}, + dictWord{138, 10, 665}, + dictWord{136, 0, 55}, + dictWord{8, 0, 430}, + dictWord{136, 11, 430}, + dictWord{4, 0, 208}, + dictWord{ + 5, + 0, + 106, + }, + dictWord{6, 0, 531}, + dictWord{8, 0, 408}, + dictWord{9, 0, 188}, + dictWord{138, 0, 572}, + dictWord{12, 0, 56}, + dictWord{11, 10, 827}, + dictWord{14, 10, 34}, + dictWord{143, 10, 148}, + dictWord{134, 0, 1693}, + dictWord{133, 11, 444}, + dictWord{132, 10, 479}, + dictWord{140, 0, 441}, + dictWord{9, 0, 449}, + dictWord{ + 10, + 0, + 192, + }, + dictWord{138, 0, 740}, + dictWord{134, 0, 928}, + dictWord{4, 0, 241}, + dictWord{7, 10, 607}, + dictWord{136, 10, 99}, + dictWord{8, 11, 123}, + dictWord{ + 15, + 11, + 6, + }, + dictWord{144, 11, 7}, + dictWord{6, 11, 285}, + dictWord{8, 11, 654}, + dictWord{11, 11, 749}, + dictWord{12, 11, 190}, + dictWord{12, 11, 327}, + dictWord{ + 13, + 11, + 120, + }, + dictWord{13, 11, 121}, + dictWord{13, 11, 327}, + dictWord{15, 11, 47}, + dictWord{146, 11, 40}, + dictWord{4, 10, 41}, + dictWord{5, 10, 74}, + dictWord{ + 7, + 10, + 1627, + }, + dictWord{11, 10, 871}, + dictWord{140, 10, 619}, + dictWord{7, 0, 1525}, + dictWord{11, 10, 329}, + dictWord{11, 10, 965}, + dictWord{12, 10, 241}, + dictWord{14, 10, 354}, + dictWord{15, 10, 22}, + dictWord{148, 10, 63}, + dictWord{132, 0, 259}, + dictWord{135, 11, 183}, + dictWord{9, 10, 209}, + dictWord{ + 137, + 10, + 300, + }, + dictWord{5, 11, 937}, + dictWord{135, 11, 100}, + dictWord{133, 10, 98}, + dictWord{4, 0, 173}, + dictWord{5, 0, 312}, + dictWord{5, 0, 512}, + dictWord{ + 135, + 0, + 1285, + }, + dictWord{141, 0, 185}, + dictWord{7, 0, 1603}, + dictWord{7, 0, 1691}, + dictWord{9, 0, 464}, + dictWord{11, 0, 195}, + dictWord{12, 0, 279}, + dictWord{ + 12, + 0, + 448, + }, + dictWord{14, 0, 11}, + dictWord{147, 0, 102}, + dictWord{135, 0, 1113}, + dictWord{133, 10, 984}, + dictWord{4, 0, 452}, + dictWord{5, 0, 583}, + dictWord{ + 135, + 0, + 720, + }, + dictWord{4, 0, 547}, + dictWord{5, 0, 817}, + dictWord{6, 0, 433}, + dictWord{7, 0, 593}, + dictWord{7, 0, 1378}, + dictWord{8, 0, 161}, + dictWord{9, 0, 284}, + dictWord{ + 10, + 0, + 313, + }, + dictWord{139, 0, 886}, + dictWord{8, 0, 722}, + dictWord{4, 10, 182}, + dictWord{6, 10, 205}, + dictWord{135, 10, 220}, + dictWord{150, 0, 13}, + dictWord{ + 4, + 10, + 42, + }, + dictWord{9, 10, 205}, + dictWord{9, 10, 786}, + dictWord{138, 10, 659}, + dictWord{6, 0, 289}, + dictWord{7, 0, 1670}, + dictWord{12, 0, 57}, + dictWord{151, 0, 4}, + dictWord{132, 10, 635}, + dictWord{14, 0, 43}, + dictWord{146, 0, 21}, + dictWord{139, 10, 533}, + dictWord{135, 0, 1694}, + dictWord{8, 0, 420}, + dictWord{ + 139, + 0, + 193, + }, + dictWord{135, 0, 409}, + dictWord{132, 10, 371}, + dictWord{4, 10, 272}, + dictWord{135, 10, 836}, + dictWord{5, 10, 825}, + dictWord{134, 10, 1640}, + dictWord{5, 11, 251}, + dictWord{5, 11, 956}, + dictWord{8, 11, 268}, + dictWord{9, 11, 214}, + dictWord{146, 11, 142}, + dictWord{138, 0, 308}, + dictWord{6, 0, 1863}, + dictWord{141, 11, 37}, + dictWord{137, 10, 879}, + dictWord{7, 10, 317}, + dictWord{135, 10, 569}, + dictWord{132, 11, 294}, + dictWord{134, 0, 790}, + dictWord{ + 5, + 0, + 1002, + }, + dictWord{136, 0, 745}, + dictWord{5, 11, 346}, + dictWord{5, 11, 711}, + dictWord{136, 11, 390}, + dictWord{135, 0, 289}, + dictWord{5, 0, 504}, + dictWord{ + 11, + 0, + 68, + }, + dictWord{137, 10, 307}, + dictWord{4, 0, 239}, + dictWord{6, 0, 477}, + dictWord{7, 0, 1607}, + dictWord{139, 0, 617}, + dictWord{149, 0, 13}, + dictWord{ + 133, + 0, + 609, + }, + dictWord{133, 11, 624}, + dictWord{5, 11, 783}, + dictWord{7, 11, 1998}, + dictWord{135, 11, 2047}, + dictWord{133, 10, 525}, + dictWord{132, 0, 367}, + dictWord{132, 11, 594}, + dictWord{6, 0, 528}, + dictWord{133, 10, 493}, + dictWord{4, 10, 174}, + dictWord{135, 10, 911}, + dictWord{8, 10, 417}, + dictWord{ + 137, + 10, + 782, + }, + dictWord{132, 0, 694}, + dictWord{7, 0, 548}, + dictWord{137, 0, 58}, + dictWord{4, 10, 32}, + dictWord{5, 10, 215}, + dictWord{6, 10, 269}, + dictWord{7, 10, 1782}, + dictWord{7, 10, 1892}, + dictWord{10, 10, 16}, + dictWord{11, 10, 822}, + dictWord{11, 10, 954}, + dictWord{141, 10, 481}, + dictWord{140, 0, 687}, + dictWord{ + 7, + 0, + 1749, + }, + dictWord{136, 10, 477}, + dictWord{132, 11, 569}, + dictWord{133, 10, 308}, + dictWord{135, 10, 1088}, + dictWord{4, 0, 661}, + dictWord{138, 0, 1004}, + dictWord{5, 11, 37}, + dictWord{6, 11, 39}, + dictWord{6, 11, 451}, + dictWord{7, 11, 218}, + dictWord{7, 11, 667}, + dictWord{7, 11, 1166}, + dictWord{7, 11, 1687}, + dictWord{8, 11, 662}, + dictWord{144, 11, 2}, + dictWord{9, 0, 445}, + dictWord{12, 0, 53}, + dictWord{13, 0, 492}, + dictWord{5, 10, 126}, + dictWord{8, 10, 297}, + dictWord{ + 9, + 10, + 366, + }, + dictWord{140, 10, 374}, + dictWord{7, 10, 1551}, + dictWord{139, 10, 361}, + dictWord{148, 0, 74}, + dictWord{134, 11, 508}, + dictWord{135, 0, 213}, + dictWord{132, 10, 175}, + dictWord{132, 10, 685}, + dictWord{6, 0, 760}, + dictWord{6, 0, 834}, + dictWord{134, 0, 1248}, + dictWord{7, 11, 453}, + dictWord{7, 11, 635}, + dictWord{7, 11, 796}, + dictWord{8, 11, 331}, + dictWord{9, 11, 328}, + dictWord{9, 11, 330}, + dictWord{9, 11, 865}, + dictWord{10, 11, 119}, + dictWord{10, 11, 235}, + dictWord{11, 11, 111}, + dictWord{11, 11, 129}, + dictWord{11, 11, 240}, + dictWord{12, 11, 31}, + dictWord{12, 11, 66}, + dictWord{12, 11, 222}, + dictWord{12, 11, 269}, + dictWord{12, 11, 599}, + dictWord{12, 11, 689}, + dictWord{13, 11, 186}, + dictWord{13, 11, 364}, + dictWord{142, 11, 345}, + dictWord{7, 0, 1672}, + dictWord{ + 139, + 0, + 189, + }, + dictWord{133, 10, 797}, + dictWord{133, 10, 565}, + dictWord{6, 0, 1548}, + dictWord{6, 11, 98}, + dictWord{7, 11, 585}, + dictWord{135, 11, 702}, + dictWord{ + 9, + 0, + 968, + }, + dictWord{15, 0, 192}, + dictWord{149, 0, 56}, + dictWord{4, 10, 252}, + dictWord{6, 11, 37}, + dictWord{7, 11, 299}, + dictWord{7, 10, 1068}, + dictWord{ + 7, + 11, + 1666, + }, + dictWord{8, 11, 195}, + dictWord{8, 11, 316}, + dictWord{9, 11, 178}, + dictWord{9, 11, 276}, + dictWord{9, 11, 339}, + dictWord{9, 11, 536}, + dictWord{ + 10, + 11, + 102, + }, + dictWord{10, 11, 362}, + dictWord{10, 10, 434}, + dictWord{10, 11, 785}, + dictWord{11, 11, 55}, + dictWord{11, 11, 149}, + dictWord{11, 10, 228}, + dictWord{ + 11, + 10, + 426, + }, + dictWord{11, 11, 773}, + dictWord{13, 10, 231}, + dictWord{13, 11, 416}, + dictWord{13, 11, 419}, + dictWord{14, 11, 38}, + dictWord{14, 11, 41}, + dictWord{14, 11, 210}, + dictWord{18, 10, 106}, + dictWord{148, 10, 87}, + dictWord{4, 0, 751}, + dictWord{11, 0, 390}, + dictWord{140, 0, 32}, + dictWord{4, 0, 409}, + dictWord{133, 0, 78}, + dictWord{11, 11, 458}, + dictWord{12, 11, 15}, + dictWord{140, 11, 432}, + dictWord{7, 0, 1602}, + dictWord{10, 0, 257}, + dictWord{10, 0, 698}, + dictWord{11, 0, 544}, + dictWord{11, 0, 585}, + dictWord{12, 0, 212}, + dictWord{13, 0, 307}, + dictWord{5, 10, 231}, + dictWord{7, 10, 601}, + dictWord{9, 10, 277}, + dictWord{ + 9, + 10, + 674, + }, + dictWord{10, 10, 178}, + dictWord{10, 10, 418}, + dictWord{10, 10, 509}, + dictWord{11, 10, 531}, + dictWord{12, 10, 113}, + dictWord{12, 10, 475}, + dictWord{13, 10, 99}, + dictWord{142, 10, 428}, + dictWord{6, 0, 473}, + dictWord{145, 0, 105}, + dictWord{6, 0, 1949}, + dictWord{15, 0, 156}, + dictWord{133, 11, 645}, + dictWord{7, 10, 1591}, + dictWord{144, 10, 43}, + dictWord{135, 0, 1779}, + dictWord{135, 10, 1683}, + dictWord{4, 11, 290}, + dictWord{135, 11, 1356}, + dictWord{134, 0, 763}, + dictWord{6, 11, 70}, + dictWord{7, 11, 1292}, + dictWord{10, 11, 762}, + dictWord{139, 11, 288}, + dictWord{142, 0, 29}, + dictWord{140, 11, 428}, + dictWord{7, 0, 883}, + dictWord{7, 11, 131}, + dictWord{7, 11, 422}, + dictWord{8, 11, 210}, + dictWord{140, 11, 573}, + dictWord{134, 0, 488}, + dictWord{4, 10, 399}, + dictWord{5, 10, 119}, + dictWord{5, 10, 494}, + dictWord{7, 10, 751}, + dictWord{137, 10, 556}, + dictWord{133, 0, 617}, + dictWord{132, 11, 936}, + dictWord{ + 139, + 0, + 50, + }, + dictWord{7, 0, 1518}, + dictWord{139, 0, 694}, + dictWord{137, 0, 785}, + dictWord{4, 0, 546}, + dictWord{135, 0, 2042}, + dictWord{7, 11, 716}, + dictWord{ + 13, + 11, + 97, + }, + dictWord{141, 11, 251}, + dictWord{132, 11, 653}, + dictWord{145, 0, 22}, + dictWord{134, 0, 1016}, + dictWord{4, 0, 313}, + dictWord{133, 0, 577}, + dictWord{ + 136, + 11, + 657, + }, + dictWord{8, 0, 184}, + dictWord{141, 0, 433}, + dictWord{135, 0, 935}, + dictWord{6, 0, 720}, + dictWord{9, 0, 114}, + dictWord{146, 11, 80}, + dictWord{ + 12, + 0, + 186, + }, + dictWord{12, 0, 292}, + dictWord{14, 0, 100}, + dictWord{18, 0, 70}, + dictWord{7, 10, 594}, + dictWord{7, 10, 851}, + dictWord{7, 10, 1858}, + dictWord{ + 9, + 10, + 411, + }, + dictWord{9, 10, 574}, + dictWord{9, 10, 666}, + dictWord{9, 10, 737}, + dictWord{10, 10, 346}, + dictWord{10, 10, 712}, + dictWord{11, 10, 246}, + dictWord{ + 11, + 10, + 432, + }, + dictWord{11, 10, 517}, + dictWord{11, 10, 647}, + dictWord{11, 10, 679}, + dictWord{11, 10, 727}, + dictWord{12, 10, 304}, + dictWord{12, 10, 305}, + dictWord{12, 10, 323}, + dictWord{12, 10, 483}, + dictWord{12, 10, 572}, + dictWord{12, 10, 593}, + dictWord{12, 10, 602}, + dictWord{13, 10, 95}, + dictWord{13, 10, 101}, + dictWord{13, 10, 171}, + dictWord{13, 10, 315}, + dictWord{13, 10, 378}, + dictWord{13, 10, 425}, + dictWord{13, 10, 475}, + dictWord{14, 10, 63}, + dictWord{ + 14, + 10, + 380, + }, + dictWord{14, 10, 384}, + dictWord{15, 10, 133}, + dictWord{18, 10, 112}, + dictWord{148, 10, 72}, + dictWord{135, 10, 1093}, + dictWord{135, 11, 1836}, + dictWord{132, 10, 679}, + dictWord{137, 10, 203}, + dictWord{11, 0, 402}, + dictWord{12, 0, 109}, + dictWord{12, 0, 431}, + dictWord{13, 0, 179}, + dictWord{13, 0, 206}, + dictWord{14, 0, 217}, + dictWord{16, 0, 3}, + dictWord{148, 0, 53}, + dictWord{7, 11, 1368}, + dictWord{8, 11, 232}, + dictWord{8, 11, 361}, + dictWord{10, 11, 682}, + dictWord{138, 11, 742}, + dictWord{137, 10, 714}, + dictWord{5, 0, 886}, + dictWord{6, 0, 46}, + dictWord{6, 0, 1790}, + dictWord{7, 0, 14}, + dictWord{7, 0, 732}, + dictWord{ + 7, + 0, + 1654, + }, + dictWord{8, 0, 95}, + dictWord{8, 0, 327}, + dictWord{8, 0, 616}, + dictWord{9, 0, 892}, + dictWord{10, 0, 598}, + dictWord{10, 0, 769}, + dictWord{11, 0, 134}, + dictWord{11, 0, 747}, + dictWord{12, 0, 378}, + dictWord{14, 0, 97}, + dictWord{137, 11, 534}, + dictWord{4, 0, 969}, + dictWord{136, 10, 825}, + dictWord{137, 11, 27}, + dictWord{6, 0, 727}, + dictWord{142, 11, 12}, + dictWord{133, 0, 1021}, + dictWord{134, 0, 1190}, + dictWord{134, 11, 1657}, + dictWord{5, 10, 143}, + dictWord{ + 5, + 10, + 769, + }, + dictWord{6, 10, 1760}, + dictWord{7, 10, 682}, + dictWord{7, 10, 1992}, + dictWord{136, 10, 736}, + dictWord{132, 0, 153}, + dictWord{135, 11, 127}, + dictWord{133, 0, 798}, + dictWord{132, 0, 587}, + dictWord{6, 0, 598}, + dictWord{7, 0, 42}, + dictWord{8, 0, 695}, + dictWord{10, 0, 212}, + dictWord{11, 0, 158}, + dictWord{ + 14, + 0, + 196, + }, + dictWord{145, 0, 85}, + dictWord{133, 10, 860}, + dictWord{6, 0, 1929}, + dictWord{134, 0, 1933}, + dictWord{5, 0, 957}, + dictWord{5, 0, 1008}, + dictWord{ + 9, + 0, + 577, + }, + dictWord{12, 0, 141}, + dictWord{6, 10, 422}, + dictWord{7, 10, 0}, + dictWord{7, 10, 1544}, + dictWord{8, 11, 364}, + dictWord{11, 10, 990}, + dictWord{ + 12, + 10, + 453, + }, + dictWord{13, 10, 47}, + dictWord{141, 10, 266}, + dictWord{134, 0, 1319}, + dictWord{4, 0, 129}, + dictWord{135, 0, 465}, + dictWord{7, 0, 470}, + dictWord{ + 7, + 0, + 1057, + }, + dictWord{7, 0, 1201}, + dictWord{9, 0, 755}, + dictWord{11, 0, 906}, + dictWord{140, 0, 527}, + dictWord{7, 0, 908}, + dictWord{146, 0, 7}, + dictWord{5, 0, 148}, + dictWord{136, 0, 450}, + dictWord{5, 10, 515}, + dictWord{137, 10, 131}, + dictWord{7, 10, 1605}, + dictWord{11, 10, 962}, + dictWord{146, 10, 139}, + dictWord{ + 132, + 10, + 646, + }, + dictWord{134, 0, 1166}, + dictWord{4, 10, 396}, + dictWord{7, 10, 728}, + dictWord{9, 10, 117}, + dictWord{13, 10, 202}, + dictWord{148, 10, 51}, + dictWord{ + 6, + 10, + 121, + }, + dictWord{6, 10, 124}, + dictWord{6, 10, 357}, + dictWord{7, 10, 1138}, + dictWord{7, 10, 1295}, + dictWord{8, 10, 162}, + dictWord{139, 10, 655}, + dictWord{14, 0, 374}, + dictWord{142, 11, 374}, + dictWord{138, 0, 253}, + dictWord{139, 0, 1003}, + dictWord{5, 11, 909}, + dictWord{9, 11, 849}, + dictWord{ + 138, + 11, + 805, + }, + dictWord{133, 10, 237}, + dictWord{7, 11, 525}, + dictWord{7, 11, 1579}, + dictWord{8, 11, 497}, + dictWord{136, 11, 573}, + dictWord{137, 0, 46}, + dictWord{ + 132, + 0, + 879, + }, + dictWord{134, 0, 806}, + dictWord{135, 0, 1868}, + dictWord{6, 0, 1837}, + dictWord{134, 0, 1846}, + dictWord{6, 0, 730}, + dictWord{134, 0, 881}, + dictWord{7, 0, 965}, + dictWord{7, 0, 1460}, + dictWord{7, 0, 1604}, + dictWord{7, 11, 193}, + dictWord{7, 11, 397}, + dictWord{7, 11, 1105}, + dictWord{8, 11, 124}, + dictWord{ + 8, + 11, + 619, + }, + dictWord{9, 11, 305}, + dictWord{10, 11, 264}, + dictWord{11, 11, 40}, + dictWord{12, 11, 349}, + dictWord{13, 11, 134}, + dictWord{13, 11, 295}, + dictWord{14, 11, 155}, + dictWord{15, 11, 120}, + dictWord{146, 11, 105}, + dictWord{136, 0, 506}, + dictWord{143, 0, 10}, + dictWord{4, 11, 262}, + dictWord{7, 11, 342}, + dictWord{7, 10, 571}, + dictWord{7, 10, 1877}, + dictWord{10, 10, 366}, + dictWord{141, 11, 23}, + dictWord{133, 11, 641}, + dictWord{10, 0, 22}, + dictWord{9, 10, 513}, + dictWord{10, 10, 39}, + dictWord{12, 10, 122}, + dictWord{140, 10, 187}, + dictWord{135, 11, 1431}, + dictWord{150, 11, 49}, + dictWord{4, 11, 99}, + dictWord{ + 6, + 11, + 250, + }, + dictWord{6, 11, 346}, + dictWord{8, 11, 127}, + dictWord{138, 11, 81}, + dictWord{6, 0, 2014}, + dictWord{8, 0, 928}, + dictWord{10, 0, 960}, + dictWord{10, 0, 979}, + dictWord{140, 0, 996}, + dictWord{134, 0, 296}, + dictWord{132, 11, 915}, + dictWord{5, 11, 75}, + dictWord{9, 11, 517}, + dictWord{10, 11, 470}, + dictWord{ + 12, + 11, + 155, + }, + dictWord{141, 11, 224}, + dictWord{137, 10, 873}, + dictWord{4, 0, 854}, + dictWord{140, 11, 18}, + dictWord{134, 0, 587}, + dictWord{7, 10, 107}, + dictWord{ + 7, + 10, + 838, + }, + dictWord{8, 10, 550}, + dictWord{138, 10, 401}, + dictWord{11, 0, 636}, + dictWord{15, 0, 145}, + dictWord{17, 0, 34}, + dictWord{19, 0, 50}, + dictWord{ + 23, + 0, + 20, + }, + dictWord{11, 10, 588}, + dictWord{11, 10, 864}, + dictWord{11, 10, 968}, + dictWord{143, 10, 160}, + dictWord{135, 11, 216}, + dictWord{7, 0, 982}, + dictWord{ + 10, + 0, + 32, + }, + dictWord{143, 0, 56}, + dictWord{133, 10, 768}, + dictWord{133, 11, 954}, + dictWord{6, 11, 304}, + dictWord{7, 11, 1114}, + dictWord{8, 11, 418}, + dictWord{ + 10, + 11, + 345, + }, + dictWord{11, 11, 341}, + dictWord{11, 11, 675}, + dictWord{141, 11, 40}, + dictWord{9, 11, 410}, + dictWord{139, 11, 425}, + dictWord{136, 0, 941}, + dictWord{5, 0, 435}, + dictWord{132, 10, 894}, + dictWord{5, 0, 85}, + dictWord{6, 0, 419}, + dictWord{7, 0, 134}, + dictWord{7, 0, 305}, + dictWord{7, 0, 361}, + dictWord{ + 7, + 0, + 1337, + }, + dictWord{8, 0, 71}, + dictWord{140, 0, 519}, + dictWord{140, 0, 688}, + dictWord{135, 0, 740}, + dictWord{5, 0, 691}, + dictWord{7, 0, 345}, + dictWord{9, 0, 94}, + dictWord{140, 0, 169}, + dictWord{5, 0, 183}, + dictWord{6, 0, 582}, + dictWord{10, 0, 679}, + dictWord{140, 0, 435}, + dictWord{134, 11, 14}, + dictWord{6, 0, 945}, + dictWord{135, 0, 511}, + dictWord{134, 11, 1708}, + dictWord{5, 11, 113}, + dictWord{6, 11, 243}, + dictWord{7, 11, 1865}, + dictWord{11, 11, 161}, + dictWord{16, 11, 37}, + dictWord{145, 11, 99}, + dictWord{132, 11, 274}, + dictWord{137, 0, 539}, + dictWord{7, 0, 1993}, + dictWord{8, 0, 684}, + dictWord{134, 10, 272}, + dictWord{ + 6, + 0, + 659, + }, + dictWord{134, 0, 982}, + dictWord{4, 10, 9}, + dictWord{5, 10, 128}, + dictWord{7, 10, 368}, + dictWord{11, 10, 480}, + dictWord{148, 10, 3}, + dictWord{ + 134, + 0, + 583, + }, + dictWord{132, 0, 803}, + dictWord{133, 0, 704}, + dictWord{4, 0, 179}, + dictWord{5, 0, 198}, + dictWord{133, 0, 697}, + dictWord{7, 0, 347}, + dictWord{7, 0, 971}, + dictWord{8, 0, 181}, + dictWord{10, 0, 711}, + dictWord{135, 11, 166}, + dictWord{136, 10, 682}, + dictWord{4, 10, 2}, + dictWord{7, 10, 545}, + dictWord{7, 10, 894}, + dictWord{136, 11, 521}, + dictWord{135, 0, 481}, + dictWord{132, 0, 243}, + dictWord{5, 0, 203}, + dictWord{7, 0, 19}, + dictWord{7, 0, 71}, + dictWord{7, 0, 113}, + dictWord{ + 10, + 0, + 405, + }, + dictWord{11, 0, 357}, + dictWord{142, 0, 240}, + dictWord{5, 11, 725}, + dictWord{5, 11, 727}, + dictWord{135, 11, 1811}, + dictWord{6, 0, 826}, + dictWord{ + 137, + 11, + 304, + }, + dictWord{7, 0, 1450}, + dictWord{139, 0, 99}, + dictWord{133, 11, 654}, + dictWord{134, 0, 492}, + dictWord{5, 0, 134}, + dictWord{6, 0, 408}, + dictWord{ + 6, + 0, + 495, + }, + dictWord{7, 0, 1593}, + dictWord{6, 11, 273}, + dictWord{10, 11, 188}, + dictWord{13, 11, 377}, + dictWord{146, 11, 77}, + dictWord{9, 10, 769}, + dictWord{ + 140, + 10, + 185, + }, + dictWord{135, 11, 410}, + dictWord{142, 0, 4}, + dictWord{4, 0, 665}, + dictWord{134, 11, 1785}, + dictWord{4, 0, 248}, + dictWord{7, 0, 137}, + dictWord{ + 137, + 0, + 349, + }, + dictWord{5, 10, 530}, + dictWord{142, 10, 113}, + dictWord{7, 0, 1270}, + dictWord{139, 0, 612}, + dictWord{132, 11, 780}, + dictWord{5, 0, 371}, + dictWord{135, 0, 563}, + dictWord{135, 0, 826}, + dictWord{6, 0, 1535}, + dictWord{23, 0, 21}, + dictWord{151, 0, 23}, + dictWord{4, 0, 374}, + dictWord{7, 0, 547}, + dictWord{ + 7, + 0, + 1700, + }, + dictWord{7, 0, 1833}, + dictWord{139, 0, 858}, + dictWord{133, 10, 556}, + dictWord{7, 11, 612}, + dictWord{8, 11, 545}, + dictWord{8, 11, 568}, + dictWord{ + 8, + 11, + 642, + }, + dictWord{9, 11, 717}, + dictWord{10, 11, 541}, + dictWord{10, 11, 763}, + dictWord{11, 11, 449}, + dictWord{12, 11, 489}, + dictWord{13, 11, 153}, + dictWord{ + 13, + 11, + 296, + }, + dictWord{14, 11, 138}, + dictWord{14, 11, 392}, + dictWord{15, 11, 50}, + dictWord{16, 11, 6}, + dictWord{16, 11, 12}, + dictWord{148, 11, 9}, + dictWord{ + 9, + 0, + 311, + }, + dictWord{141, 0, 42}, + dictWord{8, 10, 16}, + dictWord{140, 10, 568}, + dictWord{6, 0, 1968}, + dictWord{6, 0, 2027}, + dictWord{138, 0, 991}, + dictWord{ + 6, + 0, + 1647, + }, + dictWord{7, 0, 1552}, + dictWord{7, 0, 2010}, + dictWord{9, 0, 494}, + dictWord{137, 0, 509}, + dictWord{133, 11, 948}, + dictWord{6, 10, 186}, + dictWord{ + 137, + 10, + 426, + }, + dictWord{134, 0, 769}, + dictWord{134, 0, 642}, + dictWord{132, 10, 585}, + dictWord{6, 0, 123}, + dictWord{7, 0, 214}, + dictWord{9, 0, 728}, + dictWord{ + 10, + 0, + 157, + }, + dictWord{11, 0, 346}, + dictWord{11, 0, 662}, + dictWord{143, 0, 106}, + dictWord{142, 11, 381}, + dictWord{135, 0, 1435}, + dictWord{4, 11, 532}, + dictWord{ + 5, + 11, + 706, + }, + dictWord{135, 11, 662}, + dictWord{5, 11, 837}, + dictWord{134, 11, 1651}, + dictWord{4, 10, 93}, + dictWord{5, 10, 252}, + dictWord{6, 10, 229}, + dictWord{ + 7, + 10, + 291, + }, + dictWord{9, 10, 550}, + dictWord{139, 10, 644}, + dictWord{148, 0, 79}, + dictWord{137, 10, 749}, + dictWord{134, 0, 1425}, + dictWord{ + 137, + 10, + 162, + }, + dictWord{4, 11, 362}, + dictWord{7, 11, 52}, + dictWord{7, 11, 303}, + dictWord{140, 11, 166}, + dictWord{132, 10, 381}, + dictWord{4, 11, 330}, + dictWord{ + 7, + 11, + 933, + }, + dictWord{7, 11, 2012}, + dictWord{136, 11, 292}, + dictWord{135, 11, 767}, + dictWord{4, 0, 707}, + dictWord{5, 0, 588}, + dictWord{6, 0, 393}, + dictWord{ + 13, + 0, + 106, + }, + dictWord{18, 0, 49}, + dictWord{147, 0, 41}, + dictWord{6, 0, 211}, + dictWord{7, 0, 1690}, + dictWord{11, 0, 486}, + dictWord{140, 0, 369}, + dictWord{ + 137, + 11, + 883, + }, + dictWord{4, 11, 703}, + dictWord{135, 11, 207}, + dictWord{4, 0, 187}, + dictWord{5, 0, 184}, + dictWord{5, 0, 690}, + dictWord{7, 0, 1869}, + dictWord{10, 0, 756}, + dictWord{139, 0, 783}, + dictWord{132, 11, 571}, + dictWord{134, 0, 1382}, + dictWord{5, 0, 175}, + dictWord{6, 10, 77}, + dictWord{6, 10, 157}, + dictWord{7, 10, 974}, + dictWord{7, 10, 1301}, + dictWord{7, 10, 1339}, + dictWord{7, 10, 1490}, + dictWord{7, 10, 1873}, + dictWord{137, 10, 628}, + dictWord{134, 0, 1493}, + dictWord{ + 5, + 11, + 873, + }, + dictWord{133, 11, 960}, + dictWord{134, 0, 1007}, + dictWord{12, 11, 93}, + dictWord{12, 11, 501}, + dictWord{13, 11, 362}, + dictWord{14, 11, 151}, + dictWord{15, 11, 40}, + dictWord{15, 11, 59}, + dictWord{16, 11, 46}, + dictWord{17, 11, 25}, + dictWord{18, 11, 14}, + dictWord{18, 11, 134}, + dictWord{19, 11, 25}, + dictWord{ + 19, + 11, + 69, + }, + dictWord{20, 11, 16}, + dictWord{20, 11, 19}, + dictWord{20, 11, 66}, + dictWord{21, 11, 23}, + dictWord{21, 11, 25}, + dictWord{150, 11, 42}, + dictWord{ + 11, + 10, + 919, + }, + dictWord{141, 10, 409}, + dictWord{134, 0, 219}, + dictWord{5, 0, 582}, + dictWord{6, 0, 1646}, + dictWord{7, 0, 99}, + dictWord{7, 0, 1962}, + dictWord{ + 7, + 0, + 1986, + }, + dictWord{8, 0, 515}, + dictWord{8, 0, 773}, + dictWord{9, 0, 23}, + dictWord{9, 0, 491}, + dictWord{12, 0, 620}, + dictWord{142, 0, 93}, + dictWord{133, 0, 851}, + dictWord{5, 11, 33}, + dictWord{134, 11, 470}, + dictWord{135, 11, 1291}, + dictWord{134, 0, 1278}, + dictWord{135, 11, 1882}, + dictWord{135, 10, 1489}, + dictWord{132, 0, 1000}, + dictWord{138, 0, 982}, + dictWord{8, 0, 762}, + dictWord{8, 0, 812}, + dictWord{137, 0, 910}, + dictWord{6, 11, 47}, + dictWord{7, 11, 90}, + dictWord{ + 7, + 11, + 664, + }, + dictWord{7, 11, 830}, + dictWord{7, 11, 1380}, + dictWord{7, 11, 2025}, + dictWord{8, 11, 448}, + dictWord{136, 11, 828}, + dictWord{4, 0, 98}, + dictWord{ + 4, + 0, + 940, + }, + dictWord{6, 0, 1819}, + dictWord{6, 0, 1834}, + dictWord{6, 0, 1841}, + dictWord{7, 0, 1365}, + dictWord{8, 0, 859}, + dictWord{8, 0, 897}, + dictWord{8, 0, 918}, + dictWord{9, 0, 422}, + dictWord{9, 0, 670}, + dictWord{10, 0, 775}, + dictWord{10, 0, 894}, + dictWord{10, 0, 909}, + dictWord{10, 0, 910}, + dictWord{10, 0, 935}, + dictWord{ + 11, + 0, + 210, + }, + dictWord{12, 0, 750}, + dictWord{12, 0, 755}, + dictWord{13, 0, 26}, + dictWord{13, 0, 457}, + dictWord{13, 0, 476}, + dictWord{16, 0, 100}, + dictWord{16, 0, 109}, + dictWord{18, 0, 173}, + dictWord{18, 0, 175}, + dictWord{8, 10, 398}, + dictWord{9, 10, 681}, + dictWord{139, 10, 632}, + dictWord{9, 11, 417}, + dictWord{ + 137, + 11, + 493, + }, + dictWord{136, 10, 645}, + dictWord{138, 0, 906}, + dictWord{134, 0, 1730}, + dictWord{134, 10, 20}, + dictWord{133, 11, 1019}, + dictWord{134, 0, 1185}, + dictWord{10, 0, 40}, + dictWord{136, 10, 769}, + dictWord{9, 0, 147}, + dictWord{134, 11, 208}, + dictWord{140, 0, 650}, + dictWord{5, 0, 209}, + dictWord{6, 0, 30}, + dictWord{11, 0, 56}, + dictWord{139, 0, 305}, + dictWord{132, 0, 553}, + dictWord{138, 11, 344}, + dictWord{6, 11, 68}, + dictWord{7, 11, 398}, + dictWord{7, 11, 448}, + dictWord{ + 7, + 11, + 1629, + }, + dictWord{7, 11, 1813}, + dictWord{8, 11, 387}, + dictWord{8, 11, 442}, + dictWord{9, 11, 710}, + dictWord{10, 11, 282}, + dictWord{138, 11, 722}, + dictWord{5, 0, 597}, + dictWord{14, 0, 20}, + dictWord{142, 11, 20}, + dictWord{135, 0, 1614}, + dictWord{135, 10, 1757}, + dictWord{4, 0, 150}, + dictWord{5, 0, 303}, + dictWord{6, 0, 327}, + dictWord{135, 10, 937}, + dictWord{16, 0, 49}, + dictWord{7, 10, 1652}, + dictWord{144, 11, 49}, + dictWord{8, 0, 192}, + dictWord{10, 0, 78}, + dictWord{ + 141, + 0, + 359, + }, + dictWord{135, 0, 786}, + dictWord{143, 0, 134}, + dictWord{6, 0, 1638}, + dictWord{7, 0, 79}, + dictWord{7, 0, 496}, + dictWord{9, 0, 138}, + dictWord{ + 10, + 0, + 336, + }, + dictWord{11, 0, 12}, + dictWord{12, 0, 412}, + dictWord{12, 0, 440}, + dictWord{142, 0, 305}, + dictWord{136, 11, 491}, + dictWord{4, 10, 579}, + dictWord{ + 5, + 10, + 226, + }, + dictWord{5, 10, 323}, + dictWord{135, 10, 960}, + dictWord{7, 0, 204}, + dictWord{7, 0, 415}, + dictWord{8, 0, 42}, + dictWord{10, 0, 85}, + dictWord{139, 0, 564}, + dictWord{132, 0, 614}, + dictWord{4, 11, 403}, + dictWord{5, 11, 441}, + dictWord{7, 11, 450}, + dictWord{11, 11, 101}, + dictWord{12, 11, 193}, + dictWord{141, 11, 430}, + dictWord{135, 11, 1927}, + dictWord{135, 11, 1330}, + dictWord{4, 0, 3}, + dictWord{5, 0, 247}, + dictWord{5, 0, 644}, + dictWord{7, 0, 744}, + dictWord{7, 0, 1207}, + dictWord{7, 0, 1225}, + dictWord{7, 0, 1909}, + dictWord{146, 0, 147}, + dictWord{136, 0, 942}, + dictWord{4, 0, 1019}, + dictWord{134, 0, 2023}, + dictWord{5, 11, 679}, + dictWord{133, 10, 973}, + dictWord{5, 0, 285}, + dictWord{9, 0, 67}, + dictWord{13, 0, 473}, + dictWord{143, 0, 82}, + dictWord{7, 11, 328}, + dictWord{137, 11, 326}, + dictWord{151, 0, 8}, + dictWord{6, 10, 135}, + dictWord{135, 10, 1176}, + dictWord{135, 11, 1128}, + dictWord{134, 0, 1309}, + dictWord{135, 11, 1796}, + dictWord{ + 135, + 10, + 314, + }, + dictWord{4, 11, 574}, + dictWord{7, 11, 350}, + dictWord{7, 11, 1024}, + dictWord{8, 11, 338}, + dictWord{9, 11, 677}, + dictWord{10, 11, 808}, + dictWord{ + 139, + 11, + 508, + }, + dictWord{7, 11, 818}, + dictWord{17, 11, 14}, + dictWord{17, 11, 45}, + dictWord{18, 11, 75}, + dictWord{148, 11, 18}, + dictWord{146, 10, 4}, + dictWord{ + 135, + 11, + 1081, + }, + dictWord{4, 0, 29}, + dictWord{6, 0, 532}, + dictWord{7, 0, 1628}, + dictWord{7, 0, 1648}, + dictWord{9, 0, 350}, + dictWord{10, 0, 433}, + dictWord{11, 0, 97}, + dictWord{11, 0, 557}, + dictWord{11, 0, 745}, + dictWord{12, 0, 289}, + dictWord{12, 0, 335}, + dictWord{12, 0, 348}, + dictWord{12, 0, 606}, + dictWord{13, 0, 116}, + dictWord{13, 0, 233}, + dictWord{13, 0, 466}, + dictWord{14, 0, 181}, + dictWord{14, 0, 209}, + dictWord{14, 0, 232}, + dictWord{14, 0, 236}, + dictWord{14, 0, 300}, + dictWord{ + 16, + 0, + 41, + }, + dictWord{148, 0, 97}, + dictWord{7, 0, 318}, + dictWord{6, 10, 281}, + dictWord{8, 10, 282}, + dictWord{8, 10, 480}, + dictWord{8, 10, 499}, + dictWord{9, 10, 198}, + dictWord{10, 10, 143}, + dictWord{10, 10, 169}, + dictWord{10, 10, 211}, + dictWord{10, 10, 417}, + dictWord{10, 10, 574}, + dictWord{11, 10, 147}, + dictWord{ + 11, + 10, + 395, + }, + dictWord{12, 10, 75}, + dictWord{12, 10, 407}, + dictWord{12, 10, 608}, + dictWord{13, 10, 500}, + dictWord{142, 10, 251}, + dictWord{135, 11, 1676}, + dictWord{135, 11, 2037}, + dictWord{135, 0, 1692}, + dictWord{5, 0, 501}, + dictWord{7, 0, 1704}, + dictWord{9, 0, 553}, + dictWord{11, 0, 520}, + dictWord{12, 0, 557}, + dictWord{141, 0, 249}, + dictWord{6, 0, 1527}, + dictWord{14, 0, 324}, + dictWord{15, 0, 55}, + dictWord{15, 0, 80}, + dictWord{14, 11, 324}, + dictWord{15, 11, 55}, + dictWord{143, 11, 80}, + dictWord{135, 10, 1776}, + dictWord{8, 0, 988}, + dictWord{137, 11, 297}, + dictWord{132, 10, 419}, + dictWord{142, 0, 223}, + dictWord{ + 139, + 11, + 234, + }, + dictWord{7, 0, 1123}, + dictWord{12, 0, 508}, + dictWord{14, 0, 102}, + dictWord{14, 0, 226}, + dictWord{144, 0, 57}, + dictWord{4, 10, 138}, + dictWord{ + 7, + 10, + 1012, + }, + dictWord{7, 10, 1280}, + dictWord{137, 10, 76}, + dictWord{7, 0, 1764}, + dictWord{5, 10, 29}, + dictWord{140, 10, 638}, + dictWord{134, 0, 2015}, + dictWord{134, 0, 1599}, + dictWord{138, 11, 56}, + dictWord{6, 11, 306}, + dictWord{7, 11, 1140}, + dictWord{7, 11, 1340}, + dictWord{8, 11, 133}, + dictWord{ + 138, + 11, + 449, + }, + dictWord{139, 11, 1011}, + dictWord{6, 10, 1710}, + dictWord{135, 10, 2038}, + dictWord{7, 11, 1763}, + dictWord{140, 11, 310}, + dictWord{6, 0, 129}, + dictWord{4, 10, 17}, + dictWord{5, 10, 23}, + dictWord{7, 10, 995}, + dictWord{11, 10, 383}, + dictWord{11, 10, 437}, + dictWord{12, 10, 460}, + dictWord{140, 10, 532}, + dictWord{5, 11, 329}, + dictWord{136, 11, 260}, + dictWord{133, 10, 862}, + dictWord{132, 0, 534}, + dictWord{6, 0, 811}, + dictWord{135, 0, 626}, + dictWord{ + 132, + 11, + 657, + }, + dictWord{4, 0, 25}, + dictWord{5, 0, 60}, + dictWord{6, 0, 504}, + dictWord{7, 0, 614}, + dictWord{7, 0, 1155}, + dictWord{12, 0, 0}, + dictWord{152, 11, 7}, + dictWord{ + 7, + 0, + 1248, + }, + dictWord{11, 0, 621}, + dictWord{139, 0, 702}, + dictWord{137, 0, 321}, + dictWord{8, 10, 70}, + dictWord{12, 10, 171}, + dictWord{141, 10, 272}, + dictWord{ + 10, + 10, + 233, + }, + dictWord{139, 10, 76}, + dictWord{4, 0, 379}, + dictWord{7, 0, 1397}, + dictWord{134, 10, 442}, + dictWord{5, 11, 66}, + dictWord{7, 11, 1896}, + dictWord{ + 136, + 11, + 288, + }, + dictWord{134, 11, 1643}, + dictWord{134, 10, 1709}, + dictWord{4, 11, 21}, + dictWord{5, 11, 91}, + dictWord{5, 11, 570}, + dictWord{5, 11, 648}, + dictWord{5, 11, 750}, + dictWord{5, 11, 781}, + dictWord{6, 11, 54}, + dictWord{6, 11, 112}, + dictWord{6, 11, 402}, + dictWord{6, 11, 1732}, + dictWord{7, 11, 315}, + dictWord{ + 7, + 11, + 749, + }, + dictWord{7, 11, 1347}, + dictWord{7, 11, 1900}, + dictWord{9, 11, 78}, + dictWord{9, 11, 508}, + dictWord{10, 11, 611}, + dictWord{11, 11, 510}, + dictWord{ + 11, + 11, + 728, + }, + dictWord{13, 11, 36}, + dictWord{14, 11, 39}, + dictWord{16, 11, 83}, + dictWord{17, 11, 124}, + dictWord{148, 11, 30}, + dictWord{4, 0, 118}, + dictWord{ + 6, + 0, + 274, + }, + dictWord{6, 0, 361}, + dictWord{7, 0, 75}, + dictWord{141, 0, 441}, + dictWord{10, 11, 322}, + dictWord{10, 11, 719}, + dictWord{139, 11, 407}, + dictWord{ + 147, + 10, + 119, + }, + dictWord{12, 11, 549}, + dictWord{14, 11, 67}, + dictWord{147, 11, 60}, + dictWord{11, 10, 69}, + dictWord{12, 10, 105}, + dictWord{12, 10, 117}, + dictWord{13, 10, 213}, + dictWord{14, 10, 13}, + dictWord{14, 10, 62}, + dictWord{14, 10, 177}, + dictWord{14, 10, 421}, + dictWord{15, 10, 19}, + dictWord{146, 10, 141}, + dictWord{9, 0, 841}, + dictWord{137, 10, 309}, + dictWord{7, 10, 608}, + dictWord{7, 10, 976}, + dictWord{8, 11, 125}, + dictWord{8, 11, 369}, + dictWord{8, 11, 524}, + dictWord{9, 10, 146}, + dictWord{10, 10, 206}, + dictWord{10, 11, 486}, + dictWord{10, 10, 596}, + dictWord{11, 11, 13}, + dictWord{11, 11, 381}, + dictWord{11, 11, 736}, + dictWord{11, 11, 766}, + dictWord{11, 11, 845}, + dictWord{13, 11, 114}, + dictWord{13, 10, 218}, + dictWord{13, 11, 292}, + dictWord{14, 11, 47}, + dictWord{ + 142, + 10, + 153, + }, + dictWord{12, 0, 693}, + dictWord{135, 11, 759}, + dictWord{5, 0, 314}, + dictWord{6, 0, 221}, + dictWord{7, 0, 419}, + dictWord{10, 0, 650}, + dictWord{11, 0, 396}, + dictWord{12, 0, 156}, + dictWord{13, 0, 369}, + dictWord{14, 0, 333}, + dictWord{145, 0, 47}, + dictWord{6, 11, 1684}, + dictWord{6, 11, 1731}, + dictWord{7, 11, 356}, + dictWord{7, 11, 1932}, + dictWord{8, 11, 54}, + dictWord{8, 11, 221}, + dictWord{9, 11, 225}, + dictWord{9, 11, 356}, + dictWord{10, 11, 77}, + dictWord{10, 11, 446}, + dictWord{10, 11, 731}, + dictWord{12, 11, 404}, + dictWord{141, 11, 491}, + dictWord{132, 11, 375}, + dictWord{4, 10, 518}, + dictWord{135, 10, 1136}, + dictWord{ + 4, + 0, + 913, + }, + dictWord{4, 11, 411}, + dictWord{11, 11, 643}, + dictWord{140, 11, 115}, + dictWord{4, 11, 80}, + dictWord{133, 11, 44}, + dictWord{8, 10, 689}, + dictWord{ + 137, + 10, + 863, + }, + dictWord{138, 0, 880}, + dictWord{4, 10, 18}, + dictWord{7, 10, 145}, + dictWord{7, 10, 444}, + dictWord{7, 10, 1278}, + dictWord{8, 10, 49}, + dictWord{ + 8, + 10, + 400, + }, + dictWord{9, 10, 71}, + dictWord{9, 10, 250}, + dictWord{10, 10, 459}, + dictWord{12, 10, 160}, + dictWord{144, 10, 24}, + dictWord{136, 0, 475}, + dictWord{ + 5, + 0, + 1016, + }, + dictWord{5, 11, 299}, + dictWord{135, 11, 1083}, + dictWord{7, 0, 602}, + dictWord{8, 0, 179}, + dictWord{10, 0, 781}, + dictWord{140, 0, 126}, + dictWord{ + 6, + 0, + 329, + }, + dictWord{138, 0, 111}, + dictWord{135, 0, 1864}, + dictWord{4, 11, 219}, + dictWord{7, 11, 1761}, + dictWord{137, 11, 86}, + dictWord{6, 0, 1888}, + dictWord{ + 6, + 0, + 1892, + }, + dictWord{6, 0, 1901}, + dictWord{6, 0, 1904}, + dictWord{9, 0, 953}, + dictWord{9, 0, 985}, + dictWord{9, 0, 991}, + dictWord{9, 0, 1001}, + dictWord{12, 0, 818}, + dictWord{12, 0, 846}, + dictWord{12, 0, 847}, + dictWord{12, 0, 861}, + dictWord{12, 0, 862}, + dictWord{12, 0, 873}, + dictWord{12, 0, 875}, + dictWord{12, 0, 877}, + dictWord{12, 0, 879}, + dictWord{12, 0, 881}, + dictWord{12, 0, 884}, + dictWord{12, 0, 903}, + dictWord{12, 0, 915}, + dictWord{12, 0, 926}, + dictWord{12, 0, 939}, + dictWord{ + 15, + 0, + 182, + }, + dictWord{15, 0, 219}, + dictWord{15, 0, 255}, + dictWord{18, 0, 191}, + dictWord{18, 0, 209}, + dictWord{18, 0, 211}, + dictWord{149, 0, 41}, + dictWord{ + 5, + 11, + 328, + }, + dictWord{135, 11, 918}, + dictWord{137, 0, 780}, + dictWord{12, 0, 82}, + dictWord{143, 0, 36}, + dictWord{133, 10, 1010}, + dictWord{5, 0, 821}, + dictWord{ + 134, + 0, + 1687, + }, + dictWord{133, 11, 514}, + dictWord{132, 0, 956}, + dictWord{134, 0, 1180}, + dictWord{10, 0, 112}, + dictWord{5, 10, 87}, + dictWord{7, 10, 313}, + dictWord{ + 7, + 10, + 1103, + }, + dictWord{10, 10, 582}, + dictWord{11, 10, 389}, + dictWord{11, 10, 813}, + dictWord{12, 10, 385}, + dictWord{13, 10, 286}, + dictWord{14, 10, 124}, + dictWord{146, 10, 108}, + dictWord{5, 0, 71}, + dictWord{7, 0, 1407}, + dictWord{9, 0, 704}, + dictWord{10, 0, 261}, + dictWord{10, 0, 619}, + dictWord{11, 0, 547}, + dictWord{11, 0, 619}, + dictWord{143, 0, 157}, + dictWord{4, 0, 531}, + dictWord{5, 0, 455}, + dictWord{5, 11, 301}, + dictWord{6, 11, 571}, + dictWord{14, 11, 49}, + dictWord{ + 146, + 11, + 102, + }, + dictWord{132, 10, 267}, + dictWord{6, 0, 385}, + dictWord{7, 0, 2008}, + dictWord{9, 0, 337}, + dictWord{138, 0, 517}, + dictWord{133, 11, 726}, + dictWord{133, 11, 364}, + dictWord{4, 11, 76}, + dictWord{7, 11, 1550}, + dictWord{9, 11, 306}, + dictWord{9, 11, 430}, + dictWord{9, 11, 663}, + dictWord{10, 11, 683}, + dictWord{11, 11, 427}, + dictWord{11, 11, 753}, + dictWord{12, 11, 334}, + dictWord{12, 11, 442}, + dictWord{14, 11, 258}, + dictWord{14, 11, 366}, + dictWord{ + 143, + 11, + 131, + }, + dictWord{6, 0, 1865}, + dictWord{6, 0, 1879}, + dictWord{6, 0, 1881}, + dictWord{6, 0, 1894}, + dictWord{6, 0, 1908}, + dictWord{9, 0, 915}, + dictWord{9, 0, 926}, + dictWord{9, 0, 940}, + dictWord{9, 0, 943}, + dictWord{9, 0, 966}, + dictWord{9, 0, 980}, + dictWord{9, 0, 989}, + dictWord{9, 0, 1005}, + dictWord{9, 0, 1010}, + dictWord{ + 12, + 0, + 813, + }, + dictWord{12, 0, 817}, + dictWord{12, 0, 840}, + dictWord{12, 0, 843}, + dictWord{12, 0, 855}, + dictWord{12, 0, 864}, + dictWord{12, 0, 871}, + dictWord{12, 0, 872}, + dictWord{12, 0, 899}, + dictWord{12, 0, 905}, + dictWord{12, 0, 924}, + dictWord{15, 0, 171}, + dictWord{15, 0, 181}, + dictWord{15, 0, 224}, + dictWord{15, 0, 235}, + dictWord{15, 0, 251}, + dictWord{146, 0, 184}, + dictWord{137, 11, 52}, + dictWord{5, 0, 16}, + dictWord{6, 0, 86}, + dictWord{6, 0, 603}, + dictWord{7, 0, 292}, + dictWord{7, 0, 561}, + dictWord{8, 0, 257}, + dictWord{8, 0, 382}, + dictWord{9, 0, 721}, + dictWord{9, 0, 778}, + dictWord{11, 0, 581}, + dictWord{140, 0, 466}, + dictWord{4, 0, 486}, + dictWord{ + 5, + 0, + 491, + }, + dictWord{135, 10, 1121}, + dictWord{4, 0, 72}, + dictWord{6, 0, 265}, + dictWord{135, 0, 1300}, + dictWord{135, 11, 1183}, + dictWord{10, 10, 249}, + dictWord{139, 10, 209}, + dictWord{132, 10, 561}, + dictWord{137, 11, 519}, + dictWord{4, 11, 656}, + dictWord{4, 10, 760}, + dictWord{135, 11, 779}, + dictWord{ + 9, + 10, + 154, + }, + dictWord{140, 10, 485}, + dictWord{135, 11, 1793}, + dictWord{135, 11, 144}, + dictWord{136, 10, 255}, + dictWord{133, 0, 621}, + dictWord{4, 10, 368}, + dictWord{135, 10, 641}, + dictWord{135, 11, 1373}, + dictWord{7, 11, 554}, + dictWord{7, 11, 605}, + dictWord{141, 11, 10}, + dictWord{137, 0, 234}, + dictWord{ + 5, + 0, + 815, + }, + dictWord{6, 0, 1688}, + dictWord{134, 0, 1755}, + dictWord{5, 11, 838}, + dictWord{5, 11, 841}, + dictWord{134, 11, 1649}, + dictWord{7, 0, 1987}, + dictWord{ + 7, + 0, + 2040, + }, + dictWord{136, 0, 743}, + dictWord{133, 11, 1012}, + dictWord{6, 0, 197}, + dictWord{136, 0, 205}, + dictWord{6, 0, 314}, + dictWord{134, 11, 314}, + dictWord{144, 11, 53}, + dictWord{6, 11, 251}, + dictWord{7, 11, 365}, + dictWord{7, 11, 1357}, + dictWord{7, 11, 1497}, + dictWord{8, 11, 154}, + dictWord{141, 11, 281}, + dictWord{133, 11, 340}, + dictWord{6, 0, 452}, + dictWord{7, 0, 312}, + dictWord{138, 0, 219}, + dictWord{138, 0, 589}, + dictWord{4, 0, 333}, + dictWord{9, 0, 176}, + dictWord{12, 0, 353}, + dictWord{141, 0, 187}, + dictWord{9, 10, 92}, + dictWord{147, 10, 91}, + dictWord{134, 0, 1110}, + dictWord{11, 0, 47}, + dictWord{139, 11, 495}, + dictWord{6, 10, 525}, + dictWord{8, 10, 806}, + dictWord{9, 10, 876}, + dictWord{140, 10, 284}, + dictWord{8, 11, 261}, + dictWord{9, 11, 144}, + dictWord{9, 11, 466}, + dictWord{10, 11, 370}, + dictWord{12, 11, 470}, + dictWord{13, 11, 144}, + dictWord{142, 11, 348}, + dictWord{137, 11, 897}, + dictWord{8, 0, 863}, + dictWord{8, 0, 864}, + dictWord{8, 0, 868}, + dictWord{8, 0, 884}, + dictWord{10, 0, 866}, + dictWord{10, 0, 868}, + dictWord{10, 0, 873}, + dictWord{10, 0, 911}, + dictWord{10, 0, 912}, + dictWord{ + 10, + 0, + 944, + }, + dictWord{12, 0, 727}, + dictWord{6, 11, 248}, + dictWord{9, 11, 546}, + dictWord{10, 11, 535}, + dictWord{11, 11, 681}, + dictWord{141, 11, 135}, + dictWord{ + 6, + 0, + 300, + }, + dictWord{135, 0, 1515}, + dictWord{134, 0, 1237}, + dictWord{139, 10, 958}, + dictWord{133, 10, 594}, + dictWord{140, 11, 250}, + dictWord{ + 134, + 0, + 1685, + }, + dictWord{134, 11, 567}, + dictWord{7, 0, 135}, + dictWord{8, 0, 7}, + dictWord{8, 0, 62}, + dictWord{9, 0, 243}, + dictWord{10, 0, 658}, + dictWord{10, 0, 697}, + dictWord{11, 0, 456}, + dictWord{139, 0, 756}, + dictWord{9, 0, 395}, + dictWord{138, 0, 79}, + dictWord{6, 10, 1641}, + dictWord{136, 10, 820}, + dictWord{4, 10, 302}, + dictWord{135, 10, 1766}, + dictWord{134, 11, 174}, + dictWord{135, 10, 1313}, + dictWord{135, 0, 631}, + dictWord{134, 10, 1674}, + dictWord{134, 11, 395}, + dictWord{138, 0, 835}, + dictWord{7, 0, 406}, + dictWord{7, 0, 459}, + dictWord{8, 0, 606}, + dictWord{139, 0, 726}, + dictWord{134, 11, 617}, + dictWord{134, 0, 979}, + dictWord{ + 6, + 10, + 389, + }, + dictWord{7, 10, 149}, + dictWord{9, 10, 142}, + dictWord{138, 10, 94}, + dictWord{5, 11, 878}, + dictWord{133, 11, 972}, + dictWord{6, 10, 8}, + dictWord{ + 7, + 10, + 1881, + }, + dictWord{8, 10, 91}, + dictWord{136, 11, 511}, + dictWord{133, 0, 612}, + dictWord{132, 11, 351}, + dictWord{4, 0, 372}, + dictWord{7, 0, 482}, + dictWord{ + 8, + 0, + 158, + }, + dictWord{9, 0, 602}, + dictWord{9, 0, 615}, + dictWord{10, 0, 245}, + dictWord{10, 0, 678}, + dictWord{10, 0, 744}, + dictWord{11, 0, 248}, + dictWord{ + 139, + 0, + 806, + }, + dictWord{5, 0, 854}, + dictWord{135, 0, 1991}, + dictWord{132, 11, 286}, + dictWord{135, 11, 344}, + dictWord{7, 11, 438}, + dictWord{7, 11, 627}, + dictWord{ + 7, + 11, + 1516, + }, + dictWord{8, 11, 40}, + dictWord{9, 11, 56}, + dictWord{9, 11, 294}, + dictWord{10, 11, 30}, + dictWord{10, 11, 259}, + dictWord{11, 11, 969}, + dictWord{ + 146, + 11, + 148, + }, + dictWord{135, 0, 1492}, + dictWord{5, 11, 259}, + dictWord{7, 11, 414}, + dictWord{7, 11, 854}, + dictWord{142, 11, 107}, + dictWord{135, 10, 1746}, + dictWord{6, 0, 833}, + dictWord{134, 0, 998}, + dictWord{135, 10, 24}, + dictWord{6, 0, 750}, + dictWord{135, 0, 1739}, + dictWord{4, 10, 503}, + dictWord{ + 135, + 10, + 1661, + }, + dictWord{5, 10, 130}, + dictWord{7, 10, 1314}, + dictWord{9, 10, 610}, + dictWord{10, 10, 718}, + dictWord{11, 10, 601}, + dictWord{11, 10, 819}, + dictWord{ + 11, + 10, + 946, + }, + dictWord{140, 10, 536}, + dictWord{10, 10, 149}, + dictWord{11, 10, 280}, + dictWord{142, 10, 336}, + dictWord{132, 11, 738}, + dictWord{ + 135, + 10, + 1946, + }, + dictWord{5, 0, 195}, + dictWord{135, 0, 1685}, + dictWord{7, 0, 1997}, + dictWord{8, 0, 730}, + dictWord{139, 0, 1006}, + dictWord{151, 11, 17}, + dictWord{ + 133, + 11, + 866, + }, + dictWord{14, 0, 463}, + dictWord{14, 0, 470}, + dictWord{150, 0, 61}, + dictWord{5, 0, 751}, + dictWord{8, 0, 266}, + dictWord{11, 0, 578}, + dictWord{ + 4, + 10, + 392, + }, + dictWord{135, 10, 1597}, + dictWord{5, 10, 433}, + dictWord{9, 10, 633}, + dictWord{139, 10, 629}, + dictWord{135, 0, 821}, + dictWord{6, 0, 715}, + dictWord{ + 134, + 0, + 1325, + }, + dictWord{133, 11, 116}, + dictWord{6, 0, 868}, + dictWord{132, 11, 457}, + dictWord{134, 0, 959}, + dictWord{6, 10, 234}, + dictWord{138, 11, 199}, + dictWord{7, 0, 1053}, + dictWord{7, 10, 1950}, + dictWord{8, 10, 680}, + dictWord{11, 10, 817}, + dictWord{147, 10, 88}, + dictWord{7, 10, 1222}, + dictWord{ + 138, + 10, + 386, + }, + dictWord{5, 0, 950}, + dictWord{5, 0, 994}, + dictWord{6, 0, 351}, + dictWord{134, 0, 1124}, + dictWord{134, 0, 1081}, + dictWord{7, 0, 1595}, + dictWord{6, 10, 5}, + dictWord{11, 10, 249}, + dictWord{12, 10, 313}, + dictWord{16, 10, 66}, + dictWord{145, 10, 26}, + dictWord{148, 0, 59}, + dictWord{5, 11, 527}, + dictWord{6, 11, 189}, + dictWord{135, 11, 859}, + dictWord{5, 10, 963}, + dictWord{6, 10, 1773}, + dictWord{11, 11, 104}, + dictWord{11, 11, 554}, + dictWord{15, 11, 60}, + dictWord{ + 143, + 11, + 125, + }, + dictWord{135, 0, 47}, + dictWord{137, 0, 684}, + dictWord{134, 11, 116}, + dictWord{134, 0, 1606}, + dictWord{134, 0, 777}, + dictWord{7, 0, 1020}, + dictWord{ + 8, + 10, + 509, + }, + dictWord{136, 10, 792}, + dictWord{135, 0, 1094}, + dictWord{132, 0, 350}, + dictWord{133, 11, 487}, + dictWord{4, 11, 86}, + dictWord{5, 11, 667}, + dictWord{5, 11, 753}, + dictWord{6, 11, 316}, + dictWord{6, 11, 455}, + dictWord{135, 11, 946}, + dictWord{7, 0, 1812}, + dictWord{13, 0, 259}, + dictWord{13, 0, 356}, + dictWord{14, 0, 242}, + dictWord{147, 0, 114}, + dictWord{132, 10, 931}, + dictWord{133, 0, 967}, + dictWord{4, 0, 473}, + dictWord{7, 0, 623}, + dictWord{8, 0, 808}, + dictWord{ + 9, + 0, + 871, + }, + dictWord{9, 0, 893}, + dictWord{11, 0, 38}, + dictWord{11, 0, 431}, + dictWord{12, 0, 112}, + dictWord{12, 0, 217}, + dictWord{12, 0, 243}, + dictWord{12, 0, 562}, + dictWord{12, 0, 663}, + dictWord{12, 0, 683}, + dictWord{13, 0, 141}, + dictWord{13, 0, 197}, + dictWord{13, 0, 227}, + dictWord{13, 0, 406}, + dictWord{13, 0, 487}, + dictWord{14, 0, 156}, + dictWord{14, 0, 203}, + dictWord{14, 0, 224}, + dictWord{14, 0, 256}, + dictWord{18, 0, 58}, + dictWord{150, 0, 0}, + dictWord{138, 0, 286}, + dictWord{ + 7, + 10, + 943, + }, + dictWord{139, 10, 614}, + dictWord{135, 10, 1837}, + dictWord{150, 11, 45}, + dictWord{132, 0, 798}, + dictWord{4, 0, 222}, + dictWord{7, 0, 286}, + dictWord{136, 0, 629}, + dictWord{4, 11, 79}, + dictWord{7, 11, 1773}, + dictWord{10, 11, 450}, + dictWord{11, 11, 589}, + dictWord{13, 11, 332}, + dictWord{13, 11, 493}, + dictWord{14, 11, 183}, + dictWord{14, 11, 334}, + dictWord{14, 11, 362}, + dictWord{14, 11, 368}, + dictWord{14, 11, 376}, + dictWord{14, 11, 379}, + dictWord{ + 19, + 11, + 90, + }, + dictWord{19, 11, 103}, + dictWord{19, 11, 127}, + dictWord{148, 11, 90}, + dictWord{5, 0, 337}, + dictWord{11, 0, 513}, + dictWord{11, 0, 889}, + dictWord{ + 11, + 0, + 961, + }, + dictWord{12, 0, 461}, + dictWord{13, 0, 79}, + dictWord{15, 0, 121}, + dictWord{4, 10, 90}, + dictWord{5, 10, 545}, + dictWord{7, 10, 754}, + dictWord{9, 10, 186}, + dictWord{10, 10, 72}, + dictWord{10, 10, 782}, + dictWord{11, 10, 577}, + dictWord{11, 10, 610}, + dictWord{12, 10, 354}, + dictWord{12, 10, 362}, + dictWord{ + 140, + 10, + 595, + }, + dictWord{141, 0, 306}, + dictWord{136, 0, 146}, + dictWord{7, 0, 1646}, + dictWord{9, 10, 329}, + dictWord{11, 10, 254}, + dictWord{141, 11, 124}, + dictWord{ + 4, + 0, + 465, + }, + dictWord{135, 0, 1663}, + dictWord{132, 0, 525}, + dictWord{133, 11, 663}, + dictWord{10, 0, 299}, + dictWord{18, 0, 74}, + dictWord{9, 10, 187}, + dictWord{ + 11, + 10, + 1016, + }, + dictWord{145, 10, 44}, + dictWord{7, 0, 165}, + dictWord{7, 0, 919}, + dictWord{4, 10, 506}, + dictWord{136, 10, 517}, + dictWord{5, 10, 295}, + dictWord{ + 135, + 10, + 1680, + }, + dictWord{133, 11, 846}, + dictWord{134, 0, 1064}, + dictWord{5, 11, 378}, + dictWord{7, 11, 1402}, + dictWord{7, 11, 1414}, + dictWord{8, 11, 465}, + dictWord{9, 11, 286}, + dictWord{10, 11, 185}, + dictWord{10, 11, 562}, + dictWord{10, 11, 635}, + dictWord{11, 11, 31}, + dictWord{11, 11, 393}, + dictWord{ + 12, + 11, + 456, + }, + dictWord{13, 11, 312}, + dictWord{18, 11, 65}, + dictWord{18, 11, 96}, + dictWord{147, 11, 89}, + dictWord{132, 0, 596}, + dictWord{7, 10, 987}, + dictWord{ + 9, + 10, + 688, + }, + dictWord{10, 10, 522}, + dictWord{11, 10, 788}, + dictWord{140, 10, 566}, + dictWord{6, 0, 82}, + dictWord{7, 0, 138}, + dictWord{7, 0, 517}, + dictWord{7, 0, 1741}, + dictWord{11, 0, 238}, + dictWord{4, 11, 648}, + dictWord{134, 10, 1775}, + dictWord{7, 0, 1233}, + dictWord{7, 10, 700}, + dictWord{7, 10, 940}, + dictWord{8, 10, 514}, + dictWord{9, 10, 116}, + dictWord{9, 10, 535}, + dictWord{10, 10, 118}, + dictWord{11, 10, 107}, + dictWord{11, 10, 148}, + dictWord{11, 10, 922}, + dictWord{ + 12, + 10, + 254, + }, + dictWord{12, 10, 421}, + dictWord{142, 10, 238}, + dictWord{4, 0, 962}, + dictWord{6, 0, 1824}, + dictWord{8, 0, 894}, + dictWord{12, 0, 708}, + dictWord{ + 12, + 0, + 725, + }, + dictWord{14, 0, 451}, + dictWord{20, 0, 94}, + dictWord{22, 0, 59}, + dictWord{150, 0, 62}, + dictWord{5, 11, 945}, + dictWord{6, 11, 1656}, + dictWord{6, 11, 1787}, + dictWord{7, 11, 167}, + dictWord{8, 11, 824}, + dictWord{9, 11, 391}, + dictWord{10, 11, 375}, + dictWord{139, 11, 185}, + dictWord{5, 0, 495}, + dictWord{7, 0, 834}, + dictWord{9, 0, 733}, + dictWord{139, 0, 378}, + dictWord{4, 10, 743}, + dictWord{135, 11, 1273}, + dictWord{6, 0, 1204}, + dictWord{7, 11, 1645}, + dictWord{8, 11, 352}, + dictWord{137, 11, 249}, + dictWord{139, 10, 292}, + dictWord{133, 0, 559}, + dictWord{132, 11, 152}, + dictWord{9, 0, 499}, + dictWord{10, 0, 341}, + dictWord{ + 15, + 0, + 144, + }, + dictWord{19, 0, 49}, + dictWord{7, 10, 1283}, + dictWord{9, 10, 227}, + dictWord{11, 10, 325}, + dictWord{11, 10, 408}, + dictWord{14, 10, 180}, + dictWord{ + 146, + 10, + 47, + }, + dictWord{6, 0, 21}, + dictWord{6, 0, 1737}, + dictWord{7, 0, 1444}, + dictWord{136, 0, 224}, + dictWord{133, 11, 1006}, + dictWord{7, 0, 1446}, + dictWord{ + 9, + 0, + 97, + }, + dictWord{17, 0, 15}, + dictWord{5, 10, 81}, + dictWord{7, 10, 146}, + dictWord{7, 10, 1342}, + dictWord{8, 10, 53}, + dictWord{8, 10, 561}, + dictWord{8, 10, 694}, + dictWord{8, 10, 754}, + dictWord{9, 10, 115}, + dictWord{9, 10, 894}, + dictWord{10, 10, 462}, + dictWord{10, 10, 813}, + dictWord{11, 10, 230}, + dictWord{11, 10, 657}, + dictWord{11, 10, 699}, + dictWord{11, 10, 748}, + dictWord{12, 10, 119}, + dictWord{12, 10, 200}, + dictWord{12, 10, 283}, + dictWord{142, 10, 273}, + dictWord{ + 5, + 10, + 408, + }, + dictWord{137, 10, 747}, + dictWord{135, 11, 431}, + dictWord{135, 11, 832}, + dictWord{6, 0, 729}, + dictWord{134, 0, 953}, + dictWord{4, 0, 727}, + dictWord{ + 8, + 0, + 565, + }, + dictWord{5, 11, 351}, + dictWord{7, 11, 264}, + dictWord{136, 11, 565}, + dictWord{134, 0, 1948}, + dictWord{5, 0, 519}, + dictWord{5, 11, 40}, + dictWord{ + 7, + 11, + 598, + }, + dictWord{7, 11, 1638}, + dictWord{8, 11, 78}, + dictWord{9, 11, 166}, + dictWord{9, 11, 640}, + dictWord{9, 11, 685}, + dictWord{9, 11, 773}, + dictWord{ + 11, + 11, + 215, + }, + dictWord{13, 11, 65}, + dictWord{14, 11, 172}, + dictWord{14, 11, 317}, + dictWord{145, 11, 6}, + dictWord{8, 11, 60}, + dictWord{9, 11, 343}, + dictWord{ + 139, + 11, + 769, + }, + dictWord{137, 11, 455}, + dictWord{134, 0, 1193}, + dictWord{140, 0, 790}, + dictWord{7, 11, 1951}, + dictWord{8, 11, 765}, + dictWord{8, 11, 772}, + dictWord{140, 11, 671}, + dictWord{7, 11, 108}, + dictWord{8, 11, 219}, + dictWord{8, 11, 388}, + dictWord{9, 11, 639}, + dictWord{9, 11, 775}, + dictWord{11, 11, 275}, + dictWord{140, 11, 464}, + dictWord{132, 11, 468}, + dictWord{7, 10, 30}, + dictWord{8, 10, 86}, + dictWord{8, 10, 315}, + dictWord{8, 10, 700}, + dictWord{9, 10, 576}, + dictWord{ + 9, + 10, + 858, + }, + dictWord{11, 10, 310}, + dictWord{11, 10, 888}, + dictWord{11, 10, 904}, + dictWord{12, 10, 361}, + dictWord{141, 10, 248}, + dictWord{5, 11, 15}, + dictWord{6, 11, 56}, + dictWord{7, 11, 1758}, + dictWord{8, 11, 500}, + dictWord{9, 11, 730}, + dictWord{11, 11, 331}, + dictWord{13, 11, 150}, + dictWord{142, 11, 282}, + dictWord{4, 0, 402}, + dictWord{7, 0, 2}, + dictWord{8, 0, 323}, + dictWord{136, 0, 479}, + dictWord{138, 10, 839}, + dictWord{11, 0, 580}, + dictWord{142, 0, 201}, + dictWord{ + 5, + 0, + 59, + }, + dictWord{135, 0, 672}, + dictWord{137, 10, 617}, + dictWord{146, 0, 34}, + dictWord{134, 11, 1886}, + dictWord{4, 0, 961}, + dictWord{136, 0, 896}, + dictWord{ + 6, + 0, + 1285, + }, + dictWord{5, 11, 205}, + dictWord{6, 11, 438}, + dictWord{137, 11, 711}, + dictWord{134, 10, 428}, + dictWord{7, 10, 524}, + dictWord{8, 10, 169}, + dictWord{8, 10, 234}, + dictWord{9, 10, 480}, + dictWord{138, 10, 646}, + dictWord{148, 0, 46}, + dictWord{141, 0, 479}, + dictWord{133, 11, 534}, + dictWord{6, 0, 2019}, + dictWord{134, 10, 1648}, + dictWord{4, 0, 85}, + dictWord{7, 0, 549}, + dictWord{7, 10, 1205}, + dictWord{138, 10, 637}, + dictWord{4, 0, 663}, + dictWord{5, 0, 94}, + dictWord{ + 7, + 11, + 235, + }, + dictWord{7, 11, 1475}, + dictWord{15, 11, 68}, + dictWord{146, 11, 120}, + dictWord{6, 11, 443}, + dictWord{9, 11, 237}, + dictWord{9, 11, 571}, + dictWord{ + 9, + 11, + 695, + }, + dictWord{10, 11, 139}, + dictWord{11, 11, 715}, + dictWord{12, 11, 417}, + dictWord{141, 11, 421}, + dictWord{132, 0, 783}, + dictWord{4, 0, 682}, + dictWord{8, 0, 65}, + dictWord{9, 10, 39}, + dictWord{10, 10, 166}, + dictWord{11, 10, 918}, + dictWord{12, 10, 635}, + dictWord{20, 10, 10}, + dictWord{22, 10, 27}, + dictWord{ + 22, + 10, + 43, + }, + dictWord{150, 10, 52}, + dictWord{6, 0, 11}, + dictWord{135, 0, 187}, + dictWord{132, 0, 522}, + dictWord{4, 0, 52}, + dictWord{135, 0, 661}, + dictWord{ + 4, + 0, + 383, + }, + dictWord{133, 0, 520}, + dictWord{135, 11, 546}, + dictWord{11, 0, 343}, + dictWord{142, 0, 127}, + dictWord{4, 11, 578}, + dictWord{7, 10, 157}, + dictWord{ + 7, + 11, + 624, + }, + dictWord{7, 11, 916}, + dictWord{8, 10, 279}, + dictWord{10, 11, 256}, + dictWord{11, 11, 87}, + dictWord{139, 11, 703}, + dictWord{134, 10, 604}, + dictWord{ + 4, + 0, + 281, + }, + dictWord{5, 0, 38}, + dictWord{7, 0, 194}, + dictWord{7, 0, 668}, + dictWord{7, 0, 1893}, + dictWord{137, 0, 397}, + dictWord{7, 10, 945}, + dictWord{11, 10, 713}, + dictWord{139, 10, 744}, + dictWord{139, 10, 1022}, + dictWord{9, 0, 635}, + dictWord{139, 0, 559}, + dictWord{5, 11, 923}, + dictWord{7, 11, 490}, + dictWord{ + 12, + 11, + 553, + }, + dictWord{13, 11, 100}, + dictWord{14, 11, 118}, + dictWord{143, 11, 75}, + dictWord{132, 0, 975}, + dictWord{132, 10, 567}, + dictWord{137, 10, 859}, + dictWord{7, 10, 1846}, + dictWord{7, 11, 1846}, + dictWord{8, 10, 628}, + dictWord{136, 11, 628}, + dictWord{148, 0, 116}, + dictWord{138, 11, 750}, + dictWord{14, 0, 51}, + dictWord{14, 11, 51}, + dictWord{15, 11, 7}, + dictWord{148, 11, 20}, + dictWord{132, 0, 858}, + dictWord{134, 0, 1075}, + dictWord{4, 11, 924}, + dictWord{ + 133, + 10, + 762, + }, + dictWord{136, 0, 535}, + dictWord{133, 0, 448}, + dictWord{10, 10, 784}, + dictWord{141, 10, 191}, + dictWord{133, 10, 298}, + dictWord{7, 0, 610}, + dictWord{135, 0, 1501}, + dictWord{7, 10, 633}, + dictWord{7, 10, 905}, + dictWord{7, 10, 909}, + dictWord{7, 10, 1538}, + dictWord{9, 10, 767}, + dictWord{140, 10, 636}, + dictWord{4, 11, 265}, + dictWord{7, 11, 807}, + dictWord{135, 11, 950}, + dictWord{5, 11, 93}, + dictWord{12, 11, 267}, + dictWord{144, 11, 26}, + dictWord{136, 0, 191}, + dictWord{139, 10, 301}, + dictWord{135, 10, 1970}, + dictWord{135, 0, 267}, + dictWord{4, 0, 319}, + dictWord{5, 0, 699}, + dictWord{138, 0, 673}, + dictWord{ + 6, + 0, + 336, + }, + dictWord{7, 0, 92}, + dictWord{7, 0, 182}, + dictWord{8, 0, 453}, + dictWord{8, 0, 552}, + dictWord{9, 0, 204}, + dictWord{9, 0, 285}, + dictWord{10, 0, 99}, + dictWord{ + 11, + 0, + 568, + }, + dictWord{11, 0, 950}, + dictWord{12, 0, 94}, + dictWord{16, 0, 20}, + dictWord{16, 0, 70}, + dictWord{19, 0, 55}, + dictWord{12, 10, 644}, + dictWord{144, 10, 90}, + dictWord{6, 0, 551}, + dictWord{7, 0, 1308}, + dictWord{7, 10, 845}, + dictWord{7, 11, 994}, + dictWord{8, 10, 160}, + dictWord{137, 10, 318}, + dictWord{19, 11, 1}, + dictWord{ + 19, + 11, + 26, + }, + dictWord{150, 11, 9}, + dictWord{7, 0, 1406}, + dictWord{9, 0, 218}, + dictWord{141, 0, 222}, + dictWord{5, 0, 256}, + dictWord{138, 0, 69}, + dictWord{ + 5, + 11, + 233, + }, + dictWord{5, 11, 320}, + dictWord{6, 11, 140}, + dictWord{7, 11, 330}, + dictWord{136, 11, 295}, + dictWord{6, 0, 1980}, + dictWord{136, 0, 952}, + dictWord{ + 4, + 0, + 833, + }, + dictWord{137, 11, 678}, + dictWord{133, 11, 978}, + dictWord{4, 11, 905}, + dictWord{6, 11, 1701}, + dictWord{137, 11, 843}, + dictWord{138, 10, 735}, + dictWord{136, 10, 76}, + dictWord{17, 0, 39}, + dictWord{148, 0, 36}, + dictWord{18, 0, 81}, + dictWord{146, 11, 81}, + dictWord{14, 0, 352}, + dictWord{17, 0, 53}, + dictWord{ + 18, + 0, + 146, + }, + dictWord{18, 0, 152}, + dictWord{19, 0, 11}, + dictWord{150, 0, 54}, + dictWord{135, 0, 634}, + dictWord{138, 10, 841}, + dictWord{132, 0, 618}, + dictWord{ + 4, + 0, + 339, + }, + dictWord{7, 0, 259}, + dictWord{17, 0, 73}, + dictWord{4, 11, 275}, + dictWord{140, 11, 376}, + dictWord{132, 11, 509}, + dictWord{7, 11, 273}, + dictWord{ + 139, + 11, + 377, + }, + dictWord{4, 0, 759}, + dictWord{13, 0, 169}, + dictWord{137, 10, 804}, + dictWord{6, 10, 96}, + dictWord{135, 10, 1426}, + dictWord{4, 10, 651}, + dictWord{133, 10, 289}, + dictWord{7, 0, 1075}, + dictWord{8, 10, 35}, + dictWord{9, 10, 511}, + dictWord{10, 10, 767}, + dictWord{147, 10, 118}, + dictWord{6, 0, 649}, + dictWord{6, 0, 670}, + dictWord{136, 0, 482}, + dictWord{5, 0, 336}, + dictWord{6, 0, 341}, + dictWord{6, 0, 478}, + dictWord{6, 0, 1763}, + dictWord{136, 0, 386}, + dictWord{ + 5, + 11, + 802, + }, + dictWord{7, 11, 2021}, + dictWord{8, 11, 805}, + dictWord{14, 11, 94}, + dictWord{15, 11, 65}, + dictWord{16, 11, 4}, + dictWord{16, 11, 77}, + dictWord{16, 11, 80}, + dictWord{145, 11, 5}, + dictWord{6, 0, 1035}, + dictWord{5, 11, 167}, + dictWord{5, 11, 899}, + dictWord{6, 11, 410}, + dictWord{137, 11, 777}, + dictWord{ + 134, + 11, + 1705, + }, + dictWord{5, 0, 924}, + dictWord{133, 0, 969}, + dictWord{132, 10, 704}, + dictWord{135, 0, 73}, + dictWord{135, 11, 10}, + dictWord{135, 10, 1078}, + dictWord{ + 5, + 11, + 11, + }, + dictWord{6, 11, 117}, + dictWord{6, 11, 485}, + dictWord{7, 11, 1133}, + dictWord{9, 11, 582}, + dictWord{9, 11, 594}, + dictWord{11, 11, 21}, + dictWord{ + 11, + 11, + 818, + }, + dictWord{12, 11, 535}, + dictWord{141, 11, 86}, + dictWord{135, 0, 1971}, + dictWord{4, 11, 264}, + dictWord{7, 11, 1067}, + dictWord{8, 11, 204}, + dictWord{8, 11, 385}, + dictWord{139, 11, 953}, + dictWord{6, 0, 1458}, + dictWord{135, 0, 1344}, + dictWord{5, 0, 396}, + dictWord{134, 0, 501}, + dictWord{4, 10, 720}, + dictWord{133, 10, 306}, + dictWord{4, 0, 929}, + dictWord{5, 0, 799}, + dictWord{8, 0, 46}, + dictWord{8, 0, 740}, + dictWord{133, 10, 431}, + dictWord{7, 11, 646}, + dictWord{ + 7, + 11, + 1730, + }, + dictWord{11, 11, 446}, + dictWord{141, 11, 178}, + dictWord{7, 0, 276}, + dictWord{5, 10, 464}, + dictWord{6, 10, 236}, + dictWord{7, 10, 696}, + dictWord{ + 7, + 10, + 914, + }, + dictWord{7, 10, 1108}, + dictWord{7, 10, 1448}, + dictWord{9, 10, 15}, + dictWord{9, 10, 564}, + dictWord{10, 10, 14}, + dictWord{12, 10, 565}, + dictWord{ + 13, + 10, + 449, + }, + dictWord{14, 10, 53}, + dictWord{15, 10, 13}, + dictWord{16, 10, 64}, + dictWord{145, 10, 41}, + dictWord{4, 0, 892}, + dictWord{133, 0, 770}, + dictWord{ + 6, + 10, + 1767, + }, + dictWord{12, 10, 194}, + dictWord{145, 10, 107}, + dictWord{135, 0, 158}, + dictWord{5, 10, 840}, + dictWord{138, 11, 608}, + dictWord{134, 0, 1432}, + dictWord{138, 11, 250}, + dictWord{8, 11, 794}, + dictWord{9, 11, 400}, + dictWord{10, 11, 298}, + dictWord{142, 11, 228}, + dictWord{151, 0, 25}, + dictWord{ + 7, + 11, + 1131, + }, + dictWord{135, 11, 1468}, + dictWord{135, 0, 2001}, + dictWord{9, 10, 642}, + dictWord{11, 10, 236}, + dictWord{142, 10, 193}, + dictWord{4, 10, 68}, + dictWord{5, 10, 634}, + dictWord{6, 10, 386}, + dictWord{7, 10, 794}, + dictWord{8, 10, 273}, + dictWord{9, 10, 563}, + dictWord{10, 10, 105}, + dictWord{10, 10, 171}, + dictWord{11, 10, 94}, + dictWord{139, 10, 354}, + dictWord{136, 11, 724}, + dictWord{132, 0, 478}, + dictWord{11, 11, 512}, + dictWord{13, 11, 205}, + dictWord{ + 19, + 11, + 30, + }, + dictWord{22, 11, 36}, + dictWord{151, 11, 19}, + dictWord{7, 0, 1461}, + dictWord{140, 0, 91}, + dictWord{6, 11, 190}, + dictWord{7, 11, 768}, + dictWord{ + 135, + 11, + 1170, + }, + dictWord{4, 0, 602}, + dictWord{8, 0, 211}, + dictWord{4, 10, 95}, + dictWord{7, 10, 416}, + dictWord{139, 10, 830}, + dictWord{7, 10, 731}, + dictWord{13, 10, 20}, + dictWord{143, 10, 11}, + dictWord{6, 0, 1068}, + dictWord{135, 0, 1872}, + dictWord{4, 0, 13}, + dictWord{5, 0, 567}, + dictWord{7, 0, 1498}, + dictWord{9, 0, 124}, + dictWord{11, 0, 521}, + dictWord{12, 0, 405}, + dictWord{135, 11, 1023}, + dictWord{135, 0, 1006}, + dictWord{132, 0, 735}, + dictWord{138, 0, 812}, + dictWord{4, 0, 170}, + dictWord{135, 0, 323}, + dictWord{6, 11, 137}, + dictWord{9, 11, 75}, + dictWord{9, 11, 253}, + dictWord{10, 11, 194}, + dictWord{138, 11, 444}, + dictWord{5, 0, 304}, + dictWord{7, 0, 1403}, + dictWord{5, 10, 864}, + dictWord{10, 10, 648}, + dictWord{11, 10, 671}, + dictWord{143, 10, 46}, + dictWord{135, 11, 1180}, + dictWord{ + 133, + 10, + 928, + }, + dictWord{4, 0, 148}, + dictWord{133, 0, 742}, + dictWord{11, 10, 986}, + dictWord{140, 10, 682}, + dictWord{133, 0, 523}, + dictWord{135, 11, 1743}, + dictWord{7, 0, 730}, + dictWord{18, 0, 144}, + dictWord{19, 0, 61}, + dictWord{8, 10, 44}, + dictWord{9, 10, 884}, + dictWord{10, 10, 580}, + dictWord{11, 10, 399}, + dictWord{ + 11, + 10, + 894, + }, + dictWord{143, 10, 122}, + dictWord{5, 11, 760}, + dictWord{7, 11, 542}, + dictWord{8, 11, 135}, + dictWord{136, 11, 496}, + dictWord{136, 0, 981}, + dictWord{133, 0, 111}, + dictWord{10, 0, 132}, + dictWord{11, 0, 191}, + dictWord{11, 0, 358}, + dictWord{139, 0, 460}, + dictWord{7, 11, 319}, + dictWord{7, 11, 355}, + dictWord{ + 7, + 11, + 763, + }, + dictWord{10, 11, 389}, + dictWord{145, 11, 43}, + dictWord{134, 0, 890}, + dictWord{134, 0, 1420}, + dictWord{136, 11, 557}, + dictWord{ + 133, + 10, + 518, + }, + dictWord{133, 0, 444}, + dictWord{135, 0, 1787}, + dictWord{135, 10, 1852}, + dictWord{8, 0, 123}, + dictWord{15, 0, 6}, + dictWord{144, 0, 7}, + dictWord{ + 6, + 0, + 2041, + }, + dictWord{10, 11, 38}, + dictWord{139, 11, 784}, + dictWord{136, 0, 932}, + dictWord{5, 0, 937}, + dictWord{135, 0, 100}, + dictWord{6, 0, 995}, + dictWord{ + 4, + 11, + 58, + }, + dictWord{5, 11, 286}, + dictWord{6, 11, 319}, + dictWord{7, 11, 402}, + dictWord{7, 11, 1254}, + dictWord{7, 11, 1903}, + dictWord{8, 11, 356}, + dictWord{ + 140, + 11, + 408, + }, + dictWord{4, 11, 389}, + dictWord{9, 11, 181}, + dictWord{9, 11, 255}, + dictWord{10, 11, 8}, + dictWord{10, 11, 29}, + dictWord{10, 11, 816}, + dictWord{ + 11, + 11, + 311, + }, + dictWord{11, 11, 561}, + dictWord{12, 11, 67}, + dictWord{141, 11, 181}, + dictWord{138, 0, 255}, + dictWord{5, 0, 138}, + dictWord{4, 10, 934}, + dictWord{ + 136, + 10, + 610, + }, + dictWord{4, 0, 965}, + dictWord{10, 0, 863}, + dictWord{138, 0, 898}, + dictWord{10, 10, 804}, + dictWord{138, 10, 832}, + dictWord{12, 0, 631}, + dictWord{ + 8, + 10, + 96, + }, + dictWord{9, 10, 36}, + dictWord{10, 10, 607}, + dictWord{11, 10, 423}, + dictWord{11, 10, 442}, + dictWord{12, 10, 309}, + dictWord{14, 10, 199}, + dictWord{ + 15, + 10, + 90, + }, + dictWord{145, 10, 110}, + dictWord{134, 0, 1394}, + dictWord{4, 0, 652}, + dictWord{8, 0, 320}, + dictWord{22, 0, 6}, + dictWord{22, 0, 16}, + dictWord{ + 9, + 10, + 13, + }, + dictWord{9, 10, 398}, + dictWord{9, 10, 727}, + dictWord{10, 10, 75}, + dictWord{10, 10, 184}, + dictWord{10, 10, 230}, + dictWord{10, 10, 564}, + dictWord{ + 10, + 10, + 569, + }, + dictWord{11, 10, 973}, + dictWord{12, 10, 70}, + dictWord{12, 10, 189}, + dictWord{13, 10, 57}, + dictWord{141, 10, 257}, + dictWord{6, 0, 897}, + dictWord{ + 134, + 0, + 1333, + }, + dictWord{4, 0, 692}, + dictWord{133, 0, 321}, + dictWord{133, 11, 373}, + dictWord{135, 0, 922}, + dictWord{5, 0, 619}, + dictWord{133, 0, 698}, + dictWord{ + 137, + 10, + 631, + }, + dictWord{5, 10, 345}, + dictWord{135, 10, 1016}, + dictWord{9, 0, 957}, + dictWord{9, 0, 1018}, + dictWord{12, 0, 828}, + dictWord{12, 0, 844}, + dictWord{ + 12, + 0, + 897, + }, + dictWord{12, 0, 901}, + dictWord{12, 0, 943}, + dictWord{15, 0, 180}, + dictWord{18, 0, 197}, + dictWord{18, 0, 200}, + dictWord{18, 0, 213}, + dictWord{ + 18, + 0, + 214, + }, + dictWord{146, 0, 226}, + dictWord{5, 0, 917}, + dictWord{134, 0, 1659}, + dictWord{135, 0, 1100}, + dictWord{134, 0, 1173}, + dictWord{134, 0, 1930}, + dictWord{5, 0, 251}, + dictWord{5, 0, 956}, + dictWord{8, 0, 268}, + dictWord{9, 0, 214}, + dictWord{146, 0, 142}, + dictWord{133, 10, 673}, + dictWord{137, 10, 850}, + dictWord{ + 4, + 10, + 287, + }, + dictWord{133, 10, 1018}, + dictWord{132, 11, 672}, + dictWord{5, 0, 346}, + dictWord{5, 0, 711}, + dictWord{8, 0, 390}, + dictWord{11, 11, 752}, + dictWord{139, 11, 885}, + dictWord{5, 10, 34}, + dictWord{10, 10, 724}, + dictWord{12, 10, 444}, + dictWord{13, 10, 354}, + dictWord{18, 10, 32}, + dictWord{23, 10, 24}, + dictWord{23, 10, 31}, + dictWord{152, 10, 5}, + dictWord{4, 11, 710}, + dictWord{134, 11, 606}, + dictWord{134, 0, 744}, + dictWord{134, 10, 382}, + dictWord{ + 133, + 11, + 145, + }, + dictWord{4, 10, 329}, + dictWord{7, 11, 884}, + dictWord{140, 11, 124}, + dictWord{4, 11, 467}, + dictWord{5, 11, 405}, + dictWord{134, 11, 544}, + dictWord{ + 9, + 10, + 846, + }, + dictWord{138, 10, 827}, + dictWord{133, 0, 624}, + dictWord{9, 11, 372}, + dictWord{15, 11, 2}, + dictWord{19, 11, 10}, + dictWord{147, 11, 18}, + dictWord{ + 4, + 11, + 387, + }, + dictWord{135, 11, 1288}, + dictWord{5, 0, 783}, + dictWord{7, 0, 1998}, + dictWord{135, 0, 2047}, + dictWord{132, 10, 906}, + dictWord{136, 10, 366}, + dictWord{135, 11, 550}, + dictWord{4, 10, 123}, + dictWord{4, 10, 649}, + dictWord{5, 10, 605}, + dictWord{7, 10, 1509}, + dictWord{136, 10, 36}, + dictWord{ + 134, + 0, + 1125, + }, + dictWord{132, 0, 594}, + dictWord{133, 10, 767}, + dictWord{135, 11, 1227}, + dictWord{136, 11, 467}, + dictWord{4, 11, 576}, + dictWord{ + 135, + 11, + 1263, + }, + dictWord{4, 0, 268}, + dictWord{7, 0, 1534}, + dictWord{135, 11, 1534}, + dictWord{4, 10, 273}, + dictWord{5, 10, 658}, + dictWord{5, 11, 919}, + dictWord{ + 5, + 10, + 995, + }, + dictWord{134, 11, 1673}, + dictWord{133, 0, 563}, + dictWord{134, 10, 72}, + dictWord{135, 10, 1345}, + dictWord{4, 11, 82}, + dictWord{5, 11, 333}, + dictWord{ + 5, + 11, + 904, + }, + dictWord{6, 11, 207}, + dictWord{7, 11, 325}, + dictWord{7, 11, 1726}, + dictWord{8, 11, 101}, + dictWord{10, 11, 778}, + dictWord{139, 11, 220}, + dictWord{5, 0, 37}, + dictWord{6, 0, 39}, + dictWord{6, 0, 451}, + dictWord{7, 0, 218}, + dictWord{7, 0, 667}, + dictWord{7, 0, 1166}, + dictWord{7, 0, 1687}, + dictWord{8, 0, 662}, + dictWord{16, 0, 2}, + dictWord{133, 10, 589}, + dictWord{134, 0, 1332}, + dictWord{133, 11, 903}, + dictWord{134, 0, 508}, + dictWord{5, 10, 117}, + dictWord{6, 10, 514}, + dictWord{6, 10, 541}, + dictWord{7, 10, 1164}, + dictWord{7, 10, 1436}, + dictWord{8, 10, 220}, + dictWord{8, 10, 648}, + dictWord{10, 10, 688}, + dictWord{11, 10, 560}, + dictWord{140, 11, 147}, + dictWord{6, 11, 555}, + dictWord{135, 11, 485}, + dictWord{133, 10, 686}, + dictWord{7, 0, 453}, + dictWord{7, 0, 635}, + dictWord{7, 0, 796}, + dictWord{8, 0, 331}, + dictWord{9, 0, 330}, + dictWord{9, 0, 865}, + dictWord{10, 0, 119}, + dictWord{10, 0, 235}, + dictWord{11, 0, 111}, + dictWord{11, 0, 129}, + dictWord{ + 11, + 0, + 240, + }, + dictWord{12, 0, 31}, + dictWord{12, 0, 66}, + dictWord{12, 0, 222}, + dictWord{12, 0, 269}, + dictWord{12, 0, 599}, + dictWord{12, 0, 684}, + dictWord{12, 0, 689}, + dictWord{12, 0, 691}, + dictWord{142, 0, 345}, + dictWord{135, 0, 1834}, + dictWord{4, 11, 705}, + dictWord{7, 11, 615}, + dictWord{138, 11, 251}, + dictWord{ + 136, + 11, + 345, + }, + dictWord{137, 0, 527}, + dictWord{6, 0, 98}, + dictWord{7, 0, 702}, + dictWord{135, 0, 991}, + dictWord{11, 0, 576}, + dictWord{14, 0, 74}, + dictWord{7, 10, 196}, + dictWord{10, 10, 765}, + dictWord{11, 10, 347}, + dictWord{11, 10, 552}, + dictWord{11, 10, 790}, + dictWord{12, 10, 263}, + dictWord{13, 10, 246}, + dictWord{ + 13, + 10, + 270, + }, + dictWord{13, 10, 395}, + dictWord{14, 10, 176}, + dictWord{14, 10, 190}, + dictWord{14, 10, 398}, + dictWord{14, 10, 412}, + dictWord{15, 10, 32}, + dictWord{ + 15, + 10, + 63, + }, + dictWord{16, 10, 88}, + dictWord{147, 10, 105}, + dictWord{134, 11, 90}, + dictWord{13, 0, 84}, + dictWord{141, 0, 122}, + dictWord{6, 0, 37}, + dictWord{ + 7, + 0, + 299, + }, + dictWord{7, 0, 1666}, + dictWord{8, 0, 195}, + dictWord{8, 0, 316}, + dictWord{9, 0, 178}, + dictWord{9, 0, 276}, + dictWord{9, 0, 339}, + dictWord{9, 0, 536}, + dictWord{ + 10, + 0, + 102, + }, + dictWord{10, 0, 362}, + dictWord{10, 0, 785}, + dictWord{11, 0, 55}, + dictWord{11, 0, 149}, + dictWord{11, 0, 773}, + dictWord{13, 0, 416}, + dictWord{ + 13, + 0, + 419, + }, + dictWord{14, 0, 38}, + dictWord{14, 0, 41}, + dictWord{142, 0, 210}, + dictWord{5, 10, 381}, + dictWord{135, 10, 1792}, + dictWord{7, 11, 813}, + dictWord{ + 12, + 11, + 497, + }, + dictWord{141, 11, 56}, + dictWord{7, 10, 616}, + dictWord{138, 10, 413}, + dictWord{133, 0, 645}, + dictWord{6, 11, 125}, + dictWord{135, 11, 1277}, + dictWord{132, 0, 290}, + dictWord{6, 0, 70}, + dictWord{7, 0, 1292}, + dictWord{10, 0, 762}, + dictWord{139, 0, 288}, + dictWord{6, 10, 120}, + dictWord{7, 10, 1188}, + dictWord{ + 7, + 10, + 1710, + }, + dictWord{8, 10, 286}, + dictWord{9, 10, 667}, + dictWord{11, 10, 592}, + dictWord{139, 10, 730}, + dictWord{135, 11, 1784}, + dictWord{7, 0, 1315}, + dictWord{135, 11, 1315}, + dictWord{134, 0, 1955}, + dictWord{135, 10, 1146}, + dictWord{7, 0, 131}, + dictWord{7, 0, 422}, + dictWord{8, 0, 210}, + dictWord{ + 140, + 0, + 573, + }, + dictWord{4, 10, 352}, + dictWord{135, 10, 687}, + dictWord{139, 0, 797}, + dictWord{143, 0, 38}, + dictWord{14, 0, 179}, + dictWord{15, 0, 151}, + dictWord{ + 150, + 0, + 11, + }, + dictWord{7, 0, 488}, + dictWord{4, 10, 192}, + dictWord{5, 10, 49}, + dictWord{6, 10, 200}, + dictWord{6, 10, 293}, + dictWord{134, 10, 1696}, + dictWord{ + 132, + 0, + 936, + }, + dictWord{135, 11, 703}, + dictWord{6, 11, 160}, + dictWord{7, 11, 1106}, + dictWord{9, 11, 770}, + dictWord{10, 11, 618}, + dictWord{11, 11, 112}, + dictWord{ + 140, + 11, + 413, + }, + dictWord{5, 0, 453}, + dictWord{134, 0, 441}, + dictWord{135, 0, 595}, + dictWord{132, 10, 650}, + dictWord{132, 10, 147}, + dictWord{6, 0, 991}, + dictWord{6, 0, 1182}, + dictWord{12, 11, 271}, + dictWord{145, 11, 109}, + dictWord{133, 10, 934}, + dictWord{140, 11, 221}, + dictWord{132, 0, 653}, + dictWord{ + 7, + 0, + 505, + }, + dictWord{135, 0, 523}, + dictWord{134, 0, 903}, + dictWord{135, 11, 479}, + dictWord{7, 11, 304}, + dictWord{9, 11, 646}, + dictWord{9, 11, 862}, + dictWord{ + 10, + 11, + 262, + }, + dictWord{11, 11, 696}, + dictWord{12, 11, 208}, + dictWord{15, 11, 79}, + dictWord{147, 11, 108}, + dictWord{146, 0, 80}, + dictWord{135, 11, 981}, + dictWord{142, 0, 432}, + dictWord{132, 0, 314}, + dictWord{137, 11, 152}, + dictWord{7, 0, 1368}, + dictWord{8, 0, 232}, + dictWord{8, 0, 361}, + dictWord{10, 0, 682}, + dictWord{138, 0, 742}, + dictWord{135, 11, 1586}, + dictWord{9, 0, 534}, + dictWord{4, 11, 434}, + dictWord{11, 11, 663}, + dictWord{12, 11, 210}, + dictWord{13, 11, 166}, + dictWord{13, 11, 310}, + dictWord{14, 11, 373}, + dictWord{147, 11, 43}, + dictWord{7, 11, 1091}, + dictWord{135, 11, 1765}, + dictWord{6, 11, 550}, + dictWord{ + 135, + 11, + 652, + }, + dictWord{137, 0, 27}, + dictWord{142, 0, 12}, + dictWord{4, 10, 637}, + dictWord{5, 11, 553}, + dictWord{7, 11, 766}, + dictWord{138, 11, 824}, + dictWord{ + 7, + 11, + 737, + }, + dictWord{8, 11, 298}, + dictWord{136, 11, 452}, + dictWord{7, 0, 736}, + dictWord{139, 0, 264}, + dictWord{134, 0, 1657}, + dictWord{133, 11, 292}, + dictWord{138, 11, 135}, + dictWord{6, 0, 844}, + dictWord{134, 0, 1117}, + dictWord{135, 0, 127}, + dictWord{9, 10, 867}, + dictWord{138, 10, 837}, + dictWord{ + 6, + 0, + 1184, + }, + dictWord{134, 0, 1208}, + dictWord{134, 0, 1294}, + dictWord{136, 0, 364}, + dictWord{6, 0, 1415}, + dictWord{7, 0, 1334}, + dictWord{11, 0, 125}, + dictWord{ + 6, + 10, + 170, + }, + dictWord{7, 11, 393}, + dictWord{8, 10, 395}, + dictWord{8, 10, 487}, + dictWord{10, 11, 603}, + dictWord{11, 11, 206}, + dictWord{141, 10, 147}, + dictWord{137, 11, 748}, + dictWord{4, 11, 912}, + dictWord{137, 11, 232}, + dictWord{4, 10, 535}, + dictWord{136, 10, 618}, + dictWord{137, 0, 792}, + dictWord{ + 7, + 11, + 1973, + }, + dictWord{136, 11, 716}, + dictWord{135, 11, 98}, + dictWord{5, 0, 909}, + dictWord{9, 0, 849}, + dictWord{138, 0, 805}, + dictWord{4, 0, 630}, + dictWord{ + 132, + 0, + 699, + }, + dictWord{5, 11, 733}, + dictWord{14, 11, 103}, + dictWord{150, 10, 23}, + dictWord{12, 11, 158}, + dictWord{18, 11, 8}, + dictWord{19, 11, 62}, + dictWord{ + 20, + 11, + 6, + }, + dictWord{22, 11, 4}, + dictWord{23, 11, 2}, + dictWord{151, 11, 9}, + dictWord{132, 0, 968}, + dictWord{132, 10, 778}, + dictWord{132, 10, 46}, + dictWord{5, 10, 811}, + dictWord{6, 10, 1679}, + dictWord{6, 10, 1714}, + dictWord{135, 10, 2032}, + dictWord{6, 0, 1446}, + dictWord{7, 10, 1458}, + dictWord{9, 10, 407}, + dictWord{ + 139, + 10, + 15, + }, + dictWord{7, 0, 206}, + dictWord{7, 0, 397}, + dictWord{7, 0, 621}, + dictWord{7, 0, 640}, + dictWord{8, 0, 124}, + dictWord{8, 0, 619}, + dictWord{9, 0, 305}, + dictWord{ + 9, + 0, + 643, + }, + dictWord{10, 0, 264}, + dictWord{10, 0, 628}, + dictWord{11, 0, 40}, + dictWord{12, 0, 349}, + dictWord{13, 0, 134}, + dictWord{13, 0, 295}, + dictWord{ + 14, + 0, + 155, + }, + dictWord{15, 0, 120}, + dictWord{18, 0, 105}, + dictWord{6, 10, 34}, + dictWord{7, 10, 1089}, + dictWord{8, 10, 708}, + dictWord{8, 10, 721}, + dictWord{9, 10, 363}, + dictWord{148, 10, 98}, + dictWord{4, 0, 262}, + dictWord{5, 0, 641}, + dictWord{135, 0, 342}, + dictWord{137, 11, 72}, + dictWord{4, 0, 99}, + dictWord{6, 0, 250}, + dictWord{ + 6, + 0, + 346, + }, + dictWord{8, 0, 127}, + dictWord{138, 0, 81}, + dictWord{132, 0, 915}, + dictWord{5, 0, 75}, + dictWord{9, 0, 517}, + dictWord{10, 0, 470}, + dictWord{12, 0, 155}, + dictWord{141, 0, 224}, + dictWord{132, 10, 462}, + dictWord{11, 11, 600}, + dictWord{11, 11, 670}, + dictWord{141, 11, 245}, + dictWord{142, 0, 83}, + dictWord{ + 5, + 10, + 73, + }, + dictWord{6, 10, 23}, + dictWord{134, 10, 338}, + dictWord{6, 0, 1031}, + dictWord{139, 11, 923}, + dictWord{7, 11, 164}, + dictWord{7, 11, 1571}, + dictWord{ + 9, + 11, + 107, + }, + dictWord{140, 11, 225}, + dictWord{134, 0, 1470}, + dictWord{133, 0, 954}, + dictWord{6, 0, 304}, + dictWord{8, 0, 418}, + dictWord{10, 0, 345}, + dictWord{ + 11, + 0, + 341, + }, + dictWord{139, 0, 675}, + dictWord{9, 0, 410}, + dictWord{139, 0, 425}, + dictWord{4, 11, 27}, + dictWord{5, 11, 484}, + dictWord{5, 11, 510}, + dictWord{6, 11, 434}, + dictWord{7, 11, 1000}, + dictWord{7, 11, 1098}, + dictWord{8, 11, 2}, + dictWord{136, 11, 200}, + dictWord{134, 0, 734}, + dictWord{140, 11, 257}, + dictWord{ + 7, + 10, + 725, + }, + dictWord{8, 10, 498}, + dictWord{139, 10, 268}, + dictWord{134, 0, 1822}, + dictWord{135, 0, 1798}, + dictWord{135, 10, 773}, + dictWord{132, 11, 460}, + dictWord{4, 11, 932}, + dictWord{133, 11, 891}, + dictWord{134, 0, 14}, + dictWord{132, 10, 583}, + dictWord{7, 10, 1462}, + dictWord{8, 11, 625}, + dictWord{ + 139, + 10, + 659, + }, + dictWord{5, 0, 113}, + dictWord{6, 0, 243}, + dictWord{6, 0, 1708}, + dictWord{7, 0, 1865}, + dictWord{11, 0, 161}, + dictWord{16, 0, 37}, + dictWord{17, 0, 99}, + dictWord{133, 10, 220}, + dictWord{134, 11, 76}, + dictWord{5, 11, 461}, + dictWord{135, 11, 1925}, + dictWord{140, 0, 69}, + dictWord{8, 11, 92}, + dictWord{ + 137, + 11, + 221, + }, + dictWord{139, 10, 803}, + dictWord{132, 10, 544}, + dictWord{4, 0, 274}, + dictWord{134, 0, 922}, + dictWord{132, 0, 541}, + dictWord{5, 0, 627}, + dictWord{ + 6, + 10, + 437, + }, + dictWord{6, 10, 564}, + dictWord{11, 10, 181}, + dictWord{141, 10, 183}, + dictWord{135, 10, 1192}, + dictWord{7, 0, 166}, + dictWord{132, 11, 763}, + dictWord{133, 11, 253}, + dictWord{134, 0, 849}, + dictWord{9, 11, 73}, + dictWord{10, 11, 110}, + dictWord{14, 11, 185}, + dictWord{145, 11, 119}, + dictWord{5, 11, 212}, + dictWord{12, 11, 35}, + dictWord{141, 11, 382}, + dictWord{133, 0, 717}, + dictWord{137, 0, 304}, + dictWord{136, 0, 600}, + dictWord{133, 0, 654}, + dictWord{ + 6, + 0, + 273, + }, + dictWord{10, 0, 188}, + dictWord{13, 0, 377}, + dictWord{146, 0, 77}, + dictWord{4, 10, 790}, + dictWord{5, 10, 273}, + dictWord{134, 10, 394}, + dictWord{ + 132, + 0, + 543, + }, + dictWord{135, 0, 410}, + dictWord{11, 0, 98}, + dictWord{11, 0, 524}, + dictWord{141, 0, 87}, + dictWord{132, 0, 941}, + dictWord{135, 11, 1175}, + dictWord{ + 4, + 0, + 250, + }, + dictWord{7, 0, 1612}, + dictWord{11, 0, 186}, + dictWord{12, 0, 133}, + dictWord{6, 10, 127}, + dictWord{7, 10, 1511}, + dictWord{8, 10, 613}, + dictWord{ + 12, + 10, + 495, + }, + dictWord{12, 10, 586}, + dictWord{12, 10, 660}, + dictWord{12, 10, 668}, + dictWord{14, 10, 385}, + dictWord{15, 10, 118}, + dictWord{17, 10, 20}, + dictWord{ + 146, + 10, + 98, + }, + dictWord{6, 0, 1785}, + dictWord{133, 11, 816}, + dictWord{134, 0, 1339}, + dictWord{7, 0, 961}, + dictWord{7, 0, 1085}, + dictWord{7, 0, 1727}, + dictWord{ + 8, + 0, + 462, + }, + dictWord{6, 10, 230}, + dictWord{135, 11, 1727}, + dictWord{9, 0, 636}, + dictWord{135, 10, 1954}, + dictWord{132, 0, 780}, + dictWord{5, 11, 869}, + dictWord{5, 11, 968}, + dictWord{6, 11, 1626}, + dictWord{8, 11, 734}, + dictWord{136, 11, 784}, + dictWord{4, 11, 542}, + dictWord{6, 11, 1716}, + dictWord{6, 11, 1727}, + dictWord{7, 11, 1082}, + dictWord{7, 11, 1545}, + dictWord{8, 11, 56}, + dictWord{8, 11, 118}, + dictWord{8, 11, 412}, + dictWord{8, 11, 564}, + dictWord{9, 11, 888}, + dictWord{9, 11, 908}, + dictWord{10, 11, 50}, + dictWord{10, 11, 423}, + dictWord{11, 11, 685}, + dictWord{11, 11, 697}, + dictWord{11, 11, 933}, + dictWord{12, 11, 299}, + dictWord{13, 11, 126}, + dictWord{13, 11, 136}, + dictWord{13, 11, 170}, + dictWord{141, 11, 190}, + dictWord{134, 11, 226}, + dictWord{4, 11, 232}, + dictWord{ + 9, + 11, + 202, + }, + dictWord{10, 11, 474}, + dictWord{140, 11, 433}, + dictWord{137, 11, 500}, + dictWord{5, 0, 529}, + dictWord{136, 10, 68}, + dictWord{132, 10, 654}, + dictWord{ + 4, + 10, + 156, + }, + dictWord{7, 10, 998}, + dictWord{7, 10, 1045}, + dictWord{7, 10, 1860}, + dictWord{9, 10, 48}, + dictWord{9, 10, 692}, + dictWord{11, 10, 419}, + dictWord{139, 10, 602}, + dictWord{7, 0, 1276}, + dictWord{8, 0, 474}, + dictWord{9, 0, 652}, + dictWord{6, 11, 108}, + dictWord{7, 11, 1003}, + dictWord{7, 11, 1181}, + dictWord{136, 11, 343}, + dictWord{7, 11, 1264}, + dictWord{7, 11, 1678}, + dictWord{11, 11, 945}, + dictWord{12, 11, 341}, + dictWord{12, 11, 471}, + dictWord{ + 140, + 11, + 569, + }, + dictWord{134, 11, 1712}, + dictWord{5, 0, 948}, + dictWord{12, 0, 468}, + dictWord{19, 0, 96}, + dictWord{148, 0, 24}, + dictWord{4, 11, 133}, + dictWord{ + 7, + 11, + 711, + }, + dictWord{7, 11, 1298}, + dictWord{7, 11, 1585}, + dictWord{135, 11, 1929}, + dictWord{6, 0, 753}, + dictWord{140, 0, 657}, + dictWord{139, 0, 941}, + dictWord{ + 6, + 11, + 99, + }, + dictWord{7, 11, 1808}, + dictWord{145, 11, 57}, + dictWord{6, 11, 574}, + dictWord{7, 11, 428}, + dictWord{7, 11, 1250}, + dictWord{10, 11, 669}, + dictWord{ + 11, + 11, + 485, + }, + dictWord{11, 11, 840}, + dictWord{12, 11, 300}, + dictWord{142, 11, 250}, + dictWord{4, 0, 532}, + dictWord{5, 0, 706}, + dictWord{135, 0, 662}, + dictWord{ + 5, + 0, + 837, + }, + dictWord{6, 0, 1651}, + dictWord{139, 0, 985}, + dictWord{7, 0, 1861}, + dictWord{9, 10, 197}, + dictWord{10, 10, 300}, + dictWord{12, 10, 473}, + dictWord{ + 13, + 10, + 90, + }, + dictWord{141, 10, 405}, + dictWord{137, 11, 252}, + dictWord{6, 11, 323}, + dictWord{135, 11, 1564}, + dictWord{4, 0, 330}, + dictWord{4, 0, 863}, + dictWord{7, 0, 933}, + dictWord{7, 0, 2012}, + dictWord{8, 0, 292}, + dictWord{7, 11, 461}, + dictWord{8, 11, 775}, + dictWord{138, 11, 435}, + dictWord{132, 10, 606}, + dictWord{ + 4, + 11, + 655, + }, + dictWord{7, 11, 850}, + dictWord{17, 11, 75}, + dictWord{146, 11, 137}, + dictWord{135, 0, 767}, + dictWord{7, 10, 1978}, + dictWord{136, 10, 676}, + dictWord{132, 0, 641}, + dictWord{135, 11, 1559}, + dictWord{134, 0, 1233}, + dictWord{137, 0, 242}, + dictWord{17, 0, 114}, + dictWord{4, 10, 361}, + dictWord{ + 133, + 10, + 315, + }, + dictWord{137, 0, 883}, + dictWord{132, 10, 461}, + dictWord{138, 0, 274}, + dictWord{134, 0, 2008}, + dictWord{134, 0, 1794}, + dictWord{4, 0, 703}, + dictWord{135, 0, 207}, + dictWord{12, 0, 285}, + dictWord{132, 10, 472}, + dictWord{132, 0, 571}, + dictWord{5, 0, 873}, + dictWord{5, 0, 960}, + dictWord{8, 0, 823}, + dictWord{9, 0, 881}, + dictWord{136, 11, 577}, + dictWord{7, 0, 617}, + dictWord{10, 0, 498}, + dictWord{11, 0, 501}, + dictWord{12, 0, 16}, + dictWord{140, 0, 150}, + dictWord{ + 138, + 10, + 747, + }, + dictWord{132, 0, 431}, + dictWord{133, 10, 155}, + dictWord{11, 0, 283}, + dictWord{11, 0, 567}, + dictWord{7, 10, 163}, + dictWord{8, 10, 319}, + dictWord{ + 9, + 10, + 402, + }, + dictWord{10, 10, 24}, + dictWord{10, 10, 681}, + dictWord{11, 10, 200}, + dictWord{12, 10, 253}, + dictWord{12, 10, 410}, + dictWord{142, 10, 219}, + dictWord{4, 11, 413}, + dictWord{5, 11, 677}, + dictWord{8, 11, 432}, + dictWord{140, 11, 280}, + dictWord{9, 0, 401}, + dictWord{5, 10, 475}, + dictWord{7, 10, 1780}, + dictWord{11, 10, 297}, + dictWord{11, 10, 558}, + dictWord{14, 10, 322}, + dictWord{147, 10, 76}, + dictWord{6, 0, 781}, + dictWord{9, 0, 134}, + dictWord{10, 0, 2}, + dictWord{ + 10, + 0, + 27, + }, + dictWord{10, 0, 333}, + dictWord{11, 0, 722}, + dictWord{143, 0, 1}, + dictWord{5, 0, 33}, + dictWord{6, 0, 470}, + dictWord{139, 0, 424}, + dictWord{ + 135, + 0, + 2006, + }, + dictWord{12, 0, 783}, + dictWord{135, 10, 1956}, + dictWord{136, 0, 274}, + dictWord{135, 0, 1882}, + dictWord{132, 0, 794}, + dictWord{135, 0, 1848}, + dictWord{5, 10, 944}, + dictWord{134, 10, 1769}, + dictWord{6, 0, 47}, + dictWord{7, 0, 90}, + dictWord{7, 0, 664}, + dictWord{7, 0, 830}, + dictWord{7, 0, 1380}, + dictWord{ + 7, + 0, + 2025, + }, + dictWord{8, 0, 448}, + dictWord{136, 0, 828}, + dictWord{132, 10, 144}, + dictWord{134, 0, 1199}, + dictWord{4, 11, 395}, + dictWord{139, 11, 762}, + dictWord{135, 11, 1504}, + dictWord{9, 0, 417}, + dictWord{137, 0, 493}, + dictWord{9, 11, 174}, + dictWord{10, 11, 164}, + dictWord{11, 11, 440}, + dictWord{11, 11, 841}, + dictWord{143, 11, 98}, + dictWord{134, 11, 426}, + dictWord{139, 11, 1002}, + dictWord{134, 0, 295}, + dictWord{134, 0, 816}, + dictWord{6, 10, 247}, + dictWord{ + 137, + 10, + 555, + }, + dictWord{133, 0, 1019}, + dictWord{4, 0, 620}, + dictWord{5, 11, 476}, + dictWord{10, 10, 280}, + dictWord{138, 10, 797}, + dictWord{139, 0, 464}, + dictWord{5, 11, 76}, + dictWord{6, 11, 458}, + dictWord{6, 11, 497}, + dictWord{7, 11, 764}, + dictWord{7, 11, 868}, + dictWord{9, 11, 658}, + dictWord{10, 11, 594}, + dictWord{ + 11, + 11, + 173, + }, + dictWord{11, 11, 566}, + dictWord{12, 11, 20}, + dictWord{12, 11, 338}, + dictWord{141, 11, 200}, + dictWord{134, 0, 208}, + dictWord{4, 11, 526}, + dictWord{7, 11, 1029}, + dictWord{135, 11, 1054}, + dictWord{132, 11, 636}, + dictWord{6, 11, 233}, + dictWord{7, 11, 660}, + dictWord{7, 11, 1124}, + dictWord{ + 17, + 11, + 31, + }, + dictWord{19, 11, 22}, + dictWord{151, 11, 14}, + dictWord{10, 0, 442}, + dictWord{133, 10, 428}, + dictWord{10, 0, 930}, + dictWord{140, 0, 778}, + dictWord{ + 6, + 0, + 68, + }, + dictWord{7, 0, 448}, + dictWord{7, 0, 1629}, + dictWord{7, 0, 1769}, + dictWord{7, 0, 1813}, + dictWord{8, 0, 442}, + dictWord{8, 0, 516}, + dictWord{9, 0, 710}, + dictWord{ + 10, + 0, + 282, + }, + dictWord{10, 0, 722}, + dictWord{7, 10, 1717}, + dictWord{138, 10, 546}, + dictWord{134, 0, 1128}, + dictWord{11, 0, 844}, + dictWord{12, 0, 104}, + dictWord{140, 0, 625}, + dictWord{4, 11, 432}, + dictWord{135, 11, 824}, + dictWord{138, 10, 189}, + dictWord{133, 0, 787}, + dictWord{133, 10, 99}, + dictWord{ + 4, + 11, + 279, + }, + dictWord{7, 11, 301}, + dictWord{137, 11, 362}, + dictWord{8, 0, 491}, + dictWord{4, 10, 397}, + dictWord{136, 10, 555}, + dictWord{4, 11, 178}, + dictWord{ + 133, + 11, + 399, + }, + dictWord{134, 0, 711}, + dictWord{144, 0, 9}, + dictWord{4, 0, 403}, + dictWord{5, 0, 441}, + dictWord{7, 0, 450}, + dictWord{10, 0, 840}, + dictWord{11, 0, 101}, + dictWord{12, 0, 193}, + dictWord{141, 0, 430}, + dictWord{135, 11, 1246}, + dictWord{12, 10, 398}, + dictWord{20, 10, 39}, + dictWord{21, 10, 11}, + dictWord{ + 150, + 10, + 41, + }, + dictWord{4, 10, 485}, + dictWord{7, 10, 353}, + dictWord{135, 10, 1523}, + dictWord{6, 10, 366}, + dictWord{7, 10, 1384}, + dictWord{7, 10, 1601}, + dictWord{ + 135, + 11, + 1912, + }, + dictWord{7, 0, 396}, + dictWord{10, 0, 160}, + dictWord{135, 11, 396}, + dictWord{137, 10, 282}, + dictWord{134, 11, 1692}, + dictWord{4, 10, 157}, + dictWord{5, 10, 471}, + dictWord{6, 11, 202}, + dictWord{10, 11, 448}, + dictWord{11, 11, 208}, + dictWord{12, 11, 360}, + dictWord{17, 11, 117}, + dictWord{ + 17, + 11, + 118, + }, + dictWord{18, 11, 27}, + dictWord{148, 11, 67}, + dictWord{133, 0, 679}, + dictWord{137, 0, 326}, + dictWord{136, 10, 116}, + dictWord{7, 11, 872}, + dictWord{ + 10, + 11, + 516, + }, + dictWord{139, 11, 167}, + dictWord{132, 11, 224}, + dictWord{5, 11, 546}, + dictWord{7, 11, 35}, + dictWord{8, 11, 11}, + dictWord{8, 11, 12}, + dictWord{ + 9, + 11, + 315, + }, + dictWord{9, 11, 533}, + dictWord{10, 11, 802}, + dictWord{11, 11, 166}, + dictWord{12, 11, 525}, + dictWord{142, 11, 243}, + dictWord{7, 0, 1128}, + dictWord{135, 11, 1920}, + dictWord{5, 11, 241}, + dictWord{8, 11, 242}, + dictWord{9, 11, 451}, + dictWord{10, 11, 667}, + dictWord{11, 11, 598}, + dictWord{ + 140, + 11, + 429, + }, + dictWord{6, 0, 737}, + dictWord{5, 10, 160}, + dictWord{7, 10, 363}, + dictWord{7, 10, 589}, + dictWord{10, 10, 170}, + dictWord{141, 10, 55}, + dictWord{ + 135, + 0, + 1796, + }, + dictWord{142, 11, 254}, + dictWord{4, 0, 574}, + dictWord{7, 0, 350}, + dictWord{7, 0, 1024}, + dictWord{8, 0, 338}, + dictWord{9, 0, 677}, + dictWord{138, 0, 808}, + dictWord{134, 0, 1096}, + dictWord{137, 11, 516}, + dictWord{7, 0, 405}, + dictWord{10, 0, 491}, + dictWord{4, 10, 108}, + dictWord{4, 11, 366}, + dictWord{ + 139, + 10, + 498, + }, + dictWord{11, 11, 337}, + dictWord{142, 11, 303}, + dictWord{134, 11, 1736}, + dictWord{7, 0, 1081}, + dictWord{140, 11, 364}, + dictWord{7, 10, 1005}, + dictWord{140, 10, 609}, + dictWord{7, 0, 1676}, + dictWord{4, 10, 895}, + dictWord{133, 10, 772}, + dictWord{135, 0, 2037}, + dictWord{6, 0, 1207}, + dictWord{ + 11, + 11, + 916, + }, + dictWord{142, 11, 419}, + dictWord{14, 11, 140}, + dictWord{148, 11, 41}, + dictWord{6, 11, 331}, + dictWord{136, 11, 623}, + dictWord{9, 0, 944}, + dictWord{ + 9, + 0, + 969, + }, + dictWord{9, 0, 1022}, + dictWord{12, 0, 913}, + dictWord{12, 0, 936}, + dictWord{15, 0, 177}, + dictWord{15, 0, 193}, + dictWord{4, 10, 926}, + dictWord{ + 133, + 10, + 983, + }, + dictWord{5, 0, 354}, + dictWord{135, 11, 506}, + dictWord{8, 0, 598}, + dictWord{9, 0, 664}, + dictWord{138, 0, 441}, + dictWord{4, 11, 640}, + dictWord{ + 133, + 11, + 513, + }, + dictWord{137, 0, 297}, + dictWord{132, 10, 538}, + dictWord{6, 10, 294}, + dictWord{7, 10, 1267}, + dictWord{136, 10, 624}, + dictWord{7, 0, 1772}, + dictWord{ + 7, + 11, + 1888, + }, + dictWord{8, 11, 289}, + dictWord{11, 11, 45}, + dictWord{12, 11, 278}, + dictWord{140, 11, 537}, + dictWord{135, 10, 1325}, + dictWord{138, 0, 751}, + dictWord{141, 0, 37}, + dictWord{134, 0, 1828}, + dictWord{132, 10, 757}, + dictWord{132, 11, 394}, + dictWord{6, 0, 257}, + dictWord{135, 0, 1522}, + dictWord{ + 4, + 0, + 582, + }, + dictWord{9, 0, 191}, + dictWord{135, 11, 1931}, + dictWord{7, 11, 574}, + dictWord{7, 11, 1719}, + dictWord{137, 11, 145}, + dictWord{132, 11, 658}, + dictWord{10, 0, 790}, + dictWord{132, 11, 369}, + dictWord{9, 11, 781}, + dictWord{10, 11, 144}, + dictWord{11, 11, 385}, + dictWord{13, 11, 161}, + dictWord{13, 11, 228}, + dictWord{13, 11, 268}, + dictWord{148, 11, 107}, + dictWord{8, 0, 469}, + dictWord{10, 0, 47}, + dictWord{136, 11, 374}, + dictWord{6, 0, 306}, + dictWord{7, 0, 1140}, + dictWord{7, 0, 1340}, + dictWord{8, 0, 133}, + dictWord{138, 0, 449}, + dictWord{139, 0, 1011}, + dictWord{7, 10, 1875}, + dictWord{139, 10, 124}, + dictWord{ + 4, + 11, + 344, + }, + dictWord{6, 11, 498}, + dictWord{139, 11, 323}, + dictWord{137, 0, 299}, + dictWord{132, 0, 837}, + dictWord{133, 11, 906}, + dictWord{5, 0, 329}, + dictWord{ + 8, + 0, + 260, + }, + dictWord{138, 0, 10}, + dictWord{134, 0, 1320}, + dictWord{4, 0, 657}, + dictWord{146, 0, 158}, + dictWord{135, 0, 1191}, + dictWord{152, 0, 7}, + dictWord{ + 6, + 0, + 1939, + }, + dictWord{8, 0, 974}, + dictWord{138, 0, 996}, + dictWord{135, 0, 1665}, + dictWord{11, 11, 126}, + dictWord{139, 11, 287}, + dictWord{143, 0, 8}, + dictWord{ + 14, + 11, + 149, + }, + dictWord{14, 11, 399}, + dictWord{143, 11, 57}, + dictWord{5, 0, 66}, + dictWord{7, 0, 1896}, + dictWord{136, 0, 288}, + dictWord{7, 0, 175}, + dictWord{ + 10, + 0, + 494, + }, + dictWord{5, 10, 150}, + dictWord{8, 10, 603}, + dictWord{9, 10, 593}, + dictWord{9, 10, 634}, + dictWord{10, 10, 173}, + dictWord{11, 10, 462}, + dictWord{ + 11, + 10, + 515, + }, + dictWord{13, 10, 216}, + dictWord{13, 10, 288}, + dictWord{142, 10, 400}, + dictWord{134, 0, 1643}, + dictWord{136, 11, 21}, + dictWord{4, 0, 21}, + dictWord{ + 5, + 0, + 91, + }, + dictWord{5, 0, 648}, + dictWord{5, 0, 750}, + dictWord{5, 0, 781}, + dictWord{6, 0, 54}, + dictWord{6, 0, 112}, + dictWord{6, 0, 402}, + dictWord{6, 0, 1732}, + dictWord{ + 7, + 0, + 315, + }, + dictWord{7, 0, 749}, + dictWord{7, 0, 1427}, + dictWord{7, 0, 1900}, + dictWord{9, 0, 78}, + dictWord{9, 0, 508}, + dictWord{10, 0, 611}, + dictWord{10, 0, 811}, + dictWord{11, 0, 510}, + dictWord{11, 0, 728}, + dictWord{13, 0, 36}, + dictWord{14, 0, 39}, + dictWord{16, 0, 83}, + dictWord{17, 0, 124}, + dictWord{148, 0, 30}, + dictWord{ + 4, + 0, + 668, + }, + dictWord{136, 0, 570}, + dictWord{10, 0, 322}, + dictWord{10, 0, 719}, + dictWord{139, 0, 407}, + dictWord{135, 11, 1381}, + dictWord{136, 11, 193}, + dictWord{12, 10, 108}, + dictWord{141, 10, 291}, + dictWord{132, 11, 616}, + dictWord{136, 11, 692}, + dictWord{8, 0, 125}, + dictWord{8, 0, 369}, + dictWord{8, 0, 524}, + dictWord{10, 0, 486}, + dictWord{11, 0, 13}, + dictWord{11, 0, 381}, + dictWord{11, 0, 736}, + dictWord{11, 0, 766}, + dictWord{11, 0, 845}, + dictWord{13, 0, 114}, + dictWord{ + 13, + 0, + 292, + }, + dictWord{142, 0, 47}, + dictWord{134, 0, 1247}, + dictWord{6, 0, 1684}, + dictWord{6, 0, 1731}, + dictWord{7, 0, 356}, + dictWord{8, 0, 54}, + dictWord{8, 0, 221}, + dictWord{9, 0, 225}, + dictWord{9, 0, 356}, + dictWord{10, 0, 77}, + dictWord{10, 0, 446}, + dictWord{10, 0, 731}, + dictWord{12, 0, 404}, + dictWord{141, 0, 491}, + dictWord{135, 10, 1777}, + dictWord{4, 11, 305}, + dictWord{4, 10, 493}, + dictWord{144, 10, 55}, + dictWord{4, 0, 951}, + dictWord{6, 0, 1809}, + dictWord{6, 0, 1849}, + dictWord{8, 0, 846}, + dictWord{8, 0, 866}, + dictWord{8, 0, 899}, + dictWord{10, 0, 896}, + dictWord{12, 0, 694}, + dictWord{142, 0, 468}, + dictWord{5, 11, 214}, + dictWord{ + 7, + 11, + 603, + }, + dictWord{8, 11, 611}, + dictWord{9, 11, 686}, + dictWord{10, 11, 88}, + dictWord{11, 11, 459}, + dictWord{11, 11, 496}, + dictWord{12, 11, 463}, + dictWord{ + 12, + 11, + 590, + }, + dictWord{13, 11, 0}, + dictWord{142, 11, 214}, + dictWord{132, 0, 411}, + dictWord{4, 0, 80}, + dictWord{133, 0, 44}, + dictWord{140, 11, 74}, + dictWord{ + 143, + 0, + 31, + }, + dictWord{7, 0, 669}, + dictWord{6, 10, 568}, + dictWord{7, 10, 1804}, + dictWord{8, 10, 362}, + dictWord{8, 10, 410}, + dictWord{8, 10, 830}, + dictWord{9, 10, 514}, + dictWord{11, 10, 649}, + dictWord{142, 10, 157}, + dictWord{7, 0, 673}, + dictWord{134, 11, 1703}, + dictWord{132, 10, 625}, + dictWord{134, 0, 1303}, + dictWord{ + 5, + 0, + 299, + }, + dictWord{135, 0, 1083}, + dictWord{138, 0, 704}, + dictWord{6, 0, 275}, + dictWord{7, 0, 408}, + dictWord{6, 10, 158}, + dictWord{7, 10, 129}, + dictWord{ + 7, + 10, + 181, + }, + dictWord{8, 10, 276}, + dictWord{8, 10, 377}, + dictWord{10, 10, 523}, + dictWord{11, 10, 816}, + dictWord{12, 10, 455}, + dictWord{13, 10, 303}, + dictWord{ + 142, + 10, + 135, + }, + dictWord{4, 0, 219}, + dictWord{7, 0, 367}, + dictWord{7, 0, 1713}, + dictWord{7, 0, 1761}, + dictWord{9, 0, 86}, + dictWord{9, 0, 537}, + dictWord{10, 0, 165}, + dictWord{12, 0, 219}, + dictWord{140, 0, 561}, + dictWord{8, 0, 216}, + dictWord{4, 10, 1}, + dictWord{4, 11, 737}, + dictWord{6, 11, 317}, + dictWord{7, 10, 1143}, + dictWord{ + 7, + 10, + 1463, + }, + dictWord{9, 10, 207}, + dictWord{9, 10, 390}, + dictWord{9, 10, 467}, + dictWord{10, 11, 98}, + dictWord{11, 11, 294}, + dictWord{11, 10, 836}, + dictWord{ + 12, + 11, + 60, + }, + dictWord{12, 11, 437}, + dictWord{13, 11, 64}, + dictWord{13, 11, 380}, + dictWord{142, 11, 430}, + dictWord{6, 11, 1758}, + dictWord{8, 11, 520}, + dictWord{9, 11, 345}, + dictWord{9, 11, 403}, + dictWord{142, 11, 350}, + dictWord{5, 11, 47}, + dictWord{10, 11, 242}, + dictWord{138, 11, 579}, + dictWord{5, 11, 139}, + dictWord{7, 11, 1168}, + dictWord{138, 11, 539}, + dictWord{135, 0, 1319}, + dictWord{4, 10, 295}, + dictWord{4, 10, 723}, + dictWord{5, 10, 895}, + dictWord{ + 7, + 10, + 1031, + }, + dictWord{8, 10, 199}, + dictWord{8, 10, 340}, + dictWord{9, 10, 153}, + dictWord{9, 10, 215}, + dictWord{10, 10, 21}, + dictWord{10, 10, 59}, + dictWord{ + 10, + 10, + 80, + }, + dictWord{10, 10, 224}, + dictWord{10, 10, 838}, + dictWord{11, 10, 229}, + dictWord{11, 10, 652}, + dictWord{12, 10, 192}, + dictWord{13, 10, 146}, + dictWord{ + 142, + 10, + 91, + }, + dictWord{140, 0, 428}, + dictWord{137, 10, 51}, + dictWord{133, 0, 514}, + dictWord{5, 10, 309}, + dictWord{140, 10, 211}, + dictWord{6, 0, 1010}, + dictWord{5, 10, 125}, + dictWord{8, 10, 77}, + dictWord{138, 10, 15}, + dictWord{4, 0, 55}, + dictWord{5, 0, 301}, + dictWord{6, 0, 571}, + dictWord{142, 0, 49}, + dictWord{ + 146, + 0, + 102, + }, + dictWord{136, 11, 370}, + dictWord{4, 11, 107}, + dictWord{7, 11, 613}, + dictWord{8, 11, 358}, + dictWord{8, 11, 439}, + dictWord{8, 11, 504}, + dictWord{ + 9, + 11, + 501, + }, + dictWord{10, 11, 383}, + dictWord{139, 11, 477}, + dictWord{132, 11, 229}, + dictWord{133, 0, 364}, + dictWord{133, 10, 439}, + dictWord{4, 11, 903}, + dictWord{135, 11, 1816}, + dictWord{11, 0, 379}, + dictWord{140, 10, 76}, + dictWord{4, 0, 76}, + dictWord{4, 0, 971}, + dictWord{7, 0, 1550}, + dictWord{9, 0, 306}, + dictWord{ + 9, + 0, + 430, + }, + dictWord{9, 0, 663}, + dictWord{10, 0, 683}, + dictWord{10, 0, 921}, + dictWord{11, 0, 427}, + dictWord{11, 0, 753}, + dictWord{12, 0, 334}, + dictWord{12, 0, 442}, + dictWord{14, 0, 258}, + dictWord{14, 0, 366}, + dictWord{143, 0, 131}, + dictWord{137, 0, 52}, + dictWord{4, 11, 47}, + dictWord{6, 11, 373}, + dictWord{7, 11, 452}, + dictWord{7, 11, 543}, + dictWord{7, 11, 1714}, + dictWord{7, 11, 1856}, + dictWord{9, 11, 6}, + dictWord{11, 11, 257}, + dictWord{139, 11, 391}, + dictWord{4, 10, 8}, + dictWord{ + 7, + 10, + 1152, + }, + dictWord{7, 10, 1153}, + dictWord{7, 10, 1715}, + dictWord{9, 10, 374}, + dictWord{10, 10, 478}, + dictWord{139, 10, 648}, + dictWord{4, 11, 785}, + dictWord{133, 11, 368}, + dictWord{135, 10, 1099}, + dictWord{135, 11, 860}, + dictWord{5, 11, 980}, + dictWord{134, 11, 1754}, + dictWord{134, 0, 1258}, + dictWord{ + 6, + 0, + 1058, + }, + dictWord{6, 0, 1359}, + dictWord{7, 11, 536}, + dictWord{7, 11, 1331}, + dictWord{136, 11, 143}, + dictWord{4, 0, 656}, + dictWord{135, 0, 779}, + dictWord{136, 10, 87}, + dictWord{5, 11, 19}, + dictWord{6, 11, 533}, + dictWord{146, 11, 126}, + dictWord{7, 0, 144}, + dictWord{138, 10, 438}, + dictWord{5, 11, 395}, + dictWord{5, 11, 951}, + dictWord{134, 11, 1776}, + dictWord{135, 0, 1373}, + dictWord{7, 0, 554}, + dictWord{7, 0, 605}, + dictWord{141, 0, 10}, + dictWord{4, 10, 69}, + dictWord{ + 5, + 10, + 122, + }, + dictWord{9, 10, 656}, + dictWord{138, 10, 464}, + dictWord{5, 10, 849}, + dictWord{134, 10, 1633}, + dictWord{5, 0, 838}, + dictWord{5, 0, 841}, + dictWord{134, 0, 1649}, + dictWord{133, 0, 1012}, + dictWord{139, 10, 499}, + dictWord{7, 10, 476}, + dictWord{7, 10, 1592}, + dictWord{138, 10, 87}, + dictWord{ + 6, + 0, + 251, + }, + dictWord{7, 0, 365}, + dictWord{7, 0, 1357}, + dictWord{7, 0, 1497}, + dictWord{8, 0, 154}, + dictWord{141, 0, 281}, + dictWord{132, 11, 441}, + dictWord{ + 132, + 11, + 695, + }, + dictWord{7, 11, 497}, + dictWord{9, 11, 387}, + dictWord{147, 11, 81}, + dictWord{133, 0, 340}, + dictWord{14, 10, 283}, + dictWord{142, 11, 283}, + dictWord{ + 134, + 0, + 810, + }, + dictWord{135, 11, 1894}, + dictWord{139, 0, 495}, + dictWord{5, 11, 284}, + dictWord{6, 11, 49}, + dictWord{6, 11, 350}, + dictWord{7, 11, 1}, + dictWord{ + 7, + 11, + 377, + }, + dictWord{7, 11, 1693}, + dictWord{8, 11, 18}, + dictWord{8, 11, 678}, + dictWord{9, 11, 161}, + dictWord{9, 11, 585}, + dictWord{9, 11, 671}, + dictWord{ + 9, + 11, + 839, + }, + dictWord{11, 11, 912}, + dictWord{141, 11, 427}, + dictWord{5, 10, 859}, + dictWord{7, 10, 1160}, + dictWord{8, 10, 107}, + dictWord{9, 10, 291}, + dictWord{ + 9, + 10, + 439, + }, + dictWord{10, 10, 663}, + dictWord{11, 10, 609}, + dictWord{140, 10, 197}, + dictWord{8, 0, 261}, + dictWord{9, 0, 144}, + dictWord{9, 0, 466}, + dictWord{ + 10, + 0, + 370, + }, + dictWord{12, 0, 470}, + dictWord{13, 0, 144}, + dictWord{142, 0, 348}, + dictWord{137, 0, 897}, + dictWord{6, 0, 248}, + dictWord{9, 0, 546}, + dictWord{10, 0, 535}, + dictWord{11, 0, 681}, + dictWord{141, 0, 135}, + dictWord{4, 0, 358}, + dictWord{135, 0, 1496}, + dictWord{134, 0, 567}, + dictWord{136, 0, 445}, + dictWord{ + 4, + 10, + 117, + }, + dictWord{6, 10, 372}, + dictWord{7, 10, 1905}, + dictWord{142, 10, 323}, + dictWord{4, 10, 722}, + dictWord{139, 10, 471}, + dictWord{6, 0, 697}, + dictWord{ + 134, + 0, + 996, + }, + dictWord{7, 11, 2007}, + dictWord{9, 11, 101}, + dictWord{9, 11, 450}, + dictWord{10, 11, 66}, + dictWord{10, 11, 842}, + dictWord{11, 11, 536}, + dictWord{ + 140, + 11, + 587, + }, + dictWord{132, 0, 577}, + dictWord{134, 0, 1336}, + dictWord{9, 10, 5}, + dictWord{12, 10, 216}, + dictWord{12, 10, 294}, + dictWord{12, 10, 298}, + dictWord{12, 10, 400}, + dictWord{12, 10, 518}, + dictWord{13, 10, 229}, + dictWord{143, 10, 139}, + dictWord{6, 0, 174}, + dictWord{138, 0, 917}, + dictWord{ + 134, + 10, + 1774, + }, + dictWord{5, 10, 12}, + dictWord{7, 10, 375}, + dictWord{9, 10, 88}, + dictWord{9, 10, 438}, + dictWord{11, 11, 62}, + dictWord{139, 10, 270}, + dictWord{ + 134, + 11, + 1766, + }, + dictWord{6, 11, 0}, + dictWord{7, 11, 84}, + dictWord{7, 10, 816}, + dictWord{7, 10, 1241}, + dictWord{9, 10, 283}, + dictWord{9, 10, 520}, + dictWord{10, 10, 213}, + dictWord{10, 10, 307}, + dictWord{10, 10, 463}, + dictWord{10, 10, 671}, + dictWord{10, 10, 746}, + dictWord{11, 10, 401}, + dictWord{11, 10, 794}, + dictWord{ + 11, + 11, + 895, + }, + dictWord{12, 10, 517}, + dictWord{17, 11, 11}, + dictWord{18, 10, 107}, + dictWord{147, 10, 115}, + dictWord{5, 0, 878}, + dictWord{133, 0, 972}, + dictWord{ + 6, + 11, + 1665, + }, + dictWord{7, 11, 256}, + dictWord{7, 11, 1388}, + dictWord{138, 11, 499}, + dictWord{4, 10, 258}, + dictWord{136, 10, 639}, + dictWord{4, 11, 22}, + dictWord{5, 11, 10}, + dictWord{6, 10, 22}, + dictWord{7, 11, 848}, + dictWord{7, 10, 903}, + dictWord{7, 10, 1963}, + dictWord{8, 11, 97}, + dictWord{138, 10, 577}, + dictWord{ + 5, + 10, + 681, + }, + dictWord{136, 10, 782}, + dictWord{133, 11, 481}, + dictWord{132, 0, 351}, + dictWord{4, 10, 664}, + dictWord{5, 10, 804}, + dictWord{139, 10, 1013}, + dictWord{6, 11, 134}, + dictWord{7, 11, 437}, + dictWord{7, 11, 959}, + dictWord{9, 11, 37}, + dictWord{14, 11, 285}, + dictWord{14, 11, 371}, + dictWord{144, 11, 60}, + dictWord{7, 11, 486}, + dictWord{8, 11, 155}, + dictWord{11, 11, 93}, + dictWord{140, 11, 164}, + dictWord{132, 0, 286}, + dictWord{7, 0, 438}, + dictWord{7, 0, 627}, + dictWord{7, 0, 1516}, + dictWord{8, 0, 40}, + dictWord{9, 0, 56}, + dictWord{9, 0, 294}, + dictWord{10, 0, 30}, + dictWord{11, 0, 969}, + dictWord{11, 0, 995}, + dictWord{146, 0, 148}, + dictWord{5, 11, 591}, + dictWord{135, 11, 337}, + dictWord{134, 0, 1950}, + dictWord{133, 10, 32}, + dictWord{138, 11, 500}, + dictWord{5, 11, 380}, + dictWord{ + 5, + 11, + 650, + }, + dictWord{136, 11, 310}, + dictWord{4, 11, 364}, + dictWord{7, 11, 1156}, + dictWord{7, 11, 1187}, + dictWord{137, 11, 409}, + dictWord{4, 0, 738}, + dictWord{134, 11, 482}, + dictWord{4, 11, 781}, + dictWord{6, 11, 487}, + dictWord{7, 11, 926}, + dictWord{8, 11, 263}, + dictWord{139, 11, 500}, + dictWord{135, 11, 418}, + dictWord{6, 0, 2047}, + dictWord{10, 0, 969}, + dictWord{4, 10, 289}, + dictWord{7, 10, 629}, + dictWord{7, 10, 1698}, + dictWord{7, 10, 1711}, + dictWord{ + 140, + 10, + 215, + }, + dictWord{6, 10, 450}, + dictWord{136, 10, 109}, + dictWord{134, 0, 818}, + dictWord{136, 10, 705}, + dictWord{133, 0, 866}, + dictWord{4, 11, 94}, + dictWord{ + 135, + 11, + 1265, + }, + dictWord{132, 11, 417}, + dictWord{134, 0, 1467}, + dictWord{135, 10, 1238}, + dictWord{4, 0, 972}, + dictWord{6, 0, 1851}, + dictWord{ + 134, + 0, + 1857, + }, + dictWord{134, 0, 355}, + dictWord{133, 0, 116}, + dictWord{132, 0, 457}, + dictWord{135, 11, 1411}, + dictWord{4, 11, 408}, + dictWord{4, 11, 741}, + dictWord{135, 11, 500}, + dictWord{134, 10, 26}, + dictWord{142, 11, 137}, + dictWord{5, 0, 527}, + dictWord{6, 0, 189}, + dictWord{7, 0, 859}, + dictWord{136, 0, 267}, + dictWord{11, 0, 104}, + dictWord{11, 0, 554}, + dictWord{15, 0, 60}, + dictWord{143, 0, 125}, + dictWord{134, 0, 1613}, + dictWord{4, 10, 414}, + dictWord{5, 10, 467}, + dictWord{ + 9, + 10, + 654, + }, + dictWord{10, 10, 451}, + dictWord{12, 10, 59}, + dictWord{141, 10, 375}, + dictWord{135, 10, 17}, + dictWord{134, 0, 116}, + dictWord{135, 11, 541}, + dictWord{135, 10, 955}, + dictWord{6, 11, 73}, + dictWord{135, 11, 177}, + dictWord{133, 11, 576}, + dictWord{134, 0, 886}, + dictWord{133, 0, 487}, + dictWord{ + 4, + 0, + 86, + }, + dictWord{5, 0, 667}, + dictWord{5, 0, 753}, + dictWord{6, 0, 316}, + dictWord{6, 0, 455}, + dictWord{135, 0, 946}, + dictWord{142, 11, 231}, + dictWord{150, 0, 45}, + dictWord{134, 0, 863}, + dictWord{134, 0, 1953}, + dictWord{6, 10, 280}, + dictWord{10, 10, 502}, + dictWord{11, 10, 344}, + dictWord{140, 10, 38}, + dictWord{4, 0, 79}, + dictWord{7, 0, 1773}, + dictWord{10, 0, 450}, + dictWord{11, 0, 589}, + dictWord{13, 0, 332}, + dictWord{13, 0, 493}, + dictWord{14, 0, 183}, + dictWord{14, 0, 334}, + dictWord{14, 0, 362}, + dictWord{14, 0, 368}, + dictWord{14, 0, 376}, + dictWord{14, 0, 379}, + dictWord{19, 0, 90}, + dictWord{19, 0, 103}, + dictWord{19, 0, 127}, + dictWord{ + 148, + 0, + 90, + }, + dictWord{5, 10, 45}, + dictWord{7, 10, 1161}, + dictWord{11, 10, 448}, + dictWord{11, 10, 880}, + dictWord{13, 10, 139}, + dictWord{13, 10, 407}, + dictWord{ + 15, + 10, + 16, + }, + dictWord{17, 10, 95}, + dictWord{18, 10, 66}, + dictWord{18, 10, 88}, + dictWord{18, 10, 123}, + dictWord{149, 10, 7}, + dictWord{136, 10, 777}, + dictWord{ + 4, + 10, + 410, + }, + dictWord{135, 10, 521}, + dictWord{135, 10, 1778}, + dictWord{135, 11, 538}, + dictWord{142, 0, 381}, + dictWord{133, 11, 413}, + dictWord{ + 134, + 0, + 1142, + }, + dictWord{6, 0, 1189}, + dictWord{136, 11, 495}, + dictWord{5, 0, 663}, + dictWord{6, 0, 1962}, + dictWord{134, 0, 2003}, + dictWord{7, 11, 54}, + dictWord{ + 8, + 11, + 312, + }, + dictWord{10, 11, 191}, + dictWord{10, 11, 614}, + dictWord{140, 11, 567}, + dictWord{132, 10, 436}, + dictWord{133, 0, 846}, + dictWord{10, 0, 528}, + dictWord{11, 0, 504}, + dictWord{7, 10, 1587}, + dictWord{135, 10, 1707}, + dictWord{5, 0, 378}, + dictWord{8, 0, 465}, + dictWord{9, 0, 286}, + dictWord{10, 0, 185}, + dictWord{ + 10, + 0, + 562, + }, + dictWord{10, 0, 635}, + dictWord{11, 0, 31}, + dictWord{11, 0, 393}, + dictWord{13, 0, 312}, + dictWord{18, 0, 65}, + dictWord{18, 0, 96}, + dictWord{147, 0, 89}, + dictWord{7, 0, 899}, + dictWord{14, 0, 325}, + dictWord{6, 11, 468}, + dictWord{7, 11, 567}, + dictWord{7, 11, 1478}, + dictWord{8, 11, 530}, + dictWord{142, 11, 290}, + dictWord{7, 0, 1880}, + dictWord{9, 0, 680}, + dictWord{139, 0, 798}, + dictWord{134, 0, 1770}, + dictWord{132, 0, 648}, + dictWord{150, 11, 35}, + dictWord{5, 0, 945}, + dictWord{6, 0, 1656}, + dictWord{6, 0, 1787}, + dictWord{7, 0, 167}, + dictWord{8, 0, 824}, + dictWord{9, 0, 391}, + dictWord{10, 0, 375}, + dictWord{139, 0, 185}, + dictWord{ + 6, + 11, + 484, + }, + dictWord{135, 11, 822}, + dictWord{134, 0, 2046}, + dictWord{7, 0, 1645}, + dictWord{8, 0, 352}, + dictWord{137, 0, 249}, + dictWord{132, 0, 152}, + dictWord{6, 0, 611}, + dictWord{135, 0, 1733}, + dictWord{6, 11, 1724}, + dictWord{135, 11, 2022}, + dictWord{133, 0, 1006}, + dictWord{141, 11, 96}, + dictWord{ + 5, + 0, + 420, + }, + dictWord{135, 0, 1449}, + dictWord{146, 11, 149}, + dictWord{135, 0, 832}, + dictWord{135, 10, 663}, + dictWord{133, 0, 351}, + dictWord{5, 0, 40}, + dictWord{ + 7, + 0, + 598, + }, + dictWord{7, 0, 1638}, + dictWord{8, 0, 78}, + dictWord{9, 0, 166}, + dictWord{9, 0, 640}, + dictWord{9, 0, 685}, + dictWord{9, 0, 773}, + dictWord{11, 0, 215}, + dictWord{13, 0, 65}, + dictWord{14, 0, 172}, + dictWord{14, 0, 317}, + dictWord{145, 0, 6}, + dictWord{8, 0, 60}, + dictWord{9, 0, 343}, + dictWord{139, 0, 769}, + dictWord{ + 134, + 0, + 1354, + }, + dictWord{132, 0, 724}, + dictWord{137, 0, 745}, + dictWord{132, 11, 474}, + dictWord{7, 0, 1951}, + dictWord{8, 0, 765}, + dictWord{8, 0, 772}, + dictWord{ + 140, + 0, + 671, + }, + dictWord{7, 0, 108}, + dictWord{8, 0, 219}, + dictWord{8, 0, 388}, + dictWord{9, 0, 775}, + dictWord{11, 0, 275}, + dictWord{140, 0, 464}, + dictWord{137, 0, 639}, + dictWord{135, 10, 503}, + dictWord{133, 11, 366}, + dictWord{5, 0, 15}, + dictWord{6, 0, 56}, + dictWord{7, 0, 1758}, + dictWord{8, 0, 500}, + dictWord{9, 0, 730}, + dictWord{ + 11, + 0, + 331, + }, + dictWord{13, 0, 150}, + dictWord{14, 0, 282}, + dictWord{5, 11, 305}, + dictWord{9, 11, 560}, + dictWord{141, 11, 208}, + dictWord{4, 10, 113}, + dictWord{ + 5, + 10, + 163, + }, + dictWord{5, 10, 735}, + dictWord{7, 10, 1009}, + dictWord{9, 10, 9}, + dictWord{9, 10, 771}, + dictWord{12, 10, 90}, + dictWord{13, 10, 138}, + dictWord{ + 13, + 10, + 410, + }, + dictWord{143, 10, 128}, + dictWord{4, 10, 324}, + dictWord{138, 10, 104}, + dictWord{135, 11, 466}, + dictWord{142, 11, 27}, + dictWord{134, 0, 1886}, + dictWord{5, 0, 205}, + dictWord{6, 0, 438}, + dictWord{9, 0, 711}, + dictWord{4, 11, 480}, + dictWord{6, 11, 167}, + dictWord{6, 11, 302}, + dictWord{6, 11, 1642}, + dictWord{ + 7, + 11, + 130, + }, + dictWord{7, 11, 656}, + dictWord{7, 11, 837}, + dictWord{7, 11, 1547}, + dictWord{7, 11, 1657}, + dictWord{8, 11, 429}, + dictWord{9, 11, 228}, + dictWord{ + 10, + 11, + 643, + }, + dictWord{13, 11, 289}, + dictWord{13, 11, 343}, + dictWord{147, 11, 101}, + dictWord{134, 0, 865}, + dictWord{6, 0, 2025}, + dictWord{136, 0, 965}, + dictWord{ + 7, + 11, + 278, + }, + dictWord{10, 11, 739}, + dictWord{11, 11, 708}, + dictWord{141, 11, 348}, + dictWord{133, 0, 534}, + dictWord{135, 11, 1922}, + dictWord{ + 137, + 0, + 691, + }, + dictWord{4, 10, 935}, + dictWord{133, 10, 823}, + dictWord{6, 0, 443}, + dictWord{9, 0, 237}, + dictWord{9, 0, 571}, + dictWord{9, 0, 695}, + dictWord{10, 0, 139}, + dictWord{11, 0, 715}, + dictWord{12, 0, 417}, + dictWord{141, 0, 421}, + dictWord{5, 10, 269}, + dictWord{7, 10, 434}, + dictWord{7, 10, 891}, + dictWord{8, 10, 339}, + dictWord{ + 9, + 10, + 702, + }, + dictWord{11, 10, 594}, + dictWord{11, 10, 718}, + dictWord{145, 10, 100}, + dictWord{6, 0, 1555}, + dictWord{7, 0, 878}, + dictWord{9, 10, 485}, + dictWord{141, 10, 264}, + dictWord{134, 10, 1713}, + dictWord{7, 10, 1810}, + dictWord{11, 10, 866}, + dictWord{12, 10, 103}, + dictWord{141, 10, 495}, + dictWord{ + 135, + 10, + 900, + }, + dictWord{6, 0, 1410}, + dictWord{9, 11, 316}, + dictWord{139, 11, 256}, + dictWord{4, 0, 995}, + dictWord{135, 0, 1033}, + dictWord{132, 0, 578}, + dictWord{10, 0, 881}, + dictWord{12, 0, 740}, + dictWord{12, 0, 743}, + dictWord{140, 0, 759}, + dictWord{132, 0, 822}, + dictWord{133, 0, 923}, + dictWord{142, 10, 143}, + dictWord{135, 11, 1696}, + dictWord{6, 11, 363}, + dictWord{7, 11, 1955}, + dictWord{136, 11, 725}, + dictWord{132, 0, 924}, + dictWord{133, 0, 665}, + dictWord{ + 135, + 10, + 2029, + }, + dictWord{135, 0, 1901}, + dictWord{4, 0, 265}, + dictWord{6, 0, 1092}, + dictWord{6, 0, 1417}, + dictWord{7, 0, 807}, + dictWord{135, 0, 950}, + dictWord{ + 5, + 0, + 93, + }, + dictWord{12, 0, 267}, + dictWord{141, 0, 498}, + dictWord{135, 0, 1451}, + dictWord{5, 11, 813}, + dictWord{135, 11, 2046}, + dictWord{5, 10, 625}, + dictWord{135, 10, 1617}, + dictWord{135, 0, 747}, + dictWord{6, 0, 788}, + dictWord{137, 0, 828}, + dictWord{7, 0, 184}, + dictWord{11, 0, 307}, + dictWord{11, 0, 400}, + dictWord{15, 0, 130}, + dictWord{5, 11, 712}, + dictWord{7, 11, 1855}, + dictWord{8, 10, 425}, + dictWord{8, 10, 693}, + dictWord{9, 10, 720}, + dictWord{10, 10, 380}, + dictWord{10, 10, 638}, + dictWord{11, 11, 17}, + dictWord{11, 10, 473}, + dictWord{12, 10, 61}, + dictWord{13, 11, 321}, + dictWord{144, 11, 67}, + dictWord{135, 0, 198}, + dictWord{6, 11, 320}, + dictWord{7, 11, 781}, + dictWord{7, 11, 1921}, + dictWord{9, 11, 55}, + dictWord{10, 11, 186}, + dictWord{10, 11, 273}, + dictWord{10, 11, 664}, + dictWord{10, 11, 801}, + dictWord{11, 11, 996}, + dictWord{11, 11, 997}, + dictWord{13, 11, 157}, + dictWord{142, 11, 170}, + dictWord{136, 11, 271}, + dictWord{ + 135, + 0, + 994, + }, + dictWord{7, 11, 103}, + dictWord{7, 11, 863}, + dictWord{11, 11, 184}, + dictWord{14, 11, 299}, + dictWord{145, 11, 62}, + dictWord{11, 10, 551}, + dictWord{142, 10, 159}, + dictWord{5, 0, 233}, + dictWord{5, 0, 320}, + dictWord{6, 0, 140}, + dictWord{8, 0, 295}, + dictWord{8, 0, 615}, + dictWord{136, 11, 615}, + dictWord{ + 133, + 0, + 978, + }, + dictWord{4, 0, 905}, + dictWord{6, 0, 1701}, + dictWord{137, 0, 843}, + dictWord{132, 10, 168}, + dictWord{4, 0, 974}, + dictWord{8, 0, 850}, + dictWord{ + 12, + 0, + 709, + }, + dictWord{12, 0, 768}, + dictWord{140, 0, 786}, + dictWord{135, 10, 91}, + dictWord{152, 0, 6}, + dictWord{138, 10, 532}, + dictWord{135, 10, 1884}, + dictWord{132, 0, 509}, + dictWord{6, 0, 1307}, + dictWord{135, 0, 273}, + dictWord{5, 11, 77}, + dictWord{7, 11, 1455}, + dictWord{10, 11, 843}, + dictWord{19, 11, 73}, + dictWord{150, 11, 5}, + dictWord{132, 11, 458}, + dictWord{135, 11, 1420}, + dictWord{6, 11, 109}, + dictWord{138, 11, 382}, + dictWord{6, 0, 201}, + dictWord{6, 11, 330}, + dictWord{7, 10, 70}, + dictWord{7, 11, 1084}, + dictWord{10, 10, 240}, + dictWord{11, 11, 142}, + dictWord{147, 10, 93}, + dictWord{7, 0, 1041}, + dictWord{ + 140, + 11, + 328, + }, + dictWord{133, 11, 354}, + dictWord{134, 0, 1040}, + dictWord{133, 0, 693}, + dictWord{134, 0, 774}, + dictWord{139, 0, 234}, + dictWord{132, 0, 336}, + dictWord{7, 0, 1399}, + dictWord{139, 10, 392}, + dictWord{20, 0, 22}, + dictWord{148, 11, 22}, + dictWord{5, 0, 802}, + dictWord{7, 0, 2021}, + dictWord{136, 0, 805}, + dictWord{ + 5, + 0, + 167, + }, + dictWord{5, 0, 899}, + dictWord{6, 0, 410}, + dictWord{137, 0, 777}, + dictWord{137, 0, 789}, + dictWord{134, 0, 1705}, + dictWord{7, 10, 655}, + dictWord{ + 135, + 10, + 1844, + }, + dictWord{4, 10, 145}, + dictWord{6, 10, 176}, + dictWord{7, 10, 395}, + dictWord{137, 10, 562}, + dictWord{132, 10, 501}, + dictWord{135, 0, 10}, + dictWord{5, 0, 11}, + dictWord{6, 0, 117}, + dictWord{6, 0, 485}, + dictWord{7, 0, 1133}, + dictWord{9, 0, 582}, + dictWord{9, 0, 594}, + dictWord{10, 0, 82}, + dictWord{11, 0, 21}, + dictWord{11, 0, 818}, + dictWord{12, 0, 535}, + dictWord{13, 0, 86}, + dictWord{20, 0, 91}, + dictWord{23, 0, 13}, + dictWord{134, 10, 509}, + dictWord{4, 0, 264}, + dictWord{ + 7, + 0, + 1067, + }, + dictWord{8, 0, 204}, + dictWord{8, 0, 385}, + dictWord{139, 0, 953}, + dictWord{139, 11, 737}, + dictWord{138, 0, 56}, + dictWord{134, 0, 1917}, + dictWord{ + 133, + 0, + 470, + }, + dictWord{10, 11, 657}, + dictWord{14, 11, 297}, + dictWord{142, 11, 361}, + dictWord{135, 11, 412}, + dictWord{7, 0, 1198}, + dictWord{7, 11, 1198}, + dictWord{8, 11, 556}, + dictWord{14, 11, 123}, + dictWord{14, 11, 192}, + dictWord{143, 11, 27}, + dictWord{7, 11, 1985}, + dictWord{14, 11, 146}, + dictWord{15, 11, 42}, + dictWord{16, 11, 23}, + dictWord{17, 11, 86}, + dictWord{146, 11, 17}, + dictWord{11, 0, 1015}, + dictWord{136, 11, 122}, + dictWord{4, 10, 114}, + dictWord{ + 9, + 10, + 492, + }, + dictWord{13, 10, 462}, + dictWord{142, 10, 215}, + dictWord{4, 10, 77}, + dictWord{5, 10, 361}, + dictWord{6, 10, 139}, + dictWord{6, 10, 401}, + dictWord{ + 6, + 10, + 404, + }, + dictWord{7, 10, 413}, + dictWord{7, 10, 715}, + dictWord{7, 10, 1716}, + dictWord{11, 10, 279}, + dictWord{12, 10, 179}, + dictWord{12, 10, 258}, + dictWord{ + 13, + 10, + 244, + }, + dictWord{142, 10, 358}, + dictWord{134, 10, 1717}, + dictWord{7, 10, 1061}, + dictWord{8, 10, 82}, + dictWord{11, 10, 250}, + dictWord{12, 10, 420}, + dictWord{141, 10, 184}, + dictWord{133, 0, 715}, + dictWord{135, 10, 724}, + dictWord{9, 0, 919}, + dictWord{9, 0, 922}, + dictWord{9, 0, 927}, + dictWord{9, 0, 933}, + dictWord{9, 0, 962}, + dictWord{9, 0, 1000}, + dictWord{9, 0, 1002}, + dictWord{9, 0, 1021}, + dictWord{12, 0, 890}, + dictWord{12, 0, 907}, + dictWord{12, 0, 930}, + dictWord{ + 15, + 0, + 207, + }, + dictWord{15, 0, 228}, + dictWord{15, 0, 238}, + dictWord{149, 0, 61}, + dictWord{8, 0, 794}, + dictWord{9, 0, 400}, + dictWord{10, 0, 298}, + dictWord{142, 0, 228}, + dictWord{5, 11, 430}, + dictWord{5, 11, 932}, + dictWord{6, 11, 131}, + dictWord{7, 11, 417}, + dictWord{9, 11, 522}, + dictWord{11, 11, 314}, + dictWord{141, 11, 390}, + dictWord{132, 0, 867}, + dictWord{8, 0, 724}, + dictWord{132, 11, 507}, + dictWord{137, 11, 261}, + dictWord{4, 11, 343}, + dictWord{133, 11, 511}, + dictWord{ + 6, + 0, + 190, + }, + dictWord{7, 0, 768}, + dictWord{135, 0, 1170}, + dictWord{6, 10, 513}, + dictWord{135, 10, 1052}, + dictWord{7, 11, 455}, + dictWord{138, 11, 591}, + dictWord{134, 0, 1066}, + dictWord{137, 10, 899}, + dictWord{14, 0, 67}, + dictWord{147, 0, 60}, + dictWord{4, 0, 948}, + dictWord{18, 0, 174}, + dictWord{146, 0, 176}, + dictWord{135, 0, 1023}, + dictWord{7, 10, 1417}, + dictWord{12, 10, 382}, + dictWord{17, 10, 48}, + dictWord{152, 10, 12}, + dictWord{134, 11, 575}, + dictWord{ + 132, + 0, + 764, + }, + dictWord{6, 10, 545}, + dictWord{7, 10, 565}, + dictWord{7, 10, 1669}, + dictWord{10, 10, 114}, + dictWord{11, 10, 642}, + dictWord{140, 10, 618}, + dictWord{ + 6, + 0, + 137, + }, + dictWord{9, 0, 75}, + dictWord{9, 0, 253}, + dictWord{10, 0, 194}, + dictWord{138, 0, 444}, + dictWord{4, 0, 756}, + dictWord{133, 10, 5}, + dictWord{8, 0, 1008}, + dictWord{135, 10, 192}, + dictWord{132, 0, 842}, + dictWord{11, 0, 643}, + dictWord{12, 0, 115}, + dictWord{136, 10, 763}, + dictWord{139, 0, 67}, + dictWord{ + 133, + 10, + 759, + }, + dictWord{4, 0, 821}, + dictWord{5, 0, 760}, + dictWord{7, 0, 542}, + dictWord{8, 0, 135}, + dictWord{8, 0, 496}, + dictWord{135, 11, 580}, + dictWord{7, 10, 370}, + dictWord{7, 10, 1007}, + dictWord{7, 10, 1177}, + dictWord{135, 10, 1565}, + dictWord{135, 10, 1237}, + dictWord{140, 0, 736}, + dictWord{7, 0, 319}, + dictWord{ + 7, + 0, + 355, + }, + dictWord{7, 0, 763}, + dictWord{10, 0, 389}, + dictWord{145, 0, 43}, + dictWord{8, 11, 333}, + dictWord{138, 11, 182}, + dictWord{4, 10, 87}, + dictWord{5, 10, 250}, + dictWord{141, 10, 298}, + dictWord{138, 0, 786}, + dictWord{134, 0, 2044}, + dictWord{8, 11, 330}, + dictWord{140, 11, 477}, + dictWord{135, 11, 1338}, + dictWord{132, 11, 125}, + dictWord{134, 0, 1030}, + dictWord{134, 0, 1083}, + dictWord{132, 11, 721}, + dictWord{135, 10, 814}, + dictWord{7, 11, 776}, + dictWord{ + 8, + 11, + 145, + }, + dictWord{147, 11, 56}, + dictWord{134, 0, 1226}, + dictWord{4, 10, 57}, + dictWord{7, 10, 1195}, + dictWord{7, 10, 1438}, + dictWord{7, 10, 1548}, + dictWord{ + 7, + 10, + 1835, + }, + dictWord{7, 10, 1904}, + dictWord{9, 10, 757}, + dictWord{10, 10, 604}, + dictWord{139, 10, 519}, + dictWord{7, 11, 792}, + dictWord{8, 11, 147}, + dictWord{10, 11, 821}, + dictWord{139, 11, 1021}, + dictWord{137, 11, 797}, + dictWord{4, 0, 58}, + dictWord{5, 0, 286}, + dictWord{6, 0, 319}, + dictWord{7, 0, 402}, + dictWord{ + 7, + 0, + 1254, + }, + dictWord{7, 0, 1903}, + dictWord{8, 0, 356}, + dictWord{140, 0, 408}, + dictWord{4, 0, 389}, + dictWord{4, 0, 815}, + dictWord{9, 0, 181}, + dictWord{9, 0, 255}, + dictWord{10, 0, 8}, + dictWord{10, 0, 29}, + dictWord{10, 0, 816}, + dictWord{11, 0, 311}, + dictWord{11, 0, 561}, + dictWord{12, 0, 67}, + dictWord{141, 0, 181}, + dictWord{ + 7, + 11, + 1472, + }, + dictWord{135, 11, 1554}, + dictWord{7, 11, 1071}, + dictWord{7, 11, 1541}, + dictWord{7, 11, 1767}, + dictWord{7, 11, 1806}, + dictWord{7, 11, 1999}, + dictWord{9, 11, 248}, + dictWord{10, 11, 400}, + dictWord{11, 11, 162}, + dictWord{11, 11, 178}, + dictWord{11, 11, 242}, + dictWord{12, 11, 605}, + dictWord{ + 15, + 11, + 26, + }, + dictWord{144, 11, 44}, + dictWord{5, 11, 168}, + dictWord{5, 11, 930}, + dictWord{8, 11, 74}, + dictWord{9, 11, 623}, + dictWord{12, 11, 500}, + dictWord{ + 12, + 11, + 579, + }, + dictWord{13, 11, 41}, + dictWord{143, 11, 93}, + dictWord{6, 11, 220}, + dictWord{7, 11, 1101}, + dictWord{141, 11, 105}, + dictWord{5, 0, 474}, + dictWord{ + 7, + 0, + 507, + }, + dictWord{4, 10, 209}, + dictWord{7, 11, 507}, + dictWord{135, 10, 902}, + dictWord{132, 0, 427}, + dictWord{6, 0, 413}, + dictWord{7, 10, 335}, + dictWord{ + 7, + 10, + 1437, + }, + dictWord{7, 10, 1668}, + dictWord{8, 10, 553}, + dictWord{8, 10, 652}, + dictWord{8, 10, 656}, + dictWord{9, 10, 558}, + dictWord{11, 10, 743}, + dictWord{ + 149, + 10, + 18, + }, + dictWord{132, 0, 730}, + dictWord{6, 11, 19}, + dictWord{7, 11, 1413}, + dictWord{139, 11, 428}, + dictWord{133, 0, 373}, + dictWord{132, 10, 559}, + dictWord{7, 11, 96}, + dictWord{8, 11, 401}, + dictWord{137, 11, 896}, + dictWord{7, 0, 799}, + dictWord{7, 0, 1972}, + dictWord{5, 10, 1017}, + dictWord{138, 10, 511}, + dictWord{135, 0, 1793}, + dictWord{7, 11, 1961}, + dictWord{7, 11, 1965}, + dictWord{8, 11, 702}, + dictWord{136, 11, 750}, + dictWord{8, 11, 150}, + dictWord{8, 11, 737}, + dictWord{140, 11, 366}, + dictWord{132, 0, 322}, + dictWord{133, 10, 709}, + dictWord{8, 11, 800}, + dictWord{9, 11, 148}, + dictWord{9, 11, 872}, + dictWord{ + 9, + 11, + 890, + }, + dictWord{11, 11, 309}, + dictWord{11, 11, 1001}, + dictWord{13, 11, 267}, + dictWord{141, 11, 323}, + dictWord{134, 10, 1745}, + dictWord{7, 0, 290}, + dictWord{136, 10, 206}, + dictWord{7, 0, 1651}, + dictWord{145, 0, 89}, + dictWord{139, 0, 2}, + dictWord{132, 0, 672}, + dictWord{6, 0, 1860}, + dictWord{8, 0, 905}, + dictWord{ + 10, + 0, + 844, + }, + dictWord{10, 0, 846}, + dictWord{10, 0, 858}, + dictWord{12, 0, 699}, + dictWord{12, 0, 746}, + dictWord{140, 0, 772}, + dictWord{135, 11, 424}, + dictWord{133, 11, 547}, + dictWord{133, 0, 737}, + dictWord{5, 11, 490}, + dictWord{6, 11, 615}, + dictWord{6, 11, 620}, + dictWord{135, 11, 683}, + dictWord{6, 0, 746}, + dictWord{134, 0, 1612}, + dictWord{132, 10, 776}, + dictWord{9, 11, 385}, + dictWord{149, 11, 17}, + dictWord{133, 0, 145}, + dictWord{135, 10, 1272}, + dictWord{ + 7, + 0, + 884, + }, + dictWord{140, 0, 124}, + dictWord{4, 0, 387}, + dictWord{135, 0, 1288}, + dictWord{5, 11, 133}, + dictWord{136, 10, 406}, + dictWord{136, 11, 187}, + dictWord{ + 6, + 0, + 679, + }, + dictWord{8, 11, 8}, + dictWord{138, 11, 0}, + dictWord{135, 0, 550}, + dictWord{135, 11, 798}, + dictWord{136, 11, 685}, + dictWord{7, 11, 1086}, + dictWord{145, 11, 46}, + dictWord{8, 10, 175}, + dictWord{10, 10, 168}, + dictWord{138, 10, 573}, + dictWord{135, 0, 1305}, + dictWord{4, 0, 576}, + dictWord{ + 135, + 0, + 1263, + }, + dictWord{6, 0, 686}, + dictWord{134, 0, 1563}, + dictWord{134, 0, 607}, + dictWord{5, 0, 919}, + dictWord{134, 0, 1673}, + dictWord{148, 0, 37}, + dictWord{ + 8, + 11, + 774, + }, + dictWord{10, 11, 670}, + dictWord{140, 11, 51}, + dictWord{133, 10, 784}, + dictWord{139, 10, 882}, + dictWord{4, 0, 82}, + dictWord{5, 0, 333}, + dictWord{ + 5, + 0, + 904, + }, + dictWord{6, 0, 207}, + dictWord{7, 0, 325}, + dictWord{7, 0, 1726}, + dictWord{8, 0, 101}, + dictWord{10, 0, 778}, + dictWord{139, 0, 220}, + dictWord{135, 11, 371}, + dictWord{132, 0, 958}, + dictWord{133, 0, 903}, + dictWord{4, 11, 127}, + dictWord{5, 11, 350}, + dictWord{6, 11, 356}, + dictWord{8, 11, 426}, + dictWord{9, 11, 572}, + dictWord{10, 11, 247}, + dictWord{139, 11, 312}, + dictWord{140, 0, 147}, + dictWord{6, 11, 59}, + dictWord{7, 11, 885}, + dictWord{9, 11, 603}, + dictWord{ + 141, + 11, + 397, + }, + dictWord{10, 0, 367}, + dictWord{9, 10, 14}, + dictWord{9, 10, 441}, + dictWord{139, 10, 9}, + dictWord{11, 10, 966}, + dictWord{12, 10, 287}, + dictWord{ + 13, + 10, + 342, + }, + dictWord{13, 10, 402}, + dictWord{15, 10, 110}, + dictWord{143, 10, 163}, + dictWord{134, 0, 690}, + dictWord{132, 0, 705}, + dictWord{9, 0, 651}, + dictWord{ + 11, + 0, + 971, + }, + dictWord{13, 0, 273}, + dictWord{7, 10, 1428}, + dictWord{7, 10, 1640}, + dictWord{7, 10, 1867}, + dictWord{9, 10, 169}, + dictWord{9, 10, 182}, + dictWord{ + 9, + 10, + 367, + }, + dictWord{9, 10, 478}, + dictWord{9, 10, 506}, + dictWord{9, 10, 551}, + dictWord{9, 10, 557}, + dictWord{9, 10, 648}, + dictWord{9, 10, 697}, + dictWord{ + 9, + 10, + 705, + }, + dictWord{9, 10, 725}, + dictWord{9, 10, 787}, + dictWord{9, 10, 794}, + dictWord{10, 10, 198}, + dictWord{10, 10, 214}, + dictWord{10, 10, 267}, + dictWord{ + 10, + 10, + 275, + }, + dictWord{10, 10, 456}, + dictWord{10, 10, 551}, + dictWord{10, 10, 561}, + dictWord{10, 10, 613}, + dictWord{10, 10, 627}, + dictWord{10, 10, 668}, + dictWord{10, 10, 675}, + dictWord{10, 10, 691}, + dictWord{10, 10, 695}, + dictWord{10, 10, 707}, + dictWord{10, 10, 715}, + dictWord{11, 10, 183}, + dictWord{ + 11, + 10, + 201, + }, + dictWord{11, 10, 262}, + dictWord{11, 10, 352}, + dictWord{11, 10, 439}, + dictWord{11, 10, 493}, + dictWord{11, 10, 572}, + dictWord{11, 10, 591}, + dictWord{ + 11, + 10, + 608, + }, + dictWord{11, 10, 611}, + dictWord{11, 10, 646}, + dictWord{11, 10, 674}, + dictWord{11, 10, 711}, + dictWord{11, 10, 751}, + dictWord{11, 10, 761}, + dictWord{11, 10, 776}, + dictWord{11, 10, 785}, + dictWord{11, 10, 850}, + dictWord{11, 10, 853}, + dictWord{11, 10, 862}, + dictWord{11, 10, 865}, + dictWord{ + 11, + 10, + 868, + }, + dictWord{11, 10, 875}, + dictWord{11, 10, 898}, + dictWord{11, 10, 902}, + dictWord{11, 10, 903}, + dictWord{11, 10, 910}, + dictWord{11, 10, 932}, + dictWord{ + 11, + 10, + 942, + }, + dictWord{11, 10, 957}, + dictWord{11, 10, 967}, + dictWord{11, 10, 972}, + dictWord{12, 10, 148}, + dictWord{12, 10, 195}, + dictWord{12, 10, 220}, + dictWord{12, 10, 237}, + dictWord{12, 10, 318}, + dictWord{12, 10, 339}, + dictWord{12, 10, 393}, + dictWord{12, 10, 445}, + dictWord{12, 10, 450}, + dictWord{ + 12, + 10, + 474, + }, + dictWord{12, 10, 505}, + dictWord{12, 10, 509}, + dictWord{12, 10, 533}, + dictWord{12, 10, 591}, + dictWord{12, 10, 594}, + dictWord{12, 10, 597}, + dictWord{ + 12, + 10, + 621, + }, + dictWord{12, 10, 633}, + dictWord{12, 10, 642}, + dictWord{13, 10, 59}, + dictWord{13, 10, 60}, + dictWord{13, 10, 145}, + dictWord{13, 10, 239}, + dictWord{13, 10, 250}, + dictWord{13, 10, 329}, + dictWord{13, 10, 344}, + dictWord{13, 10, 365}, + dictWord{13, 10, 372}, + dictWord{13, 10, 387}, + dictWord{ + 13, + 10, + 403, + }, + dictWord{13, 10, 414}, + dictWord{13, 10, 456}, + dictWord{13, 10, 470}, + dictWord{13, 10, 478}, + dictWord{13, 10, 483}, + dictWord{13, 10, 489}, + dictWord{ + 14, + 10, + 55, + }, + dictWord{14, 10, 57}, + dictWord{14, 10, 81}, + dictWord{14, 10, 90}, + dictWord{14, 10, 148}, + dictWord{14, 10, 239}, + dictWord{14, 10, 266}, + dictWord{ + 14, + 10, + 321, + }, + dictWord{14, 10, 326}, + dictWord{14, 10, 327}, + dictWord{14, 10, 330}, + dictWord{14, 10, 347}, + dictWord{14, 10, 355}, + dictWord{14, 10, 401}, + dictWord{14, 10, 404}, + dictWord{14, 10, 411}, + dictWord{14, 10, 414}, + dictWord{14, 10, 416}, + dictWord{14, 10, 420}, + dictWord{15, 10, 61}, + dictWord{ + 15, + 10, + 74, + }, + dictWord{15, 10, 87}, + dictWord{15, 10, 88}, + dictWord{15, 10, 94}, + dictWord{15, 10, 96}, + dictWord{15, 10, 116}, + dictWord{15, 10, 149}, + dictWord{ + 15, + 10, + 154, + }, + dictWord{16, 10, 50}, + dictWord{16, 10, 63}, + dictWord{16, 10, 73}, + dictWord{17, 10, 2}, + dictWord{17, 10, 66}, + dictWord{17, 10, 92}, + dictWord{17, 10, 103}, + dictWord{17, 10, 112}, + dictWord{17, 10, 120}, + dictWord{18, 10, 50}, + dictWord{18, 10, 54}, + dictWord{18, 10, 82}, + dictWord{18, 10, 86}, + dictWord{18, 10, 90}, + dictWord{18, 10, 111}, + dictWord{18, 10, 115}, + dictWord{18, 10, 156}, + dictWord{19, 10, 40}, + dictWord{19, 10, 79}, + dictWord{20, 10, 78}, + dictWord{149, 10, 22}, + dictWord{7, 0, 887}, + dictWord{5, 10, 161}, + dictWord{135, 10, 839}, + dictWord{142, 11, 98}, + dictWord{134, 0, 90}, + dictWord{138, 11, 356}, + dictWord{ + 135, + 11, + 441, + }, + dictWord{6, 11, 111}, + dictWord{7, 11, 4}, + dictWord{8, 11, 163}, + dictWord{8, 11, 776}, + dictWord{138, 11, 566}, + dictWord{134, 0, 908}, + dictWord{ + 134, + 0, + 1261, + }, + dictWord{7, 0, 813}, + dictWord{12, 0, 497}, + dictWord{141, 0, 56}, + dictWord{134, 0, 1235}, + dictWord{135, 0, 429}, + dictWord{135, 11, 1994}, + dictWord{138, 0, 904}, + dictWord{6, 0, 125}, + dictWord{7, 0, 1277}, + dictWord{137, 0, 772}, + dictWord{151, 0, 12}, + dictWord{4, 0, 841}, + dictWord{5, 0, 386}, + dictWord{ + 133, + 11, + 386, + }, + dictWord{5, 11, 297}, + dictWord{135, 11, 1038}, + dictWord{6, 0, 860}, + dictWord{6, 0, 1069}, + dictWord{135, 11, 309}, + dictWord{136, 0, 946}, + dictWord{135, 10, 1814}, + dictWord{141, 11, 418}, + dictWord{136, 11, 363}, + dictWord{10, 0, 768}, + dictWord{139, 0, 787}, + dictWord{22, 11, 30}, + dictWord{ + 150, + 11, + 33, + }, + dictWord{6, 0, 160}, + dictWord{7, 0, 1106}, + dictWord{9, 0, 770}, + dictWord{11, 0, 112}, + dictWord{140, 0, 413}, + dictWord{11, 11, 216}, + dictWord{ + 139, + 11, + 340, + }, + dictWord{136, 10, 139}, + dictWord{135, 11, 1390}, + dictWord{135, 11, 808}, + dictWord{132, 11, 280}, + dictWord{12, 0, 271}, + dictWord{17, 0, 109}, + dictWord{7, 10, 643}, + dictWord{136, 10, 236}, + dictWord{140, 11, 54}, + dictWord{4, 11, 421}, + dictWord{133, 11, 548}, + dictWord{11, 0, 719}, + dictWord{12, 0, 36}, + dictWord{141, 0, 337}, + dictWord{7, 0, 581}, + dictWord{9, 0, 644}, + dictWord{137, 0, 699}, + dictWord{11, 11, 511}, + dictWord{13, 11, 394}, + dictWord{14, 11, 298}, + dictWord{14, 11, 318}, + dictWord{146, 11, 103}, + dictWord{7, 0, 304}, + dictWord{9, 0, 646}, + dictWord{9, 0, 862}, + dictWord{11, 0, 696}, + dictWord{12, 0, 208}, + dictWord{15, 0, 79}, + dictWord{147, 0, 108}, + dictWord{4, 0, 631}, + dictWord{7, 0, 1126}, + dictWord{135, 0, 1536}, + dictWord{135, 11, 1527}, + dictWord{8, 0, 880}, + dictWord{10, 0, 869}, + dictWord{138, 0, 913}, + dictWord{7, 0, 1513}, + dictWord{5, 10, 54}, + dictWord{6, 11, 254}, + dictWord{9, 11, 109}, + dictWord{138, 11, 103}, + dictWord{135, 0, 981}, + dictWord{133, 11, 729}, + dictWord{132, 10, 744}, + dictWord{132, 0, 434}, + dictWord{134, 0, 550}, + dictWord{7, 0, 930}, + dictWord{10, 0, 476}, + dictWord{13, 0, 452}, + dictWord{19, 0, 104}, + dictWord{6, 11, 1630}, + dictWord{10, 10, 402}, + dictWord{146, 10, 55}, + dictWord{5, 0, 553}, + dictWord{138, 0, 824}, + dictWord{136, 0, 452}, + dictWord{8, 0, 151}, + dictWord{137, 10, 624}, + dictWord{132, 10, 572}, + dictWord{132, 0, 772}, + dictWord{133, 11, 671}, + dictWord{ + 133, + 0, + 292, + }, + dictWord{138, 0, 135}, + dictWord{132, 11, 889}, + dictWord{140, 11, 207}, + dictWord{9, 0, 504}, + dictWord{6, 10, 43}, + dictWord{7, 10, 38}, + dictWord{ + 8, + 10, + 248, + }, + dictWord{138, 10, 513}, + dictWord{6, 0, 1089}, + dictWord{135, 11, 1910}, + dictWord{4, 11, 627}, + dictWord{133, 11, 775}, + dictWord{135, 0, 783}, + dictWord{133, 10, 766}, + dictWord{133, 10, 363}, + dictWord{7, 0, 387}, + dictWord{135, 11, 387}, + dictWord{7, 0, 393}, + dictWord{10, 0, 603}, + dictWord{11, 0, 206}, + dictWord{7, 11, 202}, + dictWord{11, 11, 362}, + dictWord{11, 11, 948}, + dictWord{140, 11, 388}, + dictWord{6, 11, 507}, + dictWord{7, 11, 451}, + dictWord{8, 11, 389}, + dictWord{12, 11, 490}, + dictWord{13, 11, 16}, + dictWord{13, 11, 215}, + dictWord{13, 11, 351}, + dictWord{18, 11, 132}, + dictWord{147, 11, 125}, + dictWord{ + 4, + 0, + 912, + }, + dictWord{9, 0, 232}, + dictWord{135, 11, 841}, + dictWord{6, 10, 258}, + dictWord{140, 10, 409}, + dictWord{5, 10, 249}, + dictWord{148, 10, 82}, + dictWord{ + 136, + 11, + 566, + }, + dictWord{6, 0, 977}, + dictWord{135, 11, 1214}, + dictWord{7, 0, 1973}, + dictWord{136, 0, 716}, + dictWord{135, 0, 98}, + dictWord{133, 0, 733}, + dictWord{ + 5, + 11, + 912, + }, + dictWord{134, 11, 1695}, + dictWord{5, 10, 393}, + dictWord{6, 10, 378}, + dictWord{7, 10, 1981}, + dictWord{9, 10, 32}, + dictWord{9, 10, 591}, + dictWord{10, 10, 685}, + dictWord{10, 10, 741}, + dictWord{142, 10, 382}, + dictWord{133, 10, 788}, + dictWord{10, 0, 19}, + dictWord{11, 0, 911}, + dictWord{7, 10, 1968}, + dictWord{141, 10, 509}, + dictWord{5, 0, 668}, + dictWord{5, 11, 236}, + dictWord{6, 11, 572}, + dictWord{8, 11, 492}, + dictWord{11, 11, 618}, + dictWord{144, 11, 56}, + dictWord{135, 11, 1789}, + dictWord{4, 0, 360}, + dictWord{5, 0, 635}, + dictWord{5, 0, 700}, + dictWord{5, 10, 58}, + dictWord{5, 10, 171}, + dictWord{5, 10, 683}, + dictWord{ + 6, + 10, + 291, + }, + dictWord{6, 10, 566}, + dictWord{7, 10, 1650}, + dictWord{11, 10, 523}, + dictWord{12, 10, 273}, + dictWord{12, 10, 303}, + dictWord{15, 10, 39}, + dictWord{143, 10, 111}, + dictWord{133, 0, 901}, + dictWord{134, 10, 589}, + dictWord{5, 11, 190}, + dictWord{136, 11, 318}, + dictWord{140, 0, 656}, + dictWord{ + 7, + 0, + 726, + }, + dictWord{152, 0, 9}, + dictWord{4, 10, 917}, + dictWord{133, 10, 1005}, + dictWord{135, 10, 1598}, + dictWord{134, 11, 491}, + dictWord{4, 10, 919}, + dictWord{133, 11, 434}, + dictWord{137, 0, 72}, + dictWord{6, 0, 1269}, + dictWord{6, 0, 1566}, + dictWord{134, 0, 1621}, + dictWord{9, 0, 463}, + dictWord{10, 0, 595}, + dictWord{4, 10, 255}, + dictWord{5, 10, 302}, + dictWord{6, 10, 132}, + dictWord{7, 10, 128}, + dictWord{7, 10, 283}, + dictWord{7, 10, 1299}, + dictWord{10, 10, 52}, + dictWord{ + 10, + 10, + 514, + }, + dictWord{11, 10, 925}, + dictWord{13, 10, 92}, + dictWord{142, 10, 309}, + dictWord{135, 0, 1454}, + dictWord{134, 0, 1287}, + dictWord{11, 0, 600}, + dictWord{13, 0, 245}, + dictWord{137, 10, 173}, + dictWord{136, 0, 989}, + dictWord{7, 0, 164}, + dictWord{7, 0, 1571}, + dictWord{9, 0, 107}, + dictWord{140, 0, 225}, + dictWord{6, 0, 1061}, + dictWord{141, 10, 442}, + dictWord{4, 0, 27}, + dictWord{5, 0, 484}, + dictWord{5, 0, 510}, + dictWord{6, 0, 434}, + dictWord{7, 0, 1000}, + dictWord{ + 7, + 0, + 1098, + }, + dictWord{136, 0, 2}, + dictWord{7, 11, 85}, + dictWord{7, 11, 247}, + dictWord{8, 11, 585}, + dictWord{10, 11, 163}, + dictWord{138, 11, 316}, + dictWord{ + 11, + 11, + 103, + }, + dictWord{142, 11, 0}, + dictWord{134, 0, 1127}, + dictWord{4, 0, 460}, + dictWord{134, 0, 852}, + dictWord{134, 10, 210}, + dictWord{4, 0, 932}, + dictWord{ + 133, + 0, + 891, + }, + dictWord{6, 0, 588}, + dictWord{147, 11, 83}, + dictWord{8, 0, 625}, + dictWord{4, 10, 284}, + dictWord{134, 10, 223}, + dictWord{134, 0, 76}, + dictWord{8, 0, 92}, + dictWord{137, 0, 221}, + dictWord{4, 11, 124}, + dictWord{10, 11, 457}, + dictWord{11, 11, 121}, + dictWord{11, 11, 169}, + dictWord{11, 11, 422}, + dictWord{ + 11, + 11, + 870, + }, + dictWord{12, 11, 214}, + dictWord{13, 11, 389}, + dictWord{14, 11, 187}, + dictWord{143, 11, 77}, + dictWord{9, 11, 618}, + dictWord{138, 11, 482}, + dictWord{ + 4, + 10, + 218, + }, + dictWord{7, 10, 526}, + dictWord{143, 10, 137}, + dictWord{13, 0, 9}, + dictWord{14, 0, 104}, + dictWord{14, 0, 311}, + dictWord{4, 10, 270}, + dictWord{ + 5, + 10, + 192, + }, + dictWord{6, 10, 332}, + dictWord{135, 10, 1322}, + dictWord{140, 10, 661}, + dictWord{135, 11, 1193}, + dictWord{6, 11, 107}, + dictWord{7, 11, 638}, + dictWord{7, 11, 1632}, + dictWord{137, 11, 396}, + dictWord{132, 0, 763}, + dictWord{4, 0, 622}, + dictWord{5, 11, 370}, + dictWord{134, 11, 1756}, + dictWord{ + 133, + 0, + 253, + }, + dictWord{135, 0, 546}, + dictWord{9, 0, 73}, + dictWord{10, 0, 110}, + dictWord{14, 0, 185}, + dictWord{17, 0, 119}, + dictWord{133, 11, 204}, + dictWord{7, 0, 624}, + dictWord{7, 0, 916}, + dictWord{10, 0, 256}, + dictWord{139, 0, 87}, + dictWord{7, 10, 379}, + dictWord{8, 10, 481}, + dictWord{137, 10, 377}, + dictWord{5, 0, 212}, + dictWord{12, 0, 35}, + dictWord{13, 0, 382}, + dictWord{5, 11, 970}, + dictWord{134, 11, 1706}, + dictWord{9, 0, 746}, + dictWord{5, 10, 1003}, + dictWord{134, 10, 149}, + dictWord{10, 0, 150}, + dictWord{11, 0, 849}, + dictWord{13, 0, 330}, + dictWord{8, 10, 262}, + dictWord{9, 10, 627}, + dictWord{11, 10, 214}, + dictWord{11, 10, 404}, + dictWord{11, 10, 457}, + dictWord{11, 10, 780}, + dictWord{11, 10, 913}, + dictWord{13, 10, 401}, + dictWord{142, 10, 200}, + dictWord{134, 0, 1466}, + dictWord{ + 135, + 11, + 3, + }, + dictWord{6, 0, 1299}, + dictWord{4, 11, 35}, + dictWord{5, 11, 121}, + dictWord{5, 11, 483}, + dictWord{5, 11, 685}, + dictWord{6, 11, 489}, + dictWord{7, 11, 1204}, + dictWord{136, 11, 394}, + dictWord{135, 10, 742}, + dictWord{4, 10, 142}, + dictWord{136, 10, 304}, + dictWord{4, 11, 921}, + dictWord{133, 11, 1007}, + dictWord{ + 134, + 0, + 1518, + }, + dictWord{6, 0, 1229}, + dictWord{135, 0, 1175}, + dictWord{133, 0, 816}, + dictWord{12, 0, 159}, + dictWord{4, 10, 471}, + dictWord{4, 11, 712}, + dictWord{ + 5, + 10, + 51, + }, + dictWord{6, 10, 602}, + dictWord{7, 10, 925}, + dictWord{8, 10, 484}, + dictWord{138, 10, 195}, + dictWord{134, 11, 1629}, + dictWord{5, 0, 869}, + dictWord{ + 5, + 0, + 968, + }, + dictWord{6, 0, 1626}, + dictWord{8, 0, 734}, + dictWord{136, 0, 784}, + dictWord{4, 0, 542}, + dictWord{6, 0, 1716}, + dictWord{6, 0, 1727}, + dictWord{ + 7, + 0, + 1082, + }, + dictWord{7, 0, 1545}, + dictWord{8, 0, 56}, + dictWord{8, 0, 118}, + dictWord{8, 0, 412}, + dictWord{8, 0, 564}, + dictWord{9, 0, 888}, + dictWord{9, 0, 908}, + dictWord{ + 10, + 0, + 50, + }, + dictWord{10, 0, 423}, + dictWord{11, 0, 685}, + dictWord{11, 0, 697}, + dictWord{11, 0, 933}, + dictWord{12, 0, 299}, + dictWord{13, 0, 126}, + dictWord{ + 13, + 0, + 136, + }, + dictWord{13, 0, 170}, + dictWord{13, 0, 190}, + dictWord{136, 10, 688}, + dictWord{132, 10, 697}, + dictWord{4, 0, 232}, + dictWord{9, 0, 202}, + dictWord{ + 10, + 0, + 474, + }, + dictWord{140, 0, 433}, + dictWord{136, 0, 212}, + dictWord{6, 0, 108}, + dictWord{7, 0, 1003}, + dictWord{7, 0, 1181}, + dictWord{8, 0, 111}, + dictWord{ + 136, + 0, + 343, + }, + dictWord{5, 10, 221}, + dictWord{135, 11, 1255}, + dictWord{133, 11, 485}, + dictWord{134, 0, 1712}, + dictWord{142, 0, 216}, + dictWord{5, 0, 643}, + dictWord{ + 6, + 0, + 516, + }, + dictWord{4, 11, 285}, + dictWord{5, 11, 317}, + dictWord{6, 11, 301}, + dictWord{7, 11, 7}, + dictWord{8, 11, 153}, + dictWord{10, 11, 766}, + dictWord{ + 11, + 11, + 468, + }, + dictWord{12, 11, 467}, + dictWord{141, 11, 143}, + dictWord{4, 0, 133}, + dictWord{7, 0, 711}, + dictWord{7, 0, 1298}, + dictWord{135, 0, 1585}, + dictWord{ + 134, + 0, + 650, + }, + dictWord{135, 11, 512}, + dictWord{6, 0, 99}, + dictWord{7, 0, 1808}, + dictWord{145, 0, 57}, + dictWord{6, 0, 246}, + dictWord{6, 0, 574}, + dictWord{7, 0, 428}, + dictWord{9, 0, 793}, + dictWord{10, 0, 669}, + dictWord{11, 0, 485}, + dictWord{11, 0, 840}, + dictWord{12, 0, 300}, + dictWord{14, 0, 250}, + dictWord{145, 0, 55}, + dictWord{ + 4, + 10, + 132, + }, + dictWord{5, 10, 69}, + dictWord{135, 10, 1242}, + dictWord{136, 0, 1023}, + dictWord{7, 0, 302}, + dictWord{132, 10, 111}, + dictWord{135, 0, 1871}, + dictWord{132, 0, 728}, + dictWord{9, 0, 252}, + dictWord{132, 10, 767}, + dictWord{6, 0, 461}, + dictWord{7, 0, 1590}, + dictWord{7, 10, 1416}, + dictWord{7, 10, 2005}, + dictWord{8, 10, 131}, + dictWord{8, 10, 466}, + dictWord{9, 10, 672}, + dictWord{13, 10, 252}, + dictWord{148, 10, 103}, + dictWord{6, 0, 323}, + dictWord{135, 0, 1564}, + dictWord{7, 0, 461}, + dictWord{136, 0, 775}, + dictWord{6, 10, 44}, + dictWord{136, 10, 368}, + dictWord{139, 0, 172}, + dictWord{132, 0, 464}, + dictWord{4, 10, 570}, + dictWord{133, 10, 120}, + dictWord{137, 11, 269}, + dictWord{6, 10, 227}, + dictWord{135, 10, 1589}, + dictWord{6, 11, 1719}, + dictWord{6, 11, 1735}, + dictWord{ + 7, + 11, + 2016, + }, + dictWord{7, 11, 2020}, + dictWord{8, 11, 837}, + dictWord{137, 11, 852}, + dictWord{7, 0, 727}, + dictWord{146, 0, 73}, + dictWord{132, 0, 1023}, + dictWord{135, 11, 852}, + dictWord{135, 10, 1529}, + dictWord{136, 0, 577}, + dictWord{138, 11, 568}, + dictWord{134, 0, 1037}, + dictWord{8, 11, 67}, + dictWord{ + 138, + 11, + 419, + }, + dictWord{4, 0, 413}, + dictWord{5, 0, 677}, + dictWord{8, 0, 432}, + dictWord{140, 0, 280}, + dictWord{10, 0, 600}, + dictWord{6, 10, 1667}, + dictWord{ + 7, + 11, + 967, + }, + dictWord{7, 10, 2036}, + dictWord{141, 11, 11}, + dictWord{6, 10, 511}, + dictWord{140, 10, 132}, + dictWord{6, 0, 799}, + dictWord{5, 10, 568}, + dictWord{ + 6, + 10, + 138, + }, + dictWord{135, 10, 1293}, + dictWord{8, 0, 159}, + dictWord{4, 10, 565}, + dictWord{136, 10, 827}, + dictWord{7, 0, 646}, + dictWord{7, 0, 1730}, + dictWord{ + 11, + 0, + 446, + }, + dictWord{141, 0, 178}, + dictWord{4, 10, 922}, + dictWord{133, 10, 1023}, + dictWord{135, 11, 11}, + dictWord{132, 0, 395}, + dictWord{11, 0, 145}, + dictWord{135, 10, 1002}, + dictWord{9, 0, 174}, + dictWord{10, 0, 164}, + dictWord{11, 0, 440}, + dictWord{11, 0, 514}, + dictWord{11, 0, 841}, + dictWord{15, 0, 98}, + dictWord{149, 0, 20}, + dictWord{134, 0, 426}, + dictWord{10, 0, 608}, + dictWord{139, 0, 1002}, + dictWord{7, 11, 320}, + dictWord{8, 11, 51}, + dictWord{12, 11, 481}, + dictWord{12, 11, 570}, + dictWord{148, 11, 106}, + dictWord{9, 0, 977}, + dictWord{9, 0, 983}, + dictWord{132, 11, 445}, + dictWord{138, 0, 250}, + dictWord{139, 0, 100}, + dictWord{6, 0, 1982}, + dictWord{136, 10, 402}, + dictWord{133, 11, 239}, + dictWord{4, 10, 716}, + dictWord{141, 10, 31}, + dictWord{5, 0, 476}, + dictWord{7, 11, 83}, + dictWord{7, 11, 1990}, + dictWord{8, 11, 130}, + dictWord{139, 11, 720}, + dictWord{8, 10, 691}, + dictWord{136, 10, 731}, + dictWord{5, 11, 123}, + dictWord{ + 6, + 11, + 530, + }, + dictWord{7, 11, 348}, + dictWord{135, 11, 1419}, + dictWord{5, 0, 76}, + dictWord{6, 0, 458}, + dictWord{6, 0, 497}, + dictWord{7, 0, 868}, + dictWord{9, 0, 658}, + dictWord{10, 0, 594}, + dictWord{11, 0, 173}, + dictWord{11, 0, 566}, + dictWord{12, 0, 20}, + dictWord{12, 0, 338}, + dictWord{141, 0, 200}, + dictWord{9, 11, 139}, + dictWord{ + 10, + 11, + 399, + }, + dictWord{11, 11, 469}, + dictWord{12, 11, 634}, + dictWord{141, 11, 223}, + dictWord{9, 10, 840}, + dictWord{138, 10, 803}, + dictWord{133, 10, 847}, + dictWord{11, 11, 223}, + dictWord{140, 11, 168}, + dictWord{132, 11, 210}, + dictWord{8, 0, 447}, + dictWord{9, 10, 53}, + dictWord{9, 10, 268}, + dictWord{9, 10, 901}, + dictWord{10, 10, 518}, + dictWord{10, 10, 829}, + dictWord{11, 10, 188}, + dictWord{13, 10, 74}, + dictWord{14, 10, 46}, + dictWord{15, 10, 17}, + dictWord{15, 10, 33}, + dictWord{17, 10, 40}, + dictWord{18, 10, 36}, + dictWord{19, 10, 20}, + dictWord{22, 10, 1}, + dictWord{152, 10, 2}, + dictWord{4, 0, 526}, + dictWord{7, 0, 1029}, + dictWord{135, 0, 1054}, + dictWord{19, 11, 59}, + dictWord{150, 11, 2}, + dictWord{4, 0, 636}, + dictWord{6, 0, 1875}, + dictWord{6, 0, 1920}, + dictWord{9, 0, 999}, + dictWord{ + 12, + 0, + 807, + }, + dictWord{12, 0, 825}, + dictWord{15, 0, 179}, + dictWord{15, 0, 190}, + dictWord{18, 0, 182}, + dictWord{136, 10, 532}, + dictWord{6, 0, 1699}, + dictWord{ + 7, + 0, + 660, + }, + dictWord{7, 0, 1124}, + dictWord{17, 0, 31}, + dictWord{19, 0, 22}, + dictWord{151, 0, 14}, + dictWord{135, 10, 681}, + dictWord{132, 11, 430}, + dictWord{ + 140, + 10, + 677, + }, + dictWord{4, 10, 684}, + dictWord{136, 10, 384}, + dictWord{132, 11, 756}, + dictWord{133, 11, 213}, + dictWord{7, 0, 188}, + dictWord{7, 10, 110}, + dictWord{ + 8, + 10, + 290, + }, + dictWord{8, 10, 591}, + dictWord{9, 10, 382}, + dictWord{9, 10, 649}, + dictWord{11, 10, 71}, + dictWord{11, 10, 155}, + dictWord{11, 10, 313}, + dictWord{ + 12, + 10, + 5, + }, + dictWord{13, 10, 325}, + dictWord{142, 10, 287}, + dictWord{7, 10, 360}, + dictWord{7, 10, 425}, + dictWord{9, 10, 66}, + dictWord{9, 10, 278}, + dictWord{ + 138, + 10, + 644, + }, + dictWord{142, 11, 164}, + dictWord{4, 0, 279}, + dictWord{7, 0, 301}, + dictWord{137, 0, 362}, + dictWord{134, 11, 586}, + dictWord{135, 0, 1743}, + dictWord{4, 0, 178}, + dictWord{133, 0, 399}, + dictWord{4, 10, 900}, + dictWord{133, 10, 861}, + dictWord{5, 10, 254}, + dictWord{7, 10, 985}, + dictWord{136, 10, 73}, + dictWord{133, 11, 108}, + dictWord{7, 10, 1959}, + dictWord{136, 10, 683}, + dictWord{133, 11, 219}, + dictWord{4, 11, 193}, + dictWord{5, 11, 916}, + dictWord{ + 7, + 11, + 364, + }, + dictWord{10, 11, 398}, + dictWord{10, 11, 726}, + dictWord{11, 11, 317}, + dictWord{11, 11, 626}, + dictWord{12, 11, 142}, + dictWord{12, 11, 288}, + dictWord{ + 12, + 11, + 678, + }, + dictWord{13, 11, 313}, + dictWord{15, 11, 113}, + dictWord{18, 11, 114}, + dictWord{21, 11, 30}, + dictWord{150, 11, 53}, + dictWord{6, 11, 241}, + dictWord{7, 11, 907}, + dictWord{8, 11, 832}, + dictWord{9, 11, 342}, + dictWord{10, 11, 729}, + dictWord{11, 11, 284}, + dictWord{11, 11, 445}, + dictWord{11, 11, 651}, + dictWord{11, 11, 863}, + dictWord{13, 11, 398}, + dictWord{146, 11, 99}, + dictWord{132, 0, 872}, + dictWord{134, 0, 831}, + dictWord{134, 0, 1692}, + dictWord{ + 6, + 0, + 202, + }, + dictWord{6, 0, 1006}, + dictWord{9, 0, 832}, + dictWord{10, 0, 636}, + dictWord{11, 0, 208}, + dictWord{12, 0, 360}, + dictWord{17, 0, 118}, + dictWord{18, 0, 27}, + dictWord{20, 0, 67}, + dictWord{137, 11, 734}, + dictWord{132, 10, 725}, + dictWord{7, 11, 993}, + dictWord{138, 11, 666}, + dictWord{134, 0, 1954}, + dictWord{ + 134, + 10, + 196, + }, + dictWord{7, 0, 872}, + dictWord{10, 0, 516}, + dictWord{139, 0, 167}, + dictWord{133, 10, 831}, + dictWord{4, 11, 562}, + dictWord{9, 11, 254}, + dictWord{ + 139, + 11, + 879, + }, + dictWord{137, 0, 313}, + dictWord{4, 0, 224}, + dictWord{132, 11, 786}, + dictWord{11, 0, 24}, + dictWord{12, 0, 170}, + dictWord{136, 10, 723}, + dictWord{ + 5, + 0, + 546, + }, + dictWord{7, 0, 35}, + dictWord{8, 0, 11}, + dictWord{8, 0, 12}, + dictWord{9, 0, 315}, + dictWord{9, 0, 533}, + dictWord{10, 0, 802}, + dictWord{11, 0, 166}, + dictWord{ + 12, + 0, + 525, + }, + dictWord{142, 0, 243}, + dictWord{7, 0, 1937}, + dictWord{13, 10, 80}, + dictWord{13, 10, 437}, + dictWord{145, 10, 74}, + dictWord{5, 0, 241}, + dictWord{ + 8, + 0, + 242, + }, + dictWord{9, 0, 451}, + dictWord{10, 0, 667}, + dictWord{11, 0, 598}, + dictWord{140, 0, 429}, + dictWord{150, 0, 46}, + dictWord{6, 0, 1273}, + dictWord{ + 137, + 0, + 830, + }, + dictWord{5, 10, 848}, + dictWord{6, 10, 66}, + dictWord{136, 10, 764}, + dictWord{6, 0, 825}, + dictWord{134, 0, 993}, + dictWord{4, 0, 1006}, + dictWord{ + 10, + 0, + 327, + }, + dictWord{13, 0, 271}, + dictWord{4, 10, 36}, + dictWord{7, 10, 1387}, + dictWord{139, 10, 755}, + dictWord{134, 0, 1023}, + dictWord{135, 0, 1580}, + dictWord{ + 4, + 0, + 366, + }, + dictWord{137, 0, 516}, + dictWord{132, 10, 887}, + dictWord{6, 0, 1736}, + dictWord{135, 0, 1891}, + dictWord{6, 11, 216}, + dictWord{7, 11, 901}, + dictWord{ + 7, + 11, + 1343, + }, + dictWord{136, 11, 493}, + dictWord{6, 10, 165}, + dictWord{138, 10, 388}, + dictWord{7, 11, 341}, + dictWord{139, 11, 219}, + dictWord{4, 10, 719}, + dictWord{135, 10, 155}, + dictWord{134, 0, 1935}, + dictWord{132, 0, 826}, + dictWord{6, 0, 331}, + dictWord{6, 0, 1605}, + dictWord{8, 0, 623}, + dictWord{11, 0, 139}, + dictWord{139, 0, 171}, + dictWord{135, 11, 1734}, + dictWord{10, 11, 115}, + dictWord{11, 11, 420}, + dictWord{12, 11, 154}, + dictWord{13, 11, 404}, + dictWord{ + 14, + 11, + 346, + }, + dictWord{15, 11, 54}, + dictWord{143, 11, 112}, + dictWord{7, 0, 288}, + dictWord{4, 10, 353}, + dictWord{6, 10, 146}, + dictWord{6, 10, 1789}, + dictWord{ + 7, + 10, + 990, + }, + dictWord{7, 10, 1348}, + dictWord{9, 10, 665}, + dictWord{9, 10, 898}, + dictWord{11, 10, 893}, + dictWord{142, 10, 212}, + dictWord{6, 0, 916}, + dictWord{134, 0, 1592}, + dictWord{7, 0, 1888}, + dictWord{4, 10, 45}, + dictWord{135, 10, 1257}, + dictWord{5, 11, 1011}, + dictWord{136, 11, 701}, + dictWord{ + 139, + 11, + 596, + }, + dictWord{4, 11, 54}, + dictWord{5, 11, 666}, + dictWord{7, 11, 1039}, + dictWord{7, 11, 1130}, + dictWord{9, 11, 195}, + dictWord{138, 11, 302}, + dictWord{ + 134, + 0, + 1471, + }, + dictWord{134, 0, 1570}, + dictWord{132, 0, 394}, + dictWord{140, 10, 65}, + dictWord{136, 10, 816}, + dictWord{135, 0, 1931}, + dictWord{7, 0, 574}, + dictWord{135, 0, 1719}, + dictWord{134, 11, 467}, + dictWord{132, 0, 658}, + dictWord{9, 0, 781}, + dictWord{10, 0, 144}, + dictWord{11, 0, 385}, + dictWord{13, 0, 161}, + dictWord{13, 0, 228}, + dictWord{13, 0, 268}, + dictWord{20, 0, 107}, + dictWord{134, 11, 1669}, + dictWord{136, 0, 374}, + dictWord{135, 0, 735}, + dictWord{4, 0, 344}, + dictWord{6, 0, 498}, + dictWord{139, 0, 323}, + dictWord{7, 0, 586}, + dictWord{7, 0, 1063}, + dictWord{6, 10, 559}, + dictWord{134, 10, 1691}, + dictWord{137, 0, 155}, + dictWord{133, 0, 906}, + dictWord{7, 11, 122}, + dictWord{9, 11, 259}, + dictWord{10, 11, 84}, + dictWord{11, 11, 470}, + dictWord{12, 11, 541}, + dictWord{ + 141, + 11, + 379, + }, + dictWord{134, 0, 1139}, + dictWord{10, 0, 108}, + dictWord{139, 0, 116}, + dictWord{134, 10, 456}, + dictWord{133, 10, 925}, + dictWord{5, 11, 82}, + dictWord{ + 5, + 11, + 131, + }, + dictWord{7, 11, 1755}, + dictWord{8, 11, 31}, + dictWord{9, 11, 168}, + dictWord{9, 11, 764}, + dictWord{139, 11, 869}, + dictWord{134, 11, 605}, + dictWord{ + 5, + 11, + 278, + }, + dictWord{137, 11, 68}, + dictWord{4, 11, 163}, + dictWord{5, 11, 201}, + dictWord{5, 11, 307}, + dictWord{5, 11, 310}, + dictWord{6, 11, 335}, + dictWord{ + 7, + 11, + 284, + }, + dictWord{136, 11, 165}, + dictWord{135, 11, 1660}, + dictWord{6, 11, 33}, + dictWord{135, 11, 1244}, + dictWord{4, 0, 616}, + dictWord{136, 11, 483}, + dictWord{8, 0, 857}, + dictWord{8, 0, 902}, + dictWord{8, 0, 910}, + dictWord{10, 0, 879}, + dictWord{12, 0, 726}, + dictWord{4, 11, 199}, + dictWord{139, 11, 34}, + dictWord{136, 0, 692}, + dictWord{6, 10, 193}, + dictWord{7, 10, 240}, + dictWord{7, 10, 1682}, + dictWord{10, 10, 51}, + dictWord{10, 10, 640}, + dictWord{11, 10, 410}, + dictWord{13, 10, 82}, + dictWord{14, 10, 247}, + dictWord{14, 10, 331}, + dictWord{142, 10, 377}, + dictWord{6, 0, 823}, + dictWord{134, 0, 983}, + dictWord{ + 139, + 10, + 411, + }, + dictWord{132, 0, 305}, + dictWord{136, 10, 633}, + dictWord{138, 11, 203}, + dictWord{134, 0, 681}, + dictWord{6, 11, 326}, + dictWord{7, 11, 677}, + dictWord{137, 11, 425}, + dictWord{5, 0, 214}, + dictWord{7, 0, 603}, + dictWord{8, 0, 611}, + dictWord{9, 0, 686}, + dictWord{10, 0, 88}, + dictWord{11, 0, 459}, + dictWord{ + 11, + 0, + 496, + }, + dictWord{12, 0, 463}, + dictWord{12, 0, 590}, + dictWord{141, 0, 0}, + dictWord{136, 0, 1004}, + dictWord{142, 0, 23}, + dictWord{134, 0, 1703}, + dictWord{ + 147, + 11, + 8, + }, + dictWord{145, 11, 56}, + dictWord{135, 0, 1443}, + dictWord{4, 10, 237}, + dictWord{135, 10, 514}, + dictWord{6, 0, 714}, + dictWord{145, 0, 19}, + dictWord{ + 5, + 11, + 358, + }, + dictWord{7, 11, 473}, + dictWord{7, 11, 1184}, + dictWord{10, 11, 662}, + dictWord{13, 11, 212}, + dictWord{13, 11, 304}, + dictWord{13, 11, 333}, + dictWord{145, 11, 98}, + dictWord{4, 0, 737}, + dictWord{10, 0, 98}, + dictWord{11, 0, 294}, + dictWord{12, 0, 60}, + dictWord{12, 0, 437}, + dictWord{13, 0, 64}, + dictWord{ + 13, + 0, + 380, + }, + dictWord{142, 0, 430}, + dictWord{6, 10, 392}, + dictWord{7, 10, 65}, + dictWord{135, 10, 2019}, + dictWord{6, 0, 1758}, + dictWord{8, 0, 520}, + dictWord{ + 9, + 0, + 345, + }, + dictWord{9, 0, 403}, + dictWord{142, 0, 350}, + dictWord{5, 0, 47}, + dictWord{10, 0, 242}, + dictWord{138, 0, 579}, + dictWord{5, 0, 139}, + dictWord{7, 0, 1168}, + dictWord{138, 0, 539}, + dictWord{134, 0, 1459}, + dictWord{13, 0, 388}, + dictWord{141, 11, 388}, + dictWord{134, 0, 253}, + dictWord{7, 10, 1260}, + dictWord{ + 135, + 10, + 1790, + }, + dictWord{10, 0, 252}, + dictWord{9, 10, 222}, + dictWord{139, 10, 900}, + dictWord{140, 0, 745}, + dictWord{133, 11, 946}, + dictWord{4, 0, 107}, + dictWord{ + 7, + 0, + 613, + }, + dictWord{8, 0, 439}, + dictWord{8, 0, 504}, + dictWord{9, 0, 501}, + dictWord{10, 0, 383}, + dictWord{139, 0, 477}, + dictWord{135, 11, 1485}, + dictWord{ + 132, + 0, + 871, + }, + dictWord{7, 11, 411}, + dictWord{7, 11, 590}, + dictWord{8, 11, 631}, + dictWord{9, 11, 323}, + dictWord{10, 11, 355}, + dictWord{11, 11, 491}, + dictWord{ + 12, + 11, + 143, + }, + dictWord{12, 11, 402}, + dictWord{13, 11, 73}, + dictWord{14, 11, 408}, + dictWord{15, 11, 107}, + dictWord{146, 11, 71}, + dictWord{132, 0, 229}, + dictWord{132, 0, 903}, + dictWord{140, 0, 71}, + dictWord{133, 0, 549}, + dictWord{4, 0, 47}, + dictWord{6, 0, 373}, + dictWord{7, 0, 452}, + dictWord{7, 0, 543}, + dictWord{ + 7, + 0, + 1828, + }, + dictWord{7, 0, 1856}, + dictWord{9, 0, 6}, + dictWord{11, 0, 257}, + dictWord{139, 0, 391}, + dictWord{7, 11, 1467}, + dictWord{8, 11, 328}, + dictWord{ + 10, + 11, + 544, + }, + dictWord{11, 11, 955}, + dictWord{13, 11, 320}, + dictWord{145, 11, 83}, + dictWord{5, 0, 980}, + dictWord{134, 0, 1754}, + dictWord{136, 0, 865}, + dictWord{ + 5, + 0, + 705, + }, + dictWord{137, 0, 606}, + dictWord{7, 0, 161}, + dictWord{8, 10, 201}, + dictWord{136, 10, 605}, + dictWord{143, 11, 35}, + dictWord{5, 11, 835}, + dictWord{ + 6, + 11, + 483, + }, + dictWord{140, 10, 224}, + dictWord{7, 0, 536}, + dictWord{7, 0, 1331}, + dictWord{136, 0, 143}, + dictWord{134, 0, 1388}, + dictWord{5, 0, 724}, + dictWord{ + 10, + 0, + 305, + }, + dictWord{11, 0, 151}, + dictWord{12, 0, 33}, + dictWord{12, 0, 121}, + dictWord{12, 0, 381}, + dictWord{17, 0, 3}, + dictWord{17, 0, 27}, + dictWord{17, 0, 78}, + dictWord{18, 0, 18}, + dictWord{19, 0, 54}, + dictWord{149, 0, 5}, + dictWord{4, 10, 523}, + dictWord{133, 10, 638}, + dictWord{5, 0, 19}, + dictWord{134, 0, 533}, + dictWord{ + 5, + 0, + 395, + }, + dictWord{5, 0, 951}, + dictWord{134, 0, 1776}, + dictWord{135, 0, 1908}, + dictWord{132, 0, 846}, + dictWord{10, 0, 74}, + dictWord{11, 0, 663}, + dictWord{ + 12, + 0, + 210, + }, + dictWord{13, 0, 166}, + dictWord{13, 0, 310}, + dictWord{14, 0, 373}, + dictWord{18, 0, 95}, + dictWord{19, 0, 43}, + dictWord{6, 10, 242}, + dictWord{7, 10, 227}, + dictWord{7, 10, 1581}, + dictWord{8, 10, 104}, + dictWord{9, 10, 113}, + dictWord{9, 10, 220}, + dictWord{9, 10, 427}, + dictWord{10, 10, 239}, + dictWord{11, 10, 579}, + dictWord{11, 10, 1023}, + dictWord{13, 10, 4}, + dictWord{13, 10, 204}, + dictWord{13, 10, 316}, + dictWord{148, 10, 86}, + dictWord{9, 11, 716}, + dictWord{11, 11, 108}, + dictWord{13, 11, 123}, + dictWord{14, 11, 252}, + dictWord{19, 11, 38}, + dictWord{21, 11, 3}, + dictWord{151, 11, 11}, + dictWord{8, 0, 372}, + dictWord{9, 0, 122}, + dictWord{138, 0, 175}, + dictWord{132, 11, 677}, + dictWord{7, 11, 1374}, + dictWord{136, 11, 540}, + dictWord{135, 10, 861}, + dictWord{132, 0, 695}, + dictWord{ + 7, + 0, + 497, + }, + dictWord{9, 0, 387}, + dictWord{147, 0, 81}, + dictWord{136, 0, 937}, + dictWord{134, 0, 718}, + dictWord{7, 0, 1328}, + dictWord{136, 10, 494}, + dictWord{ + 132, + 11, + 331, + }, + dictWord{6, 0, 1581}, + dictWord{133, 11, 747}, + dictWord{5, 0, 284}, + dictWord{6, 0, 49}, + dictWord{6, 0, 350}, + dictWord{7, 0, 1}, + dictWord{7, 0, 377}, + dictWord{7, 0, 1693}, + dictWord{8, 0, 18}, + dictWord{8, 0, 678}, + dictWord{9, 0, 161}, + dictWord{9, 0, 585}, + dictWord{9, 0, 671}, + dictWord{9, 0, 839}, + dictWord{11, 0, 912}, + dictWord{141, 0, 427}, + dictWord{7, 10, 1306}, + dictWord{8, 10, 505}, + dictWord{9, 10, 482}, + dictWord{10, 10, 126}, + dictWord{11, 10, 225}, + dictWord{12, 10, 347}, + dictWord{12, 10, 449}, + dictWord{13, 10, 19}, + dictWord{14, 10, 218}, + dictWord{142, 10, 435}, + dictWord{10, 10, 764}, + dictWord{12, 10, 120}, + dictWord{ + 13, + 10, + 39, + }, + dictWord{145, 10, 127}, + dictWord{4, 0, 597}, + dictWord{133, 10, 268}, + dictWord{134, 0, 1094}, + dictWord{4, 0, 1008}, + dictWord{134, 0, 1973}, + dictWord{132, 0, 811}, + dictWord{139, 0, 908}, + dictWord{135, 0, 1471}, + dictWord{133, 11, 326}, + dictWord{4, 10, 384}, + dictWord{135, 10, 1022}, + dictWord{ + 7, + 0, + 1935, + }, + dictWord{8, 0, 324}, + dictWord{12, 0, 42}, + dictWord{4, 11, 691}, + dictWord{7, 11, 1935}, + dictWord{8, 11, 324}, + dictWord{9, 11, 35}, + dictWord{10, 11, 680}, + dictWord{11, 11, 364}, + dictWord{12, 11, 42}, + dictWord{13, 11, 357}, + dictWord{146, 11, 16}, + dictWord{135, 0, 2014}, + dictWord{7, 0, 2007}, + dictWord{ + 9, + 0, + 101, + }, + dictWord{9, 0, 450}, + dictWord{10, 0, 66}, + dictWord{10, 0, 842}, + dictWord{11, 0, 536}, + dictWord{12, 0, 587}, + dictWord{6, 11, 32}, + dictWord{7, 11, 385}, + dictWord{7, 11, 757}, + dictWord{7, 11, 1916}, + dictWord{8, 11, 37}, + dictWord{8, 11, 94}, + dictWord{8, 11, 711}, + dictWord{9, 11, 541}, + dictWord{10, 11, 162}, + dictWord{ + 10, + 11, + 795, + }, + dictWord{11, 11, 989}, + dictWord{11, 11, 1010}, + dictWord{12, 11, 14}, + dictWord{142, 11, 308}, + dictWord{139, 0, 586}, + dictWord{ + 135, + 10, + 1703, + }, + dictWord{7, 0, 1077}, + dictWord{11, 0, 28}, + dictWord{9, 10, 159}, + dictWord{140, 10, 603}, + dictWord{6, 0, 1221}, + dictWord{136, 10, 583}, + dictWord{ + 6, + 11, + 152, + }, + dictWord{6, 11, 349}, + dictWord{6, 11, 1682}, + dictWord{7, 11, 1252}, + dictWord{8, 11, 112}, + dictWord{9, 11, 435}, + dictWord{9, 11, 668}, + dictWord{ + 10, + 11, + 290, + }, + dictWord{10, 11, 319}, + dictWord{10, 11, 815}, + dictWord{11, 11, 180}, + dictWord{11, 11, 837}, + dictWord{12, 11, 240}, + dictWord{13, 11, 152}, + dictWord{13, 11, 219}, + dictWord{142, 11, 158}, + dictWord{139, 0, 62}, + dictWord{132, 10, 515}, + dictWord{8, 10, 632}, + dictWord{8, 10, 697}, + dictWord{ + 137, + 10, + 854, + }, + dictWord{134, 0, 1766}, + dictWord{132, 11, 581}, + dictWord{6, 11, 126}, + dictWord{7, 11, 573}, + dictWord{8, 11, 397}, + dictWord{142, 11, 44}, + dictWord{ + 150, + 0, + 28, + }, + dictWord{11, 0, 670}, + dictWord{22, 0, 25}, + dictWord{4, 10, 136}, + dictWord{133, 10, 551}, + dictWord{6, 0, 1665}, + dictWord{7, 0, 256}, + dictWord{ + 7, + 0, + 1388, + }, + dictWord{138, 0, 499}, + dictWord{4, 0, 22}, + dictWord{5, 0, 10}, + dictWord{7, 0, 1576}, + dictWord{136, 0, 97}, + dictWord{134, 10, 1782}, + dictWord{5, 0, 481}, + dictWord{7, 10, 1287}, + dictWord{9, 10, 44}, + dictWord{10, 10, 552}, + dictWord{10, 10, 642}, + dictWord{11, 10, 839}, + dictWord{12, 10, 274}, + dictWord{ + 12, + 10, + 275, + }, + dictWord{12, 10, 372}, + dictWord{13, 10, 91}, + dictWord{142, 10, 125}, + dictWord{133, 11, 926}, + dictWord{7, 11, 1232}, + dictWord{137, 11, 531}, + dictWord{6, 0, 134}, + dictWord{7, 0, 437}, + dictWord{7, 0, 1824}, + dictWord{9, 0, 37}, + dictWord{14, 0, 285}, + dictWord{142, 0, 371}, + dictWord{7, 0, 486}, + dictWord{8, 0, 155}, + dictWord{11, 0, 93}, + dictWord{140, 0, 164}, + dictWord{6, 0, 1391}, + dictWord{134, 0, 1442}, + dictWord{133, 11, 670}, + dictWord{133, 0, 591}, + dictWord{ + 6, + 10, + 147, + }, + dictWord{7, 10, 886}, + dictWord{7, 11, 1957}, + dictWord{9, 10, 753}, + dictWord{138, 10, 268}, + dictWord{5, 0, 380}, + dictWord{5, 0, 650}, + dictWord{ + 7, + 0, + 1173, + }, + dictWord{136, 0, 310}, + dictWord{4, 0, 364}, + dictWord{7, 0, 1156}, + dictWord{7, 0, 1187}, + dictWord{137, 0, 409}, + dictWord{135, 11, 1621}, + dictWord{ + 134, + 0, + 482, + }, + dictWord{133, 11, 506}, + dictWord{4, 0, 781}, + dictWord{6, 0, 487}, + dictWord{7, 0, 926}, + dictWord{8, 0, 263}, + dictWord{139, 0, 500}, + dictWord{ + 138, + 10, + 137, + }, + dictWord{135, 11, 242}, + dictWord{139, 11, 96}, + dictWord{133, 10, 414}, + dictWord{135, 10, 1762}, + dictWord{134, 0, 804}, + dictWord{5, 11, 834}, + dictWord{7, 11, 1202}, + dictWord{8, 11, 14}, + dictWord{9, 11, 481}, + dictWord{137, 11, 880}, + dictWord{134, 10, 599}, + dictWord{4, 0, 94}, + dictWord{135, 0, 1265}, + dictWord{4, 0, 415}, + dictWord{132, 0, 417}, + dictWord{5, 0, 348}, + dictWord{6, 0, 522}, + dictWord{6, 10, 1749}, + dictWord{7, 11, 1526}, + dictWord{138, 11, 465}, + dictWord{134, 10, 1627}, + dictWord{132, 0, 1012}, + dictWord{132, 10, 488}, + dictWord{4, 11, 357}, + dictWord{6, 11, 172}, + dictWord{7, 11, 143}, + dictWord{ + 137, + 11, + 413, + }, + dictWord{4, 10, 83}, + dictWord{4, 11, 590}, + dictWord{146, 11, 76}, + dictWord{140, 10, 676}, + dictWord{7, 11, 287}, + dictWord{8, 11, 355}, + dictWord{ + 9, + 11, + 293, + }, + dictWord{137, 11, 743}, + dictWord{134, 10, 278}, + dictWord{6, 0, 1803}, + dictWord{18, 0, 165}, + dictWord{24, 0, 21}, + dictWord{5, 11, 169}, + dictWord{ + 7, + 11, + 333, + }, + dictWord{136, 11, 45}, + dictWord{12, 10, 97}, + dictWord{140, 11, 97}, + dictWord{4, 0, 408}, + dictWord{4, 0, 741}, + dictWord{135, 0, 500}, + dictWord{ + 132, + 11, + 198, + }, + dictWord{7, 10, 388}, + dictWord{7, 10, 644}, + dictWord{139, 10, 781}, + dictWord{4, 11, 24}, + dictWord{5, 11, 140}, + dictWord{5, 11, 185}, + dictWord{ + 7, + 11, + 1500, + }, + dictWord{11, 11, 565}, + dictWord{139, 11, 838}, + dictWord{6, 0, 1321}, + dictWord{9, 0, 257}, + dictWord{7, 10, 229}, + dictWord{8, 10, 59}, + dictWord{ + 9, + 10, + 190, + }, + dictWord{10, 10, 378}, + dictWord{140, 10, 191}, + dictWord{4, 11, 334}, + dictWord{133, 11, 593}, + dictWord{135, 11, 1885}, + dictWord{134, 0, 1138}, + dictWord{4, 0, 249}, + dictWord{6, 0, 73}, + dictWord{135, 0, 177}, + dictWord{133, 0, 576}, + dictWord{142, 0, 231}, + dictWord{137, 0, 288}, + dictWord{132, 10, 660}, + dictWord{7, 10, 1035}, + dictWord{138, 10, 737}, + dictWord{135, 0, 1487}, + dictWord{6, 0, 989}, + dictWord{9, 0, 433}, + dictWord{7, 10, 690}, + dictWord{9, 10, 587}, + dictWord{140, 10, 521}, + dictWord{7, 0, 1264}, + dictWord{7, 0, 1678}, + dictWord{11, 0, 945}, + dictWord{12, 0, 341}, + dictWord{12, 0, 471}, + dictWord{140, 0, 569}, + dictWord{132, 11, 709}, + dictWord{133, 11, 897}, + dictWord{5, 11, 224}, + dictWord{13, 11, 174}, + dictWord{146, 11, 52}, + dictWord{135, 11, 1840}, + dictWord{ + 134, + 10, + 1744, + }, + dictWord{12, 0, 87}, + dictWord{16, 0, 74}, + dictWord{4, 10, 733}, + dictWord{9, 10, 194}, + dictWord{10, 10, 92}, + dictWord{11, 10, 198}, + dictWord{ + 12, + 10, + 84, + }, + dictWord{141, 10, 128}, + dictWord{140, 0, 779}, + dictWord{135, 0, 538}, + dictWord{4, 11, 608}, + dictWord{133, 11, 497}, + dictWord{133, 0, 413}, + dictWord{7, 11, 1375}, + dictWord{7, 11, 1466}, + dictWord{138, 11, 331}, + dictWord{136, 0, 495}, + dictWord{6, 11, 540}, + dictWord{136, 11, 136}, + dictWord{7, 0, 54}, + dictWord{8, 0, 312}, + dictWord{10, 0, 191}, + dictWord{10, 0, 614}, + dictWord{140, 0, 567}, + dictWord{6, 0, 468}, + dictWord{7, 0, 567}, + dictWord{7, 0, 1478}, + dictWord{ + 8, + 0, + 530, + }, + dictWord{14, 0, 290}, + dictWord{133, 11, 999}, + dictWord{4, 11, 299}, + dictWord{7, 10, 306}, + dictWord{135, 11, 1004}, + dictWord{142, 11, 296}, + dictWord{134, 0, 1484}, + dictWord{133, 10, 979}, + dictWord{6, 0, 609}, + dictWord{9, 0, 815}, + dictWord{12, 11, 137}, + dictWord{14, 11, 9}, + dictWord{14, 11, 24}, + dictWord{142, 11, 64}, + dictWord{133, 11, 456}, + dictWord{6, 0, 484}, + dictWord{135, 0, 822}, + dictWord{133, 10, 178}, + dictWord{136, 11, 180}, + dictWord{ + 132, + 11, + 755, + }, + dictWord{137, 0, 900}, + dictWord{135, 0, 1335}, + dictWord{6, 0, 1724}, + dictWord{135, 0, 2022}, + dictWord{135, 11, 1139}, + dictWord{5, 0, 640}, + dictWord{132, 10, 390}, + dictWord{6, 0, 1831}, + dictWord{138, 11, 633}, + dictWord{135, 11, 566}, + dictWord{4, 11, 890}, + dictWord{5, 11, 805}, + dictWord{5, 11, 819}, + dictWord{5, 11, 961}, + dictWord{6, 11, 396}, + dictWord{6, 11, 1631}, + dictWord{6, 11, 1678}, + dictWord{7, 11, 1967}, + dictWord{7, 11, 2041}, + dictWord{ + 9, + 11, + 630, + }, + dictWord{11, 11, 8}, + dictWord{11, 11, 1019}, + dictWord{12, 11, 176}, + dictWord{13, 11, 225}, + dictWord{14, 11, 292}, + dictWord{149, 11, 24}, + dictWord{ + 132, + 0, + 474, + }, + dictWord{134, 0, 1103}, + dictWord{135, 0, 1504}, + dictWord{134, 0, 1576}, + dictWord{6, 0, 961}, + dictWord{6, 0, 1034}, + dictWord{140, 0, 655}, + dictWord{11, 11, 514}, + dictWord{149, 11, 20}, + dictWord{5, 0, 305}, + dictWord{135, 11, 1815}, + dictWord{7, 11, 1505}, + dictWord{10, 11, 190}, + dictWord{ + 10, + 11, + 634, + }, + dictWord{11, 11, 792}, + dictWord{12, 11, 358}, + dictWord{140, 11, 447}, + dictWord{5, 11, 0}, + dictWord{6, 11, 536}, + dictWord{7, 11, 604}, + dictWord{ + 13, + 11, + 445, + }, + dictWord{145, 11, 126}, + dictWord{7, 0, 1236}, + dictWord{133, 10, 105}, + dictWord{4, 0, 480}, + dictWord{6, 0, 217}, + dictWord{6, 0, 302}, + dictWord{ + 6, + 0, + 1642, + }, + dictWord{7, 0, 130}, + dictWord{7, 0, 837}, + dictWord{7, 0, 1321}, + dictWord{7, 0, 1547}, + dictWord{7, 0, 1657}, + dictWord{8, 0, 429}, + dictWord{9, 0, 228}, + dictWord{13, 0, 289}, + dictWord{13, 0, 343}, + dictWord{19, 0, 101}, + dictWord{6, 11, 232}, + dictWord{6, 11, 412}, + dictWord{7, 11, 1074}, + dictWord{8, 11, 9}, + dictWord{ + 8, + 11, + 157, + }, + dictWord{8, 11, 786}, + dictWord{9, 11, 196}, + dictWord{9, 11, 352}, + dictWord{9, 11, 457}, + dictWord{10, 11, 337}, + dictWord{11, 11, 232}, + dictWord{ + 11, + 11, + 877, + }, + dictWord{12, 11, 480}, + dictWord{140, 11, 546}, + dictWord{5, 10, 438}, + dictWord{7, 11, 958}, + dictWord{9, 10, 694}, + dictWord{12, 10, 627}, + dictWord{ + 13, + 11, + 38, + }, + dictWord{141, 10, 210}, + dictWord{4, 11, 382}, + dictWord{136, 11, 579}, + dictWord{7, 0, 278}, + dictWord{10, 0, 739}, + dictWord{11, 0, 708}, + dictWord{ + 141, + 0, + 348, + }, + dictWord{4, 11, 212}, + dictWord{135, 11, 1206}, + dictWord{135, 11, 1898}, + dictWord{6, 0, 708}, + dictWord{6, 0, 1344}, + dictWord{152, 10, 11}, + dictWord{137, 11, 768}, + dictWord{134, 0, 1840}, + dictWord{140, 0, 233}, + dictWord{8, 10, 25}, + dictWord{138, 10, 826}, + dictWord{6, 0, 2017}, + dictWord{ + 133, + 11, + 655, + }, + dictWord{6, 0, 1488}, + dictWord{139, 11, 290}, + dictWord{132, 10, 308}, + dictWord{134, 0, 1590}, + dictWord{134, 0, 1800}, + dictWord{134, 0, 1259}, + dictWord{16, 0, 28}, + dictWord{6, 11, 231}, + dictWord{7, 11, 95}, + dictWord{136, 11, 423}, + dictWord{133, 11, 300}, + dictWord{135, 10, 150}, + dictWord{ + 136, + 10, + 649, + }, + dictWord{7, 11, 1874}, + dictWord{137, 11, 641}, + dictWord{6, 11, 237}, + dictWord{7, 11, 611}, + dictWord{8, 11, 100}, + dictWord{9, 11, 416}, + dictWord{ + 11, + 11, + 335, + }, + dictWord{12, 11, 173}, + dictWord{146, 11, 101}, + dictWord{137, 0, 45}, + dictWord{134, 10, 521}, + dictWord{17, 0, 36}, + dictWord{14, 11, 26}, + dictWord{ + 146, + 11, + 150, + }, + dictWord{7, 0, 1442}, + dictWord{14, 0, 22}, + dictWord{5, 10, 339}, + dictWord{15, 10, 41}, + dictWord{15, 10, 166}, + dictWord{147, 10, 66}, + dictWord{ + 8, + 0, + 378, + }, + dictWord{6, 11, 581}, + dictWord{135, 11, 1119}, + dictWord{134, 0, 1507}, + dictWord{147, 11, 117}, + dictWord{139, 0, 39}, + dictWord{134, 0, 1054}, + dictWord{6, 0, 363}, + dictWord{7, 0, 1955}, + dictWord{136, 0, 725}, + dictWord{134, 0, 2036}, + dictWord{133, 11, 199}, + dictWord{6, 0, 1871}, + dictWord{9, 0, 935}, + dictWord{9, 0, 961}, + dictWord{9, 0, 1004}, + dictWord{9, 0, 1016}, + dictWord{12, 0, 805}, + dictWord{12, 0, 852}, + dictWord{12, 0, 853}, + dictWord{12, 0, 869}, + dictWord{ + 12, + 0, + 882, + }, + dictWord{12, 0, 896}, + dictWord{12, 0, 906}, + dictWord{12, 0, 917}, + dictWord{12, 0, 940}, + dictWord{15, 0, 170}, + dictWord{15, 0, 176}, + dictWord{ + 15, + 0, + 188, + }, + dictWord{15, 0, 201}, + dictWord{15, 0, 205}, + dictWord{15, 0, 212}, + dictWord{15, 0, 234}, + dictWord{15, 0, 244}, + dictWord{18, 0, 181}, + dictWord{18, 0, 193}, + dictWord{18, 0, 196}, + dictWord{18, 0, 201}, + dictWord{18, 0, 202}, + dictWord{18, 0, 210}, + dictWord{18, 0, 217}, + dictWord{18, 0, 235}, + dictWord{18, 0, 236}, + dictWord{18, 0, 237}, + dictWord{21, 0, 54}, + dictWord{21, 0, 55}, + dictWord{21, 0, 58}, + dictWord{21, 0, 59}, + dictWord{152, 0, 22}, + dictWord{134, 10, 1628}, + dictWord{ + 137, + 0, + 805, + }, + dictWord{5, 0, 813}, + dictWord{135, 0, 2046}, + dictWord{142, 11, 42}, + dictWord{5, 0, 712}, + dictWord{6, 0, 1240}, + dictWord{11, 0, 17}, + dictWord{ + 13, + 0, + 321, + }, + dictWord{144, 0, 67}, + dictWord{132, 0, 617}, + dictWord{135, 10, 829}, + dictWord{6, 0, 320}, + dictWord{7, 0, 781}, + dictWord{7, 0, 1921}, + dictWord{9, 0, 55}, + dictWord{10, 0, 186}, + dictWord{10, 0, 273}, + dictWord{10, 0, 664}, + dictWord{10, 0, 801}, + dictWord{11, 0, 996}, + dictWord{11, 0, 997}, + dictWord{13, 0, 157}, + dictWord{142, 0, 170}, + dictWord{136, 0, 271}, + dictWord{5, 10, 486}, + dictWord{135, 10, 1349}, + dictWord{18, 11, 91}, + dictWord{147, 11, 70}, + dictWord{10, 0, 445}, + dictWord{7, 10, 1635}, + dictWord{8, 10, 17}, + dictWord{138, 10, 295}, + dictWord{136, 11, 404}, + dictWord{7, 0, 103}, + dictWord{7, 0, 863}, + dictWord{11, 0, 184}, + dictWord{145, 0, 62}, + dictWord{138, 10, 558}, + dictWord{137, 0, 659}, + dictWord{6, 11, 312}, + dictWord{6, 11, 1715}, + dictWord{10, 11, 584}, + dictWord{ + 11, + 11, + 546, + }, + dictWord{11, 11, 692}, + dictWord{12, 11, 259}, + dictWord{12, 11, 295}, + dictWord{13, 11, 46}, + dictWord{141, 11, 154}, + dictWord{134, 0, 676}, + dictWord{132, 11, 588}, + dictWord{4, 11, 231}, + dictWord{5, 11, 61}, + dictWord{6, 11, 104}, + dictWord{7, 11, 729}, + dictWord{7, 11, 964}, + dictWord{7, 11, 1658}, + dictWord{140, 11, 414}, + dictWord{6, 11, 263}, + dictWord{138, 11, 757}, + dictWord{11, 0, 337}, + dictWord{142, 0, 303}, + dictWord{135, 11, 1363}, + dictWord{ + 132, + 11, + 320, + }, + dictWord{140, 0, 506}, + dictWord{134, 10, 447}, + dictWord{5, 0, 77}, + dictWord{7, 0, 1455}, + dictWord{10, 0, 843}, + dictWord{147, 0, 73}, + dictWord{ + 7, + 10, + 577, + }, + dictWord{7, 10, 1432}, + dictWord{9, 10, 475}, + dictWord{9, 10, 505}, + dictWord{9, 10, 526}, + dictWord{9, 10, 609}, + dictWord{9, 10, 689}, + dictWord{ + 9, + 10, + 726, + }, + dictWord{9, 10, 735}, + dictWord{9, 10, 738}, + dictWord{10, 10, 556}, + dictWord{10, 10, 674}, + dictWord{10, 10, 684}, + dictWord{11, 10, 89}, + dictWord{ + 11, + 10, + 202, + }, + dictWord{11, 10, 272}, + dictWord{11, 10, 380}, + dictWord{11, 10, 415}, + dictWord{11, 10, 505}, + dictWord{11, 10, 537}, + dictWord{11, 10, 550}, + dictWord{11, 10, 562}, + dictWord{11, 10, 640}, + dictWord{11, 10, 667}, + dictWord{11, 10, 688}, + dictWord{11, 10, 847}, + dictWord{11, 10, 927}, + dictWord{ + 11, + 10, + 930, + }, + dictWord{11, 10, 940}, + dictWord{12, 10, 144}, + dictWord{12, 10, 325}, + dictWord{12, 10, 329}, + dictWord{12, 10, 389}, + dictWord{12, 10, 403}, + dictWord{ + 12, + 10, + 451, + }, + dictWord{12, 10, 515}, + dictWord{12, 10, 604}, + dictWord{12, 10, 616}, + dictWord{12, 10, 626}, + dictWord{13, 10, 66}, + dictWord{13, 10, 131}, + dictWord{13, 10, 167}, + dictWord{13, 10, 236}, + dictWord{13, 10, 368}, + dictWord{13, 10, 411}, + dictWord{13, 10, 434}, + dictWord{13, 10, 453}, + dictWord{ + 13, + 10, + 461, + }, + dictWord{13, 10, 474}, + dictWord{14, 10, 59}, + dictWord{14, 10, 60}, + dictWord{14, 10, 139}, + dictWord{14, 10, 152}, + dictWord{14, 10, 276}, + dictWord{ + 14, + 10, + 353, + }, + dictWord{14, 10, 402}, + dictWord{15, 10, 28}, + dictWord{15, 10, 81}, + dictWord{15, 10, 123}, + dictWord{15, 10, 152}, + dictWord{18, 10, 136}, + dictWord{148, 10, 88}, + dictWord{132, 0, 458}, + dictWord{135, 0, 1420}, + dictWord{6, 0, 109}, + dictWord{10, 0, 382}, + dictWord{4, 11, 405}, + dictWord{4, 10, 609}, + dictWord{7, 10, 756}, + dictWord{7, 11, 817}, + dictWord{9, 10, 544}, + dictWord{11, 10, 413}, + dictWord{14, 11, 58}, + dictWord{14, 10, 307}, + dictWord{16, 10, 25}, + dictWord{17, 11, 37}, + dictWord{146, 11, 124}, + dictWord{6, 0, 330}, + dictWord{7, 0, 1084}, + dictWord{11, 0, 142}, + dictWord{133, 11, 974}, + dictWord{4, 10, 930}, + dictWord{133, 10, 947}, + dictWord{5, 10, 939}, + dictWord{142, 11, 394}, + dictWord{16, 0, 91}, + dictWord{145, 0, 87}, + dictWord{5, 11, 235}, + dictWord{5, 10, 962}, + dictWord{7, 11, 1239}, + dictWord{11, 11, 131}, + dictWord{140, 11, 370}, + dictWord{11, 0, 492}, + dictWord{5, 10, 651}, + dictWord{8, 10, 170}, + dictWord{9, 10, 61}, + dictWord{9, 10, 63}, + dictWord{10, 10, 23}, + dictWord{10, 10, 37}, + dictWord{10, 10, 834}, + dictWord{11, 10, 4}, + dictWord{11, 10, 281}, + dictWord{11, 10, 503}, + dictWord{ + 11, + 10, + 677, + }, + dictWord{12, 10, 96}, + dictWord{12, 10, 130}, + dictWord{12, 10, 244}, + dictWord{14, 10, 5}, + dictWord{14, 10, 40}, + dictWord{14, 10, 162}, + dictWord{ + 14, + 10, + 202, + }, + dictWord{146, 10, 133}, + dictWord{4, 10, 406}, + dictWord{5, 10, 579}, + dictWord{12, 10, 492}, + dictWord{150, 10, 15}, + dictWord{9, 11, 137}, + dictWord{138, 11, 221}, + dictWord{134, 0, 1239}, + dictWord{11, 0, 211}, + dictWord{140, 0, 145}, + dictWord{7, 11, 390}, + dictWord{138, 11, 140}, + dictWord{ + 135, + 11, + 1418, + }, + dictWord{135, 11, 1144}, + dictWord{134, 0, 1049}, + dictWord{7, 0, 321}, + dictWord{6, 10, 17}, + dictWord{7, 10, 1001}, + dictWord{7, 10, 1982}, + dictWord{ + 9, + 10, + 886, + }, + dictWord{10, 10, 489}, + dictWord{10, 10, 800}, + dictWord{11, 10, 782}, + dictWord{12, 10, 320}, + dictWord{13, 10, 467}, + dictWord{14, 10, 145}, + dictWord{14, 10, 387}, + dictWord{143, 10, 119}, + dictWord{145, 10, 17}, + dictWord{5, 11, 407}, + dictWord{11, 11, 489}, + dictWord{19, 11, 37}, + dictWord{20, 11, 73}, + dictWord{150, 11, 38}, + dictWord{133, 10, 458}, + dictWord{135, 0, 1985}, + dictWord{7, 10, 1983}, + dictWord{8, 10, 0}, + dictWord{8, 10, 171}, + dictWord{ + 9, + 10, + 120, + }, + dictWord{9, 10, 732}, + dictWord{10, 10, 473}, + dictWord{11, 10, 656}, + dictWord{11, 10, 998}, + dictWord{18, 10, 0}, + dictWord{18, 10, 2}, + dictWord{ + 147, + 10, + 21, + }, + dictWord{5, 11, 325}, + dictWord{7, 11, 1483}, + dictWord{8, 11, 5}, + dictWord{8, 11, 227}, + dictWord{9, 11, 105}, + dictWord{10, 11, 585}, + dictWord{ + 140, + 11, + 614, + }, + dictWord{136, 0, 122}, + dictWord{132, 0, 234}, + dictWord{135, 11, 1196}, + dictWord{6, 0, 976}, + dictWord{6, 0, 1098}, + dictWord{134, 0, 1441}, + dictWord{ + 7, + 0, + 253, + }, + dictWord{136, 0, 549}, + dictWord{6, 11, 621}, + dictWord{13, 11, 504}, + dictWord{144, 11, 19}, + dictWord{132, 10, 519}, + dictWord{5, 0, 430}, + dictWord{ + 5, + 0, + 932, + }, + dictWord{6, 0, 131}, + dictWord{7, 0, 417}, + dictWord{9, 0, 522}, + dictWord{11, 0, 314}, + dictWord{141, 0, 390}, + dictWord{14, 0, 149}, + dictWord{14, 0, 399}, + dictWord{143, 0, 57}, + dictWord{5, 10, 907}, + dictWord{6, 10, 31}, + dictWord{6, 11, 218}, + dictWord{7, 10, 491}, + dictWord{7, 10, 530}, + dictWord{8, 10, 592}, + dictWord{11, 10, 53}, + dictWord{11, 10, 779}, + dictWord{12, 10, 167}, + dictWord{12, 10, 411}, + dictWord{14, 10, 14}, + dictWord{14, 10, 136}, + dictWord{15, 10, 72}, + dictWord{16, 10, 17}, + dictWord{144, 10, 72}, + dictWord{140, 11, 330}, + dictWord{7, 11, 454}, + dictWord{7, 11, 782}, + dictWord{136, 11, 768}, + dictWord{ + 132, + 0, + 507, + }, + dictWord{10, 11, 676}, + dictWord{140, 11, 462}, + dictWord{6, 0, 630}, + dictWord{9, 0, 811}, + dictWord{4, 10, 208}, + dictWord{5, 10, 106}, + dictWord{ + 6, + 10, + 531, + }, + dictWord{8, 10, 408}, + dictWord{9, 10, 188}, + dictWord{138, 10, 572}, + dictWord{4, 0, 343}, + dictWord{5, 0, 511}, + dictWord{134, 10, 1693}, + dictWord{ + 134, + 11, + 164, + }, + dictWord{132, 0, 448}, + dictWord{7, 0, 455}, + dictWord{138, 0, 591}, + dictWord{135, 0, 1381}, + dictWord{12, 10, 441}, + dictWord{150, 11, 50}, + dictWord{9, 10, 449}, + dictWord{10, 10, 192}, + dictWord{138, 10, 740}, + dictWord{6, 0, 575}, + dictWord{132, 10, 241}, + dictWord{134, 0, 1175}, + dictWord{ + 134, + 0, + 653, + }, + dictWord{134, 0, 1761}, + dictWord{134, 0, 1198}, + dictWord{132, 10, 259}, + dictWord{6, 11, 343}, + dictWord{7, 11, 195}, + dictWord{9, 11, 226}, + dictWord{ + 10, + 11, + 197, + }, + dictWord{10, 11, 575}, + dictWord{11, 11, 502}, + dictWord{139, 11, 899}, + dictWord{7, 0, 1127}, + dictWord{7, 0, 1572}, + dictWord{10, 0, 297}, + dictWord{10, 0, 422}, + dictWord{11, 0, 764}, + dictWord{11, 0, 810}, + dictWord{12, 0, 264}, + dictWord{13, 0, 102}, + dictWord{13, 0, 300}, + dictWord{13, 0, 484}, + dictWord{ + 14, + 0, + 147, + }, + dictWord{14, 0, 229}, + dictWord{17, 0, 71}, + dictWord{18, 0, 118}, + dictWord{147, 0, 120}, + dictWord{135, 11, 666}, + dictWord{132, 0, 678}, + dictWord{ + 4, + 10, + 173, + }, + dictWord{5, 10, 312}, + dictWord{5, 10, 512}, + dictWord{135, 10, 1285}, + dictWord{7, 10, 1603}, + dictWord{7, 10, 1691}, + dictWord{9, 10, 464}, + dictWord{11, 10, 195}, + dictWord{12, 10, 279}, + dictWord{12, 10, 448}, + dictWord{14, 10, 11}, + dictWord{147, 10, 102}, + dictWord{16, 0, 99}, + dictWord{146, 0, 164}, + dictWord{7, 11, 1125}, + dictWord{9, 11, 143}, + dictWord{11, 11, 61}, + dictWord{14, 11, 405}, + dictWord{150, 11, 21}, + dictWord{137, 11, 260}, + dictWord{ + 4, + 10, + 452, + }, + dictWord{5, 10, 583}, + dictWord{5, 10, 817}, + dictWord{6, 10, 433}, + dictWord{7, 10, 593}, + dictWord{7, 10, 720}, + dictWord{7, 10, 1378}, + dictWord{ + 8, + 10, + 161, + }, + dictWord{9, 10, 284}, + dictWord{10, 10, 313}, + dictWord{139, 10, 886}, + dictWord{132, 10, 547}, + dictWord{136, 10, 722}, + dictWord{14, 0, 35}, + dictWord{142, 0, 191}, + dictWord{141, 0, 45}, + dictWord{138, 0, 121}, + dictWord{132, 0, 125}, + dictWord{134, 0, 1622}, + dictWord{133, 11, 959}, + dictWord{ + 8, + 10, + 420, + }, + dictWord{139, 10, 193}, + dictWord{132, 0, 721}, + dictWord{135, 10, 409}, + dictWord{136, 0, 145}, + dictWord{7, 0, 792}, + dictWord{8, 0, 147}, + dictWord{ + 10, + 0, + 821, + }, + dictWord{11, 0, 970}, + dictWord{11, 0, 1021}, + dictWord{136, 11, 173}, + dictWord{134, 11, 266}, + dictWord{132, 0, 715}, + dictWord{7, 0, 1999}, + dictWord{138, 10, 308}, + dictWord{133, 0, 531}, + dictWord{5, 0, 168}, + dictWord{5, 0, 930}, + dictWord{8, 0, 74}, + dictWord{9, 0, 623}, + dictWord{12, 0, 500}, + dictWord{ + 140, + 0, + 579, + }, + dictWord{144, 0, 65}, + dictWord{138, 11, 246}, + dictWord{6, 0, 220}, + dictWord{7, 0, 1101}, + dictWord{13, 0, 105}, + dictWord{142, 11, 314}, + dictWord{ + 5, + 10, + 1002, + }, + dictWord{136, 10, 745}, + dictWord{134, 0, 960}, + dictWord{20, 0, 0}, + dictWord{148, 11, 0}, + dictWord{4, 0, 1005}, + dictWord{4, 10, 239}, + dictWord{ + 6, + 10, + 477, + }, + dictWord{7, 10, 1607}, + dictWord{11, 10, 68}, + dictWord{139, 10, 617}, + dictWord{6, 0, 19}, + dictWord{7, 0, 1413}, + dictWord{139, 0, 428}, + dictWord{ + 149, + 10, + 13, + }, + dictWord{7, 0, 96}, + dictWord{8, 0, 401}, + dictWord{8, 0, 703}, + dictWord{9, 0, 896}, + dictWord{136, 11, 300}, + dictWord{134, 0, 1595}, + dictWord{145, 0, 116}, + dictWord{136, 0, 1021}, + dictWord{7, 0, 1961}, + dictWord{7, 0, 1965}, + dictWord{7, 0, 2030}, + dictWord{8, 0, 150}, + dictWord{8, 0, 702}, + dictWord{8, 0, 737}, + dictWord{ + 8, + 0, + 750, + }, + dictWord{140, 0, 366}, + dictWord{11, 11, 75}, + dictWord{142, 11, 267}, + dictWord{132, 10, 367}, + dictWord{8, 0, 800}, + dictWord{9, 0, 148}, + dictWord{ + 9, + 0, + 872, + }, + dictWord{9, 0, 890}, + dictWord{11, 0, 309}, + dictWord{11, 0, 1001}, + dictWord{13, 0, 267}, + dictWord{13, 0, 323}, + dictWord{5, 11, 427}, + dictWord{ + 5, + 11, + 734, + }, + dictWord{7, 11, 478}, + dictWord{136, 11, 52}, + dictWord{7, 11, 239}, + dictWord{11, 11, 217}, + dictWord{142, 11, 165}, + dictWord{132, 11, 323}, + dictWord{140, 11, 419}, + dictWord{13, 0, 299}, + dictWord{142, 0, 75}, + dictWord{6, 11, 87}, + dictWord{6, 11, 1734}, + dictWord{7, 11, 20}, + dictWord{7, 11, 1056}, + dictWord{ + 8, + 11, + 732, + }, + dictWord{9, 11, 406}, + dictWord{9, 11, 911}, + dictWord{138, 11, 694}, + dictWord{134, 0, 1383}, + dictWord{132, 10, 694}, + dictWord{ + 133, + 11, + 613, + }, + dictWord{137, 0, 779}, + dictWord{4, 0, 598}, + dictWord{140, 10, 687}, + dictWord{6, 0, 970}, + dictWord{135, 0, 424}, + dictWord{133, 0, 547}, + dictWord{ + 7, + 11, + 32, + }, + dictWord{7, 11, 984}, + dictWord{8, 11, 85}, + dictWord{8, 11, 709}, + dictWord{9, 11, 579}, + dictWord{9, 11, 847}, + dictWord{9, 11, 856}, + dictWord{10, 11, 799}, + dictWord{11, 11, 258}, + dictWord{11, 11, 1007}, + dictWord{12, 11, 331}, + dictWord{12, 11, 615}, + dictWord{13, 11, 188}, + dictWord{13, 11, 435}, + dictWord{ + 14, + 11, + 8, + }, + dictWord{15, 11, 165}, + dictWord{16, 11, 27}, + dictWord{148, 11, 40}, + dictWord{6, 0, 1222}, + dictWord{134, 0, 1385}, + dictWord{132, 0, 876}, + dictWord{ + 138, + 11, + 151, + }, + dictWord{135, 10, 213}, + dictWord{4, 11, 167}, + dictWord{135, 11, 82}, + dictWord{133, 0, 133}, + dictWord{6, 11, 24}, + dictWord{7, 11, 74}, + dictWord{ + 7, + 11, + 678, + }, + dictWord{137, 11, 258}, + dictWord{5, 11, 62}, + dictWord{6, 11, 534}, + dictWord{7, 11, 684}, + dictWord{7, 11, 1043}, + dictWord{7, 11, 1072}, + dictWord{ + 8, + 11, + 280, + }, + dictWord{8, 11, 541}, + dictWord{8, 11, 686}, + dictWord{10, 11, 519}, + dictWord{11, 11, 252}, + dictWord{140, 11, 282}, + dictWord{136, 0, 187}, + dictWord{8, 0, 8}, + dictWord{10, 0, 0}, + dictWord{10, 0, 818}, + dictWord{139, 0, 988}, + dictWord{132, 11, 359}, + dictWord{11, 0, 429}, + dictWord{15, 0, 51}, + dictWord{ + 135, + 10, + 1672, + }, + dictWord{136, 0, 685}, + dictWord{5, 11, 211}, + dictWord{7, 11, 88}, + dictWord{136, 11, 627}, + dictWord{134, 0, 472}, + dictWord{136, 0, 132}, + dictWord{ + 6, + 11, + 145, + }, + dictWord{141, 11, 336}, + dictWord{4, 10, 751}, + dictWord{11, 10, 390}, + dictWord{140, 10, 32}, + dictWord{6, 0, 938}, + dictWord{6, 0, 1060}, + dictWord{ + 4, + 11, + 263, + }, + dictWord{4, 10, 409}, + dictWord{133, 10, 78}, + dictWord{137, 0, 874}, + dictWord{8, 0, 774}, + dictWord{10, 0, 670}, + dictWord{12, 0, 51}, + dictWord{ + 4, + 11, + 916, + }, + dictWord{6, 10, 473}, + dictWord{7, 10, 1602}, + dictWord{10, 10, 698}, + dictWord{12, 10, 212}, + dictWord{13, 10, 307}, + dictWord{145, 10, 105}, + dictWord{146, 0, 92}, + dictWord{143, 10, 156}, + dictWord{132, 0, 830}, + dictWord{137, 0, 701}, + dictWord{4, 11, 599}, + dictWord{6, 11, 1634}, + dictWord{7, 11, 5}, + dictWord{7, 11, 55}, + dictWord{7, 11, 67}, + dictWord{7, 11, 97}, + dictWord{7, 11, 691}, + dictWord{7, 11, 979}, + dictWord{7, 11, 1697}, + dictWord{8, 11, 207}, + dictWord{ + 8, + 11, + 214, + }, + dictWord{8, 11, 231}, + dictWord{8, 11, 294}, + dictWord{8, 11, 336}, + dictWord{8, 11, 428}, + dictWord{8, 11, 451}, + dictWord{8, 11, 460}, + dictWord{8, 11, 471}, + dictWord{8, 11, 622}, + dictWord{8, 11, 626}, + dictWord{8, 11, 679}, + dictWord{8, 11, 759}, + dictWord{8, 11, 829}, + dictWord{9, 11, 11}, + dictWord{9, 11, 246}, + dictWord{ + 9, + 11, + 484, + }, + dictWord{9, 11, 573}, + dictWord{9, 11, 706}, + dictWord{9, 11, 762}, + dictWord{9, 11, 798}, + dictWord{9, 11, 855}, + dictWord{9, 11, 870}, + dictWord{ + 9, + 11, + 912, + }, + dictWord{10, 11, 303}, + dictWord{10, 11, 335}, + dictWord{10, 11, 424}, + dictWord{10, 11, 461}, + dictWord{10, 11, 543}, + dictWord{10, 11, 759}, + dictWord{10, 11, 814}, + dictWord{11, 11, 59}, + dictWord{11, 11, 199}, + dictWord{11, 11, 235}, + dictWord{11, 11, 475}, + dictWord{11, 11, 590}, + dictWord{11, 11, 929}, + dictWord{11, 11, 963}, + dictWord{12, 11, 114}, + dictWord{12, 11, 182}, + dictWord{12, 11, 226}, + dictWord{12, 11, 332}, + dictWord{12, 11, 439}, + dictWord{ + 12, + 11, + 575, + }, + dictWord{12, 11, 598}, + dictWord{13, 11, 8}, + dictWord{13, 11, 125}, + dictWord{13, 11, 194}, + dictWord{13, 11, 287}, + dictWord{14, 11, 197}, + dictWord{ + 14, + 11, + 383, + }, + dictWord{15, 11, 53}, + dictWord{17, 11, 63}, + dictWord{19, 11, 46}, + dictWord{19, 11, 98}, + dictWord{19, 11, 106}, + dictWord{148, 11, 85}, + dictWord{ + 4, + 0, + 127, + }, + dictWord{5, 0, 350}, + dictWord{6, 0, 356}, + dictWord{8, 0, 426}, + dictWord{9, 0, 572}, + dictWord{10, 0, 247}, + dictWord{139, 0, 312}, + dictWord{134, 0, 1215}, + dictWord{6, 0, 59}, + dictWord{9, 0, 603}, + dictWord{13, 0, 397}, + dictWord{7, 11, 1853}, + dictWord{138, 11, 437}, + dictWord{134, 0, 1762}, + dictWord{ + 147, + 11, + 126, + }, + dictWord{135, 10, 883}, + dictWord{13, 0, 293}, + dictWord{142, 0, 56}, + dictWord{133, 10, 617}, + dictWord{139, 10, 50}, + dictWord{5, 11, 187}, + dictWord{ + 7, + 10, + 1518, + }, + dictWord{139, 10, 694}, + dictWord{135, 0, 441}, + dictWord{6, 0, 111}, + dictWord{7, 0, 4}, + dictWord{8, 0, 163}, + dictWord{8, 0, 776}, + dictWord{ + 138, + 0, + 566, + }, + dictWord{132, 0, 806}, + dictWord{4, 11, 215}, + dictWord{9, 11, 38}, + dictWord{10, 11, 3}, + dictWord{11, 11, 23}, + dictWord{11, 11, 127}, + dictWord{ + 139, + 11, + 796, + }, + dictWord{14, 0, 233}, + dictWord{4, 10, 546}, + dictWord{135, 10, 2042}, + dictWord{135, 0, 1994}, + dictWord{134, 0, 1739}, + dictWord{135, 11, 1530}, + dictWord{136, 0, 393}, + dictWord{5, 0, 297}, + dictWord{7, 0, 1038}, + dictWord{14, 0, 359}, + dictWord{19, 0, 52}, + dictWord{148, 0, 47}, + dictWord{135, 0, 309}, + dictWord{ + 4, + 10, + 313, + }, + dictWord{133, 10, 577}, + dictWord{8, 10, 184}, + dictWord{141, 10, 433}, + dictWord{135, 10, 935}, + dictWord{12, 10, 186}, + dictWord{ + 12, + 10, + 292, + }, + dictWord{14, 10, 100}, + dictWord{146, 10, 70}, + dictWord{136, 0, 363}, + dictWord{14, 0, 175}, + dictWord{11, 10, 402}, + dictWord{12, 10, 109}, + dictWord{ + 12, + 10, + 431, + }, + dictWord{13, 10, 179}, + dictWord{13, 10, 206}, + dictWord{14, 10, 217}, + dictWord{16, 10, 3}, + dictWord{148, 10, 53}, + dictWord{5, 10, 886}, + dictWord{ + 6, + 10, + 46, + }, + dictWord{6, 10, 1790}, + dictWord{7, 10, 14}, + dictWord{7, 10, 732}, + dictWord{7, 10, 1654}, + dictWord{8, 10, 95}, + dictWord{8, 10, 327}, + dictWord{ + 8, + 10, + 616, + }, + dictWord{9, 10, 892}, + dictWord{10, 10, 598}, + dictWord{10, 10, 769}, + dictWord{11, 10, 134}, + dictWord{11, 10, 747}, + dictWord{12, 10, 378}, + dictWord{ + 142, + 10, + 97, + }, + dictWord{136, 0, 666}, + dictWord{135, 0, 1675}, + dictWord{6, 0, 655}, + dictWord{134, 0, 1600}, + dictWord{135, 0, 808}, + dictWord{133, 10, 1021}, + dictWord{4, 11, 28}, + dictWord{5, 11, 440}, + dictWord{7, 11, 248}, + dictWord{11, 11, 833}, + dictWord{140, 11, 344}, + dictWord{134, 11, 1654}, + dictWord{ + 132, + 0, + 280, + }, + dictWord{140, 0, 54}, + dictWord{4, 0, 421}, + dictWord{133, 0, 548}, + dictWord{132, 10, 153}, + dictWord{6, 11, 339}, + dictWord{135, 11, 923}, + dictWord{ + 133, + 11, + 853, + }, + dictWord{133, 10, 798}, + dictWord{132, 10, 587}, + dictWord{6, 11, 249}, + dictWord{7, 11, 1234}, + dictWord{139, 11, 573}, + dictWord{6, 10, 598}, + dictWord{7, 10, 42}, + dictWord{8, 10, 695}, + dictWord{10, 10, 212}, + dictWord{11, 10, 158}, + dictWord{14, 10, 196}, + dictWord{145, 10, 85}, + dictWord{7, 0, 249}, + dictWord{5, 10, 957}, + dictWord{133, 10, 1008}, + dictWord{4, 10, 129}, + dictWord{135, 10, 465}, + dictWord{6, 0, 254}, + dictWord{7, 0, 842}, + dictWord{7, 0, 1659}, + dictWord{9, 0, 109}, + dictWord{10, 0, 103}, + dictWord{7, 10, 908}, + dictWord{7, 10, 1201}, + dictWord{9, 10, 755}, + dictWord{11, 10, 906}, + dictWord{12, 10, 527}, + dictWord{146, 10, 7}, + dictWord{5, 0, 262}, + dictWord{136, 10, 450}, + dictWord{144, 0, 1}, + dictWord{10, 11, 201}, + dictWord{142, 11, 319}, + dictWord{7, 11, 49}, + dictWord{ + 7, + 11, + 392, + }, + dictWord{8, 11, 20}, + dictWord{8, 11, 172}, + dictWord{8, 11, 690}, + dictWord{9, 11, 383}, + dictWord{9, 11, 845}, + dictWord{10, 11, 48}, + dictWord{ + 11, + 11, + 293, + }, + dictWord{11, 11, 832}, + dictWord{11, 11, 920}, + dictWord{141, 11, 221}, + dictWord{5, 11, 858}, + dictWord{133, 11, 992}, + dictWord{134, 0, 805}, + dictWord{139, 10, 1003}, + dictWord{6, 0, 1630}, + dictWord{134, 11, 307}, + dictWord{7, 11, 1512}, + dictWord{135, 11, 1794}, + dictWord{6, 11, 268}, + dictWord{ + 137, + 11, + 62, + }, + dictWord{135, 10, 1868}, + dictWord{133, 0, 671}, + dictWord{4, 0, 989}, + dictWord{8, 0, 972}, + dictWord{136, 0, 998}, + dictWord{132, 11, 423}, + dictWord{132, 0, 889}, + dictWord{135, 0, 1382}, + dictWord{135, 0, 1910}, + dictWord{7, 10, 965}, + dictWord{7, 10, 1460}, + dictWord{135, 10, 1604}, + dictWord{ + 4, + 0, + 627, + }, + dictWord{5, 0, 775}, + dictWord{138, 11, 106}, + dictWord{134, 11, 348}, + dictWord{7, 0, 202}, + dictWord{11, 0, 362}, + dictWord{11, 0, 948}, + dictWord{ + 140, + 0, + 388, + }, + dictWord{138, 11, 771}, + dictWord{6, 11, 613}, + dictWord{136, 11, 223}, + dictWord{6, 0, 560}, + dictWord{7, 0, 451}, + dictWord{8, 0, 389}, + dictWord{ + 12, + 0, + 490, + }, + dictWord{13, 0, 16}, + dictWord{13, 0, 215}, + dictWord{13, 0, 351}, + dictWord{18, 0, 132}, + dictWord{147, 0, 125}, + dictWord{135, 0, 841}, + dictWord{ + 136, + 0, + 566, + }, + dictWord{136, 0, 938}, + dictWord{132, 11, 670}, + dictWord{5, 0, 912}, + dictWord{6, 0, 1695}, + dictWord{140, 11, 55}, + dictWord{9, 11, 40}, + dictWord{ + 139, + 11, + 136, + }, + dictWord{7, 0, 1361}, + dictWord{7, 10, 982}, + dictWord{10, 10, 32}, + dictWord{143, 10, 56}, + dictWord{11, 11, 259}, + dictWord{140, 11, 270}, + dictWord{ + 5, + 0, + 236, + }, + dictWord{6, 0, 572}, + dictWord{8, 0, 492}, + dictWord{11, 0, 618}, + dictWord{144, 0, 56}, + dictWord{8, 11, 572}, + dictWord{9, 11, 310}, + dictWord{9, 11, 682}, + dictWord{137, 11, 698}, + dictWord{134, 0, 1854}, + dictWord{5, 0, 190}, + dictWord{136, 0, 318}, + dictWord{133, 10, 435}, + dictWord{135, 0, 1376}, + dictWord{ + 4, + 11, + 296, + }, + dictWord{6, 11, 352}, + dictWord{7, 11, 401}, + dictWord{7, 11, 1410}, + dictWord{7, 11, 1594}, + dictWord{7, 11, 1674}, + dictWord{8, 11, 63}, + dictWord{ + 8, + 11, + 660, + }, + dictWord{137, 11, 74}, + dictWord{7, 0, 349}, + dictWord{5, 10, 85}, + dictWord{6, 10, 419}, + dictWord{7, 10, 305}, + dictWord{7, 10, 361}, + dictWord{7, 10, 1337}, + dictWord{8, 10, 71}, + dictWord{140, 10, 519}, + dictWord{4, 11, 139}, + dictWord{4, 11, 388}, + dictWord{140, 11, 188}, + dictWord{6, 0, 1972}, + dictWord{6, 0, 2013}, + dictWord{8, 0, 951}, + dictWord{10, 0, 947}, + dictWord{10, 0, 974}, + dictWord{10, 0, 1018}, + dictWord{142, 0, 476}, + dictWord{140, 10, 688}, + dictWord{ + 135, + 10, + 740, + }, + dictWord{5, 10, 691}, + dictWord{7, 10, 345}, + dictWord{9, 10, 94}, + dictWord{140, 10, 169}, + dictWord{9, 0, 344}, + dictWord{5, 10, 183}, + dictWord{6, 10, 582}, + dictWord{10, 10, 679}, + dictWord{140, 10, 435}, + dictWord{135, 10, 511}, + dictWord{132, 0, 850}, + dictWord{8, 11, 441}, + dictWord{10, 11, 314}, + dictWord{ + 143, + 11, + 3, + }, + dictWord{7, 10, 1993}, + dictWord{136, 10, 684}, + dictWord{4, 11, 747}, + dictWord{6, 11, 290}, + dictWord{6, 10, 583}, + dictWord{7, 11, 649}, + dictWord{ + 7, + 11, + 1479, + }, + dictWord{135, 11, 1583}, + dictWord{133, 11, 232}, + dictWord{133, 10, 704}, + dictWord{134, 0, 910}, + dictWord{4, 10, 179}, + dictWord{5, 10, 198}, + dictWord{133, 10, 697}, + dictWord{7, 10, 347}, + dictWord{7, 10, 971}, + dictWord{8, 10, 181}, + dictWord{138, 10, 711}, + dictWord{136, 11, 525}, + dictWord{ + 14, + 0, + 19, + }, + dictWord{14, 0, 28}, + dictWord{144, 0, 29}, + dictWord{7, 0, 85}, + dictWord{7, 0, 247}, + dictWord{8, 0, 585}, + dictWord{138, 0, 163}, + dictWord{4, 0, 487}, + dictWord{ + 7, + 11, + 472, + }, + dictWord{7, 11, 1801}, + dictWord{10, 11, 748}, + dictWord{141, 11, 458}, + dictWord{4, 10, 243}, + dictWord{5, 10, 203}, + dictWord{7, 10, 19}, + dictWord{ + 7, + 10, + 71, + }, + dictWord{7, 10, 113}, + dictWord{10, 10, 405}, + dictWord{11, 10, 357}, + dictWord{142, 10, 240}, + dictWord{7, 10, 1450}, + dictWord{139, 10, 99}, + dictWord{132, 11, 425}, + dictWord{138, 0, 145}, + dictWord{147, 0, 83}, + dictWord{6, 10, 492}, + dictWord{137, 11, 247}, + dictWord{4, 0, 1013}, + dictWord{ + 134, + 0, + 2033, + }, + dictWord{5, 10, 134}, + dictWord{6, 10, 408}, + dictWord{6, 10, 495}, + dictWord{135, 10, 1593}, + dictWord{135, 0, 1922}, + dictWord{134, 11, 1768}, + dictWord{4, 0, 124}, + dictWord{10, 0, 457}, + dictWord{11, 0, 121}, + dictWord{11, 0, 169}, + dictWord{11, 0, 870}, + dictWord{11, 0, 874}, + dictWord{12, 0, 214}, + dictWord{ + 14, + 0, + 187, + }, + dictWord{143, 0, 77}, + dictWord{5, 0, 557}, + dictWord{135, 0, 1457}, + dictWord{139, 0, 66}, + dictWord{5, 11, 943}, + dictWord{6, 11, 1779}, + dictWord{ + 142, + 10, + 4, + }, + dictWord{4, 10, 248}, + dictWord{4, 10, 665}, + dictWord{7, 10, 137}, + dictWord{137, 10, 349}, + dictWord{7, 0, 1193}, + dictWord{5, 11, 245}, + dictWord{ + 6, + 11, + 576, + }, + dictWord{7, 11, 582}, + dictWord{136, 11, 225}, + dictWord{144, 0, 82}, + dictWord{7, 10, 1270}, + dictWord{139, 10, 612}, + dictWord{5, 0, 454}, + dictWord{ + 10, + 0, + 352, + }, + dictWord{138, 11, 352}, + dictWord{18, 0, 57}, + dictWord{5, 10, 371}, + dictWord{135, 10, 563}, + dictWord{135, 0, 1333}, + dictWord{6, 0, 107}, + dictWord{ + 7, + 0, + 638, + }, + dictWord{7, 0, 1632}, + dictWord{9, 0, 396}, + dictWord{134, 11, 610}, + dictWord{5, 0, 370}, + dictWord{134, 0, 1756}, + dictWord{4, 10, 374}, + dictWord{ + 7, + 10, + 547, + }, + dictWord{7, 10, 1700}, + dictWord{7, 10, 1833}, + dictWord{139, 10, 858}, + dictWord{133, 0, 204}, + dictWord{6, 0, 1305}, + dictWord{9, 10, 311}, + dictWord{ + 141, + 10, + 42, + }, + dictWord{5, 0, 970}, + dictWord{134, 0, 1706}, + dictWord{6, 10, 1647}, + dictWord{7, 10, 1552}, + dictWord{7, 10, 2010}, + dictWord{9, 10, 494}, + dictWord{137, 10, 509}, + dictWord{13, 11, 455}, + dictWord{15, 11, 99}, + dictWord{15, 11, 129}, + dictWord{144, 11, 68}, + dictWord{135, 0, 3}, + dictWord{4, 0, 35}, + dictWord{ + 5, + 0, + 121, + }, + dictWord{5, 0, 483}, + dictWord{5, 0, 685}, + dictWord{6, 0, 489}, + dictWord{6, 0, 782}, + dictWord{6, 0, 1032}, + dictWord{7, 0, 1204}, + dictWord{136, 0, 394}, + dictWord{4, 0, 921}, + dictWord{133, 0, 1007}, + dictWord{8, 11, 360}, + dictWord{138, 11, 63}, + dictWord{135, 0, 1696}, + dictWord{134, 0, 1519}, + dictWord{ + 132, + 11, + 443, + }, + dictWord{135, 11, 944}, + dictWord{6, 10, 123}, + dictWord{7, 10, 214}, + dictWord{9, 10, 728}, + dictWord{10, 10, 157}, + dictWord{11, 10, 346}, + dictWord{11, 10, 662}, + dictWord{143, 10, 106}, + dictWord{137, 0, 981}, + dictWord{135, 10, 1435}, + dictWord{134, 0, 1072}, + dictWord{132, 0, 712}, + dictWord{ + 134, + 0, + 1629, + }, + dictWord{134, 0, 728}, + dictWord{4, 11, 298}, + dictWord{137, 11, 483}, + dictWord{6, 0, 1177}, + dictWord{6, 0, 1271}, + dictWord{5, 11, 164}, + dictWord{ + 7, + 11, + 121, + }, + dictWord{142, 11, 189}, + dictWord{7, 0, 1608}, + dictWord{4, 10, 707}, + dictWord{5, 10, 588}, + dictWord{6, 10, 393}, + dictWord{13, 10, 106}, + dictWord{ + 18, + 10, + 49, + }, + dictWord{147, 10, 41}, + dictWord{23, 0, 16}, + dictWord{151, 11, 16}, + dictWord{6, 10, 211}, + dictWord{7, 10, 1690}, + dictWord{11, 10, 486}, + dictWord{140, 10, 369}, + dictWord{133, 0, 485}, + dictWord{19, 11, 15}, + dictWord{149, 11, 27}, + dictWord{4, 11, 172}, + dictWord{9, 11, 611}, + dictWord{10, 11, 436}, + dictWord{12, 11, 673}, + dictWord{141, 11, 255}, + dictWord{5, 11, 844}, + dictWord{10, 11, 484}, + dictWord{11, 11, 754}, + dictWord{12, 11, 457}, + dictWord{ + 14, + 11, + 171, + }, + dictWord{14, 11, 389}, + dictWord{146, 11, 153}, + dictWord{4, 0, 285}, + dictWord{5, 0, 27}, + dictWord{5, 0, 317}, + dictWord{6, 0, 301}, + dictWord{7, 0, 7}, + dictWord{ + 8, + 0, + 153, + }, + dictWord{10, 0, 766}, + dictWord{11, 0, 468}, + dictWord{12, 0, 467}, + dictWord{141, 0, 143}, + dictWord{134, 0, 1462}, + dictWord{9, 11, 263}, + dictWord{ + 10, + 11, + 147, + }, + dictWord{138, 11, 492}, + dictWord{133, 11, 537}, + dictWord{6, 0, 1945}, + dictWord{6, 0, 1986}, + dictWord{6, 0, 1991}, + dictWord{134, 0, 2038}, + dictWord{134, 10, 219}, + dictWord{137, 11, 842}, + dictWord{14, 0, 52}, + dictWord{17, 0, 50}, + dictWord{5, 10, 582}, + dictWord{6, 10, 1646}, + dictWord{7, 10, 99}, + dictWord{7, 10, 1962}, + dictWord{7, 10, 1986}, + dictWord{8, 10, 515}, + dictWord{8, 10, 773}, + dictWord{9, 10, 23}, + dictWord{9, 10, 491}, + dictWord{12, 10, 620}, + dictWord{142, 10, 93}, + dictWord{138, 11, 97}, + dictWord{20, 0, 21}, + dictWord{20, 0, 44}, + dictWord{133, 10, 851}, + dictWord{136, 0, 819}, + dictWord{139, 0, 917}, + dictWord{5, 11, 230}, + dictWord{5, 11, 392}, + dictWord{6, 11, 420}, + dictWord{8, 10, 762}, + dictWord{8, 10, 812}, + dictWord{9, 11, 568}, + dictWord{9, 10, 910}, + dictWord{140, 11, 612}, + dictWord{135, 0, 784}, + dictWord{15, 0, 135}, + dictWord{143, 11, 135}, + dictWord{10, 0, 454}, + dictWord{140, 0, 324}, + dictWord{4, 11, 0}, + dictWord{5, 11, 41}, + dictWord{7, 11, 1459}, + dictWord{7, 11, 1469}, + dictWord{7, 11, 1618}, + dictWord{7, 11, 1859}, + dictWord{9, 11, 549}, + dictWord{139, 11, 905}, + dictWord{4, 10, 98}, + dictWord{7, 10, 1365}, + dictWord{9, 10, 422}, + dictWord{9, 10, 670}, + dictWord{10, 10, 775}, + dictWord{11, 10, 210}, + dictWord{13, 10, 26}, + dictWord{13, 10, 457}, + dictWord{141, 10, 476}, + dictWord{6, 0, 1719}, + dictWord{6, 0, 1735}, + dictWord{7, 0, 2016}, + dictWord{7, 0, 2020}, + dictWord{8, 0, 837}, + dictWord{137, 0, 852}, + dictWord{133, 11, 696}, + dictWord{135, 0, 852}, + dictWord{132, 0, 952}, + dictWord{134, 10, 1730}, + dictWord{132, 11, 771}, + dictWord{ + 138, + 0, + 568, + }, + dictWord{137, 0, 448}, + dictWord{139, 0, 146}, + dictWord{8, 0, 67}, + dictWord{138, 0, 419}, + dictWord{133, 11, 921}, + dictWord{137, 10, 147}, + dictWord{134, 0, 1826}, + dictWord{10, 0, 657}, + dictWord{14, 0, 297}, + dictWord{142, 0, 361}, + dictWord{6, 0, 666}, + dictWord{6, 0, 767}, + dictWord{134, 0, 1542}, + dictWord{139, 0, 729}, + dictWord{6, 11, 180}, + dictWord{7, 11, 1137}, + dictWord{8, 11, 751}, + dictWord{139, 11, 805}, + dictWord{4, 11, 183}, + dictWord{7, 11, 271}, + dictWord{11, 11, 824}, + dictWord{11, 11, 952}, + dictWord{13, 11, 278}, + dictWord{13, 11, 339}, + dictWord{13, 11, 482}, + dictWord{14, 11, 424}, + dictWord{ + 148, + 11, + 99, + }, + dictWord{4, 0, 669}, + dictWord{5, 11, 477}, + dictWord{5, 11, 596}, + dictWord{6, 11, 505}, + dictWord{7, 11, 1221}, + dictWord{11, 11, 907}, + dictWord{ + 12, + 11, + 209, + }, + dictWord{141, 11, 214}, + dictWord{135, 11, 1215}, + dictWord{5, 0, 402}, + dictWord{6, 10, 30}, + dictWord{11, 10, 56}, + dictWord{139, 10, 305}, + dictWord{ + 7, + 11, + 564, + }, + dictWord{142, 11, 168}, + dictWord{139, 0, 152}, + dictWord{7, 0, 912}, + dictWord{135, 10, 1614}, + dictWord{4, 10, 150}, + dictWord{5, 10, 303}, + dictWord{134, 10, 327}, + dictWord{7, 0, 320}, + dictWord{8, 0, 51}, + dictWord{9, 0, 868}, + dictWord{10, 0, 833}, + dictWord{12, 0, 481}, + dictWord{12, 0, 570}, + dictWord{ + 148, + 0, + 106, + }, + dictWord{132, 0, 445}, + dictWord{7, 11, 274}, + dictWord{11, 11, 263}, + dictWord{11, 11, 479}, + dictWord{11, 11, 507}, + dictWord{140, 11, 277}, + dictWord{10, 0, 555}, + dictWord{11, 0, 308}, + dictWord{19, 0, 95}, + dictWord{6, 11, 1645}, + dictWord{8, 10, 192}, + dictWord{10, 10, 78}, + dictWord{141, 10, 359}, + dictWord{135, 10, 786}, + dictWord{6, 11, 92}, + dictWord{6, 11, 188}, + dictWord{7, 11, 1269}, + dictWord{7, 11, 1524}, + dictWord{7, 11, 1876}, + dictWord{10, 11, 228}, + dictWord{139, 11, 1020}, + dictWord{4, 11, 459}, + dictWord{133, 11, 966}, + dictWord{11, 0, 386}, + dictWord{6, 10, 1638}, + dictWord{7, 10, 79}, + dictWord{ + 7, + 10, + 496, + }, + dictWord{9, 10, 138}, + dictWord{10, 10, 336}, + dictWord{12, 10, 412}, + dictWord{12, 10, 440}, + dictWord{142, 10, 305}, + dictWord{133, 0, 239}, + dictWord{ + 7, + 0, + 83, + }, + dictWord{7, 0, 1990}, + dictWord{8, 0, 130}, + dictWord{139, 0, 720}, + dictWord{138, 11, 709}, + dictWord{4, 0, 143}, + dictWord{5, 0, 550}, + dictWord{ + 133, + 0, + 752, + }, + dictWord{5, 0, 123}, + dictWord{6, 0, 530}, + dictWord{7, 0, 348}, + dictWord{135, 0, 1419}, + dictWord{135, 0, 2024}, + dictWord{6, 11, 18}, + dictWord{7, 11, 179}, + dictWord{7, 11, 721}, + dictWord{7, 11, 932}, + dictWord{8, 11, 548}, + dictWord{8, 11, 757}, + dictWord{9, 11, 54}, + dictWord{9, 11, 65}, + dictWord{9, 11, 532}, + dictWord{ + 9, + 11, + 844, + }, + dictWord{10, 11, 113}, + dictWord{10, 11, 117}, + dictWord{10, 11, 236}, + dictWord{10, 11, 315}, + dictWord{10, 11, 430}, + dictWord{10, 11, 798}, + dictWord{11, 11, 153}, + dictWord{11, 11, 351}, + dictWord{11, 11, 375}, + dictWord{12, 11, 78}, + dictWord{12, 11, 151}, + dictWord{12, 11, 392}, + dictWord{ + 14, + 11, + 248, + }, + dictWord{143, 11, 23}, + dictWord{7, 10, 204}, + dictWord{7, 10, 415}, + dictWord{8, 10, 42}, + dictWord{10, 10, 85}, + dictWord{139, 10, 564}, + dictWord{ + 134, + 0, + 958, + }, + dictWord{133, 11, 965}, + dictWord{132, 0, 210}, + dictWord{135, 11, 1429}, + dictWord{138, 11, 480}, + dictWord{134, 11, 182}, + dictWord{ + 139, + 11, + 345, + }, + dictWord{10, 11, 65}, + dictWord{10, 11, 488}, + dictWord{138, 11, 497}, + dictWord{4, 10, 3}, + dictWord{5, 10, 247}, + dictWord{5, 10, 644}, + dictWord{ + 7, + 10, + 744, + }, + dictWord{7, 10, 1207}, + dictWord{7, 10, 1225}, + dictWord{7, 10, 1909}, + dictWord{146, 10, 147}, + dictWord{132, 0, 430}, + dictWord{5, 10, 285}, + dictWord{ + 9, + 10, + 67, + }, + dictWord{13, 10, 473}, + dictWord{143, 10, 82}, + dictWord{144, 11, 16}, + dictWord{7, 11, 1162}, + dictWord{9, 11, 588}, + dictWord{10, 11, 260}, + dictWord{151, 10, 8}, + dictWord{133, 0, 213}, + dictWord{138, 0, 7}, + dictWord{135, 0, 801}, + dictWord{134, 11, 1786}, + dictWord{135, 11, 308}, + dictWord{6, 0, 936}, + dictWord{134, 0, 1289}, + dictWord{133, 0, 108}, + dictWord{132, 0, 885}, + dictWord{133, 0, 219}, + dictWord{139, 0, 587}, + dictWord{4, 0, 193}, + dictWord{5, 0, 916}, + dictWord{6, 0, 1041}, + dictWord{7, 0, 364}, + dictWord{10, 0, 398}, + dictWord{10, 0, 726}, + dictWord{11, 0, 317}, + dictWord{11, 0, 626}, + dictWord{12, 0, 142}, + dictWord{12, 0, 288}, + dictWord{12, 0, 678}, + dictWord{13, 0, 313}, + dictWord{15, 0, 113}, + dictWord{146, 0, 114}, + dictWord{135, 0, 1165}, + dictWord{6, 0, 241}, + dictWord{ + 9, + 0, + 342, + }, + dictWord{10, 0, 729}, + dictWord{11, 0, 284}, + dictWord{11, 0, 445}, + dictWord{11, 0, 651}, + dictWord{11, 0, 863}, + dictWord{13, 0, 398}, + dictWord{ + 146, + 0, + 99, + }, + dictWord{7, 0, 907}, + dictWord{136, 0, 832}, + dictWord{9, 0, 303}, + dictWord{4, 10, 29}, + dictWord{6, 10, 532}, + dictWord{7, 10, 1628}, + dictWord{7, 10, 1648}, + dictWord{9, 10, 350}, + dictWord{10, 10, 433}, + dictWord{11, 10, 97}, + dictWord{11, 10, 557}, + dictWord{11, 10, 745}, + dictWord{12, 10, 289}, + dictWord{ + 12, + 10, + 335, + }, + dictWord{12, 10, 348}, + dictWord{12, 10, 606}, + dictWord{13, 10, 116}, + dictWord{13, 10, 233}, + dictWord{13, 10, 466}, + dictWord{14, 10, 181}, + dictWord{ + 14, + 10, + 209, + }, + dictWord{14, 10, 232}, + dictWord{14, 10, 236}, + dictWord{14, 10, 300}, + dictWord{16, 10, 41}, + dictWord{148, 10, 97}, + dictWord{7, 11, 423}, + dictWord{7, 10, 1692}, + dictWord{136, 11, 588}, + dictWord{6, 0, 931}, + dictWord{134, 0, 1454}, + dictWord{5, 10, 501}, + dictWord{7, 10, 1704}, + dictWord{9, 10, 553}, + dictWord{11, 10, 520}, + dictWord{12, 10, 557}, + dictWord{141, 10, 249}, + dictWord{136, 11, 287}, + dictWord{4, 0, 562}, + dictWord{9, 0, 254}, + dictWord{ + 139, + 0, + 879, + }, + dictWord{132, 0, 786}, + dictWord{14, 11, 32}, + dictWord{18, 11, 85}, + dictWord{20, 11, 2}, + dictWord{152, 11, 16}, + dictWord{135, 0, 1294}, + dictWord{ + 7, + 11, + 723, + }, + dictWord{135, 11, 1135}, + dictWord{6, 0, 216}, + dictWord{7, 0, 901}, + dictWord{7, 0, 1343}, + dictWord{8, 0, 493}, + dictWord{134, 11, 403}, + dictWord{ + 7, + 11, + 719, + }, + dictWord{8, 11, 809}, + dictWord{136, 11, 834}, + dictWord{5, 11, 210}, + dictWord{6, 11, 213}, + dictWord{7, 11, 60}, + dictWord{10, 11, 364}, + dictWord{ + 139, + 11, + 135, + }, + dictWord{7, 0, 341}, + dictWord{11, 0, 219}, + dictWord{5, 11, 607}, + dictWord{8, 11, 326}, + dictWord{136, 11, 490}, + dictWord{4, 11, 701}, + dictWord{ + 5, + 11, + 472, + }, + dictWord{5, 11, 639}, + dictWord{7, 11, 1249}, + dictWord{9, 11, 758}, + dictWord{139, 11, 896}, + dictWord{135, 11, 380}, + dictWord{135, 11, 1947}, + dictWord{139, 0, 130}, + dictWord{135, 0, 1734}, + dictWord{10, 0, 115}, + dictWord{11, 0, 420}, + dictWord{12, 0, 154}, + dictWord{13, 0, 404}, + dictWord{14, 0, 346}, + dictWord{143, 0, 54}, + dictWord{134, 10, 129}, + dictWord{4, 11, 386}, + dictWord{7, 11, 41}, + dictWord{8, 11, 405}, + dictWord{9, 11, 497}, + dictWord{11, 11, 110}, + dictWord{11, 11, 360}, + dictWord{15, 11, 37}, + dictWord{144, 11, 84}, + dictWord{141, 11, 282}, + dictWord{5, 11, 46}, + dictWord{7, 11, 1452}, + dictWord{7, 11, 1480}, + dictWord{8, 11, 634}, + dictWord{140, 11, 472}, + dictWord{4, 11, 524}, + dictWord{136, 11, 810}, + dictWord{10, 11, 238}, + dictWord{141, 11, 33}, + dictWord{ + 133, + 0, + 604, + }, + dictWord{5, 0, 1011}, + dictWord{136, 0, 701}, + dictWord{8, 0, 856}, + dictWord{8, 0, 858}, + dictWord{8, 0, 879}, + dictWord{12, 0, 702}, + dictWord{142, 0, 447}, + dictWord{4, 0, 54}, + dictWord{5, 0, 666}, + dictWord{7, 0, 1039}, + dictWord{7, 0, 1130}, + dictWord{9, 0, 195}, + dictWord{138, 0, 302}, + dictWord{4, 10, 25}, + dictWord{ + 5, + 10, + 60, + }, + dictWord{6, 10, 504}, + dictWord{7, 10, 614}, + dictWord{7, 10, 1155}, + dictWord{140, 10, 0}, + dictWord{7, 10, 1248}, + dictWord{11, 10, 621}, + dictWord{ + 139, + 10, + 702, + }, + dictWord{133, 11, 997}, + dictWord{137, 10, 321}, + dictWord{134, 0, 1669}, + dictWord{134, 0, 1791}, + dictWord{4, 10, 379}, + dictWord{ + 135, + 10, + 1397, + }, + dictWord{138, 11, 372}, + dictWord{5, 11, 782}, + dictWord{5, 11, 829}, + dictWord{134, 11, 1738}, + dictWord{135, 0, 1228}, + dictWord{4, 10, 118}, + dictWord{6, 10, 274}, + dictWord{6, 10, 361}, + dictWord{7, 10, 75}, + dictWord{141, 10, 441}, + dictWord{132, 0, 623}, + dictWord{9, 11, 279}, + dictWord{10, 11, 407}, + dictWord{14, 11, 84}, + dictWord{150, 11, 18}, + dictWord{137, 10, 841}, + dictWord{135, 0, 798}, + dictWord{140, 10, 693}, + dictWord{5, 10, 314}, + dictWord{6, 10, 221}, + dictWord{7, 10, 419}, + dictWord{10, 10, 650}, + dictWord{11, 10, 396}, + dictWord{12, 10, 156}, + dictWord{13, 10, 369}, + dictWord{14, 10, 333}, + dictWord{ + 145, + 10, + 47, + }, + dictWord{135, 11, 1372}, + dictWord{7, 0, 122}, + dictWord{9, 0, 259}, + dictWord{10, 0, 84}, + dictWord{11, 0, 470}, + dictWord{12, 0, 541}, + dictWord{ + 141, + 0, + 379, + }, + dictWord{134, 0, 837}, + dictWord{8, 0, 1013}, + dictWord{4, 11, 78}, + dictWord{5, 11, 96}, + dictWord{5, 11, 182}, + dictWord{7, 11, 1724}, + dictWord{ + 7, + 11, + 1825, + }, + dictWord{10, 11, 394}, + dictWord{10, 11, 471}, + dictWord{11, 11, 532}, + dictWord{14, 11, 340}, + dictWord{145, 11, 88}, + dictWord{134, 0, 577}, + dictWord{135, 11, 1964}, + dictWord{132, 10, 913}, + dictWord{134, 0, 460}, + dictWord{8, 0, 891}, + dictWord{10, 0, 901}, + dictWord{10, 0, 919}, + dictWord{10, 0, 932}, + dictWord{12, 0, 715}, + dictWord{12, 0, 728}, + dictWord{12, 0, 777}, + dictWord{14, 0, 457}, + dictWord{144, 0, 103}, + dictWord{5, 0, 82}, + dictWord{5, 0, 131}, + dictWord{ + 7, + 0, + 1755, + }, + dictWord{8, 0, 31}, + dictWord{9, 0, 168}, + dictWord{9, 0, 764}, + dictWord{139, 0, 869}, + dictWord{136, 10, 475}, + dictWord{6, 0, 605}, + dictWord{ + 5, + 10, + 1016, + }, + dictWord{9, 11, 601}, + dictWord{9, 11, 619}, + dictWord{10, 11, 505}, + dictWord{10, 11, 732}, + dictWord{11, 11, 355}, + dictWord{140, 11, 139}, + dictWord{ + 7, + 10, + 602, + }, + dictWord{8, 10, 179}, + dictWord{10, 10, 781}, + dictWord{140, 10, 126}, + dictWord{134, 0, 1246}, + dictWord{6, 10, 329}, + dictWord{138, 10, 111}, + dictWord{6, 11, 215}, + dictWord{7, 11, 1028}, + dictWord{7, 11, 1473}, + dictWord{7, 11, 1721}, + dictWord{9, 11, 424}, + dictWord{138, 11, 779}, + dictWord{5, 0, 278}, + dictWord{137, 0, 68}, + dictWord{6, 0, 932}, + dictWord{6, 0, 1084}, + dictWord{144, 0, 86}, + dictWord{4, 0, 163}, + dictWord{5, 0, 201}, + dictWord{5, 0, 307}, + dictWord{ + 5, + 0, + 310, + }, + dictWord{6, 0, 335}, + dictWord{7, 0, 284}, + dictWord{7, 0, 1660}, + dictWord{136, 0, 165}, + dictWord{136, 0, 781}, + dictWord{134, 0, 707}, + dictWord{6, 0, 33}, + dictWord{135, 0, 1244}, + dictWord{5, 10, 821}, + dictWord{6, 11, 67}, + dictWord{6, 10, 1687}, + dictWord{7, 11, 258}, + dictWord{7, 11, 1630}, + dictWord{9, 11, 354}, + dictWord{9, 11, 675}, + dictWord{10, 11, 830}, + dictWord{14, 11, 80}, + dictWord{145, 11, 80}, + dictWord{6, 11, 141}, + dictWord{7, 11, 225}, + dictWord{9, 11, 59}, + dictWord{9, 11, 607}, + dictWord{10, 11, 312}, + dictWord{11, 11, 687}, + dictWord{12, 11, 555}, + dictWord{13, 11, 373}, + dictWord{13, 11, 494}, + dictWord{148, 11, 58}, + dictWord{134, 0, 1113}, + dictWord{9, 0, 388}, + dictWord{5, 10, 71}, + dictWord{7, 10, 1407}, + dictWord{9, 10, 704}, + dictWord{10, 10, 261}, + dictWord{10, 10, 619}, + dictWord{11, 10, 547}, + dictWord{11, 10, 619}, + dictWord{143, 10, 157}, + dictWord{7, 0, 1953}, + dictWord{136, 0, 720}, + dictWord{138, 0, 203}, + dictWord{ + 7, + 10, + 2008, + }, + dictWord{9, 10, 337}, + dictWord{138, 10, 517}, + dictWord{6, 0, 326}, + dictWord{7, 0, 677}, + dictWord{137, 0, 425}, + dictWord{139, 11, 81}, + dictWord{ + 7, + 0, + 1316, + }, + dictWord{7, 0, 1412}, + dictWord{7, 0, 1839}, + dictWord{9, 0, 589}, + dictWord{11, 0, 241}, + dictWord{11, 0, 676}, + dictWord{11, 0, 811}, + dictWord{11, 0, 891}, + dictWord{12, 0, 140}, + dictWord{12, 0, 346}, + dictWord{12, 0, 479}, + dictWord{13, 0, 140}, + dictWord{13, 0, 381}, + dictWord{14, 0, 188}, + dictWord{18, 0, 30}, + dictWord{148, 0, 108}, + dictWord{5, 0, 416}, + dictWord{6, 10, 86}, + dictWord{6, 10, 603}, + dictWord{7, 10, 292}, + dictWord{7, 10, 561}, + dictWord{8, 10, 257}, + dictWord{ + 8, + 10, + 382, + }, + dictWord{9, 10, 721}, + dictWord{9, 10, 778}, + dictWord{11, 10, 581}, + dictWord{140, 10, 466}, + dictWord{4, 10, 486}, + dictWord{133, 10, 491}, + dictWord{134, 0, 1300}, + dictWord{132, 10, 72}, + dictWord{7, 0, 847}, + dictWord{6, 10, 265}, + dictWord{7, 11, 430}, + dictWord{139, 11, 46}, + dictWord{5, 11, 602}, + dictWord{6, 11, 106}, + dictWord{7, 11, 1786}, + dictWord{7, 11, 1821}, + dictWord{7, 11, 2018}, + dictWord{9, 11, 418}, + dictWord{137, 11, 763}, + dictWord{5, 0, 358}, + dictWord{7, 0, 535}, + dictWord{7, 0, 1184}, + dictWord{10, 0, 662}, + dictWord{13, 0, 212}, + dictWord{13, 0, 304}, + dictWord{13, 0, 333}, + dictWord{145, 0, 98}, + dictWord{ + 5, + 11, + 65, + }, + dictWord{6, 11, 416}, + dictWord{7, 11, 1720}, + dictWord{7, 11, 1924}, + dictWord{8, 11, 677}, + dictWord{10, 11, 109}, + dictWord{11, 11, 14}, + dictWord{ + 11, + 11, + 70, + }, + dictWord{11, 11, 569}, + dictWord{11, 11, 735}, + dictWord{15, 11, 153}, + dictWord{148, 11, 80}, + dictWord{6, 0, 1823}, + dictWord{8, 0, 839}, + dictWord{ + 8, + 0, + 852, + }, + dictWord{8, 0, 903}, + dictWord{10, 0, 940}, + dictWord{12, 0, 707}, + dictWord{140, 0, 775}, + dictWord{135, 11, 1229}, + dictWord{6, 0, 1522}, + dictWord{ + 140, + 0, + 654, + }, + dictWord{136, 11, 595}, + dictWord{139, 0, 163}, + dictWord{141, 0, 314}, + dictWord{132, 0, 978}, + dictWord{4, 0, 601}, + dictWord{6, 0, 2035}, + dictWord{137, 10, 234}, + dictWord{5, 10, 815}, + dictWord{6, 10, 1688}, + dictWord{134, 10, 1755}, + dictWord{133, 0, 946}, + dictWord{136, 0, 434}, + dictWord{ + 6, + 10, + 197, + }, + dictWord{136, 10, 205}, + dictWord{7, 0, 411}, + dictWord{7, 0, 590}, + dictWord{8, 0, 631}, + dictWord{9, 0, 323}, + dictWord{10, 0, 355}, + dictWord{11, 0, 491}, + dictWord{12, 0, 143}, + dictWord{12, 0, 402}, + dictWord{13, 0, 73}, + dictWord{14, 0, 408}, + dictWord{15, 0, 107}, + dictWord{146, 0, 71}, + dictWord{7, 0, 1467}, + dictWord{ + 8, + 0, + 328, + }, + dictWord{10, 0, 544}, + dictWord{11, 0, 955}, + dictWord{12, 0, 13}, + dictWord{13, 0, 320}, + dictWord{145, 0, 83}, + dictWord{142, 0, 410}, + dictWord{ + 11, + 0, + 511, + }, + dictWord{13, 0, 394}, + dictWord{14, 0, 298}, + dictWord{14, 0, 318}, + dictWord{146, 0, 103}, + dictWord{6, 10, 452}, + dictWord{7, 10, 312}, + dictWord{ + 138, + 10, + 219, + }, + dictWord{138, 10, 589}, + dictWord{4, 10, 333}, + dictWord{9, 10, 176}, + dictWord{12, 10, 353}, + dictWord{141, 10, 187}, + dictWord{135, 11, 329}, + dictWord{132, 11, 469}, + dictWord{5, 0, 835}, + dictWord{134, 0, 483}, + dictWord{134, 11, 1743}, + dictWord{5, 11, 929}, + dictWord{6, 11, 340}, + dictWord{8, 11, 376}, + dictWord{136, 11, 807}, + dictWord{134, 10, 1685}, + dictWord{132, 0, 677}, + dictWord{5, 11, 218}, + dictWord{7, 11, 1610}, + dictWord{138, 11, 83}, + dictWord{ + 5, + 11, + 571, + }, + dictWord{135, 11, 1842}, + dictWord{132, 11, 455}, + dictWord{137, 0, 70}, + dictWord{135, 0, 1405}, + dictWord{7, 10, 135}, + dictWord{8, 10, 7}, + dictWord{ + 8, + 10, + 62, + }, + dictWord{9, 10, 243}, + dictWord{10, 10, 658}, + dictWord{10, 10, 697}, + dictWord{11, 10, 456}, + dictWord{139, 10, 756}, + dictWord{9, 10, 395}, + dictWord{138, 10, 79}, + dictWord{137, 0, 108}, + dictWord{6, 11, 161}, + dictWord{7, 11, 372}, + dictWord{137, 11, 597}, + dictWord{132, 11, 349}, + dictWord{ + 132, + 0, + 777, + }, + dictWord{132, 0, 331}, + dictWord{135, 10, 631}, + dictWord{133, 0, 747}, + dictWord{6, 11, 432}, + dictWord{6, 11, 608}, + dictWord{139, 11, 322}, + dictWord{138, 10, 835}, + dictWord{5, 11, 468}, + dictWord{7, 11, 1809}, + dictWord{10, 11, 325}, + dictWord{11, 11, 856}, + dictWord{12, 11, 345}, + dictWord{ + 143, + 11, + 104, + }, + dictWord{133, 11, 223}, + dictWord{7, 10, 406}, + dictWord{7, 10, 459}, + dictWord{8, 10, 606}, + dictWord{139, 10, 726}, + dictWord{132, 11, 566}, + dictWord{142, 0, 68}, + dictWord{4, 11, 59}, + dictWord{135, 11, 1394}, + dictWord{6, 11, 436}, + dictWord{139, 11, 481}, + dictWord{4, 11, 48}, + dictWord{5, 11, 271}, + dictWord{135, 11, 953}, + dictWord{139, 11, 170}, + dictWord{5, 11, 610}, + dictWord{136, 11, 457}, + dictWord{133, 11, 755}, + dictWord{135, 11, 1217}, + dictWord{ + 133, + 10, + 612, + }, + dictWord{132, 11, 197}, + dictWord{132, 0, 505}, + dictWord{4, 10, 372}, + dictWord{7, 10, 482}, + dictWord{8, 10, 158}, + dictWord{9, 10, 602}, + dictWord{ + 9, + 10, + 615, + }, + dictWord{10, 10, 245}, + dictWord{10, 10, 678}, + dictWord{10, 10, 744}, + dictWord{11, 10, 248}, + dictWord{139, 10, 806}, + dictWord{133, 0, 326}, + dictWord{5, 10, 854}, + dictWord{135, 10, 1991}, + dictWord{4, 0, 691}, + dictWord{146, 0, 16}, + dictWord{6, 0, 628}, + dictWord{9, 0, 35}, + dictWord{10, 0, 680}, + dictWord{10, 0, 793}, + dictWord{11, 0, 364}, + dictWord{13, 0, 357}, + dictWord{143, 0, 164}, + dictWord{138, 0, 654}, + dictWord{6, 0, 32}, + dictWord{7, 0, 385}, + dictWord{ + 7, + 0, + 757, + }, + dictWord{7, 0, 1916}, + dictWord{8, 0, 37}, + dictWord{8, 0, 94}, + dictWord{8, 0, 711}, + dictWord{9, 0, 541}, + dictWord{10, 0, 162}, + dictWord{10, 0, 795}, + dictWord{ + 11, + 0, + 989, + }, + dictWord{11, 0, 1010}, + dictWord{12, 0, 14}, + dictWord{142, 0, 308}, + dictWord{133, 11, 217}, + dictWord{6, 0, 152}, + dictWord{6, 0, 349}, + dictWord{ + 6, + 0, + 1682, + }, + dictWord{7, 0, 1252}, + dictWord{8, 0, 112}, + dictWord{9, 0, 435}, + dictWord{9, 0, 668}, + dictWord{10, 0, 290}, + dictWord{10, 0, 319}, + dictWord{10, 0, 815}, + dictWord{11, 0, 180}, + dictWord{11, 0, 837}, + dictWord{12, 0, 240}, + dictWord{13, 0, 152}, + dictWord{13, 0, 219}, + dictWord{142, 0, 158}, + dictWord{4, 0, 581}, + dictWord{134, 0, 726}, + dictWord{5, 10, 195}, + dictWord{135, 10, 1685}, + dictWord{6, 0, 126}, + dictWord{7, 0, 573}, + dictWord{8, 0, 397}, + dictWord{142, 0, 44}, + dictWord{138, 0, 89}, + dictWord{7, 10, 1997}, + dictWord{8, 10, 730}, + dictWord{139, 10, 1006}, + dictWord{134, 0, 1531}, + dictWord{134, 0, 1167}, + dictWord{ + 5, + 0, + 926, + }, + dictWord{12, 0, 203}, + dictWord{133, 10, 751}, + dictWord{4, 11, 165}, + dictWord{7, 11, 1398}, + dictWord{135, 11, 1829}, + dictWord{7, 0, 1232}, + dictWord{137, 0, 531}, + dictWord{135, 10, 821}, + dictWord{134, 0, 943}, + dictWord{133, 0, 670}, + dictWord{4, 0, 880}, + dictWord{139, 0, 231}, + dictWord{ + 134, + 0, + 1617, + }, + dictWord{135, 0, 1957}, + dictWord{5, 11, 9}, + dictWord{7, 11, 297}, + dictWord{7, 11, 966}, + dictWord{140, 11, 306}, + dictWord{6, 0, 975}, + dictWord{ + 134, + 0, + 985, + }, + dictWord{5, 10, 950}, + dictWord{5, 10, 994}, + dictWord{134, 10, 351}, + dictWord{12, 11, 21}, + dictWord{151, 11, 7}, + dictWord{5, 11, 146}, + dictWord{ + 6, + 11, + 411, + }, + dictWord{138, 11, 721}, + dictWord{7, 0, 242}, + dictWord{135, 0, 1942}, + dictWord{6, 11, 177}, + dictWord{135, 11, 467}, + dictWord{5, 0, 421}, + dictWord{ + 7, + 10, + 47, + }, + dictWord{137, 10, 684}, + dictWord{5, 0, 834}, + dictWord{7, 0, 1202}, + dictWord{8, 0, 14}, + dictWord{9, 0, 481}, + dictWord{137, 0, 880}, + dictWord{138, 0, 465}, + dictWord{6, 0, 688}, + dictWord{9, 0, 834}, + dictWord{132, 10, 350}, + dictWord{132, 0, 855}, + dictWord{4, 0, 357}, + dictWord{6, 0, 172}, + dictWord{7, 0, 143}, + dictWord{137, 0, 413}, + dictWord{133, 11, 200}, + dictWord{132, 0, 590}, + dictWord{7, 10, 1812}, + dictWord{13, 10, 259}, + dictWord{13, 10, 356}, + dictWord{ + 14, + 10, + 242, + }, + dictWord{147, 10, 114}, + dictWord{133, 10, 967}, + dictWord{11, 0, 114}, + dictWord{4, 10, 473}, + dictWord{7, 10, 623}, + dictWord{8, 10, 808}, + dictWord{ + 9, + 10, + 871, + }, + dictWord{9, 10, 893}, + dictWord{11, 10, 431}, + dictWord{12, 10, 112}, + dictWord{12, 10, 217}, + dictWord{12, 10, 243}, + dictWord{12, 10, 562}, + dictWord{ + 12, + 10, + 663, + }, + dictWord{12, 10, 683}, + dictWord{13, 10, 141}, + dictWord{13, 10, 197}, + dictWord{13, 10, 227}, + dictWord{13, 10, 406}, + dictWord{13, 10, 487}, + dictWord{14, 10, 156}, + dictWord{14, 10, 203}, + dictWord{14, 10, 224}, + dictWord{14, 10, 256}, + dictWord{18, 10, 58}, + dictWord{150, 10, 0}, + dictWord{ + 138, + 10, + 286, + }, + dictWord{4, 10, 222}, + dictWord{7, 10, 286}, + dictWord{136, 10, 629}, + dictWord{5, 0, 169}, + dictWord{7, 0, 333}, + dictWord{136, 0, 45}, + dictWord{ + 134, + 11, + 481, + }, + dictWord{132, 0, 198}, + dictWord{4, 0, 24}, + dictWord{5, 0, 140}, + dictWord{5, 0, 185}, + dictWord{7, 0, 1500}, + dictWord{11, 0, 565}, + dictWord{11, 0, 838}, + dictWord{4, 11, 84}, + dictWord{7, 11, 1482}, + dictWord{10, 11, 76}, + dictWord{138, 11, 142}, + dictWord{133, 0, 585}, + dictWord{141, 10, 306}, + dictWord{ + 133, + 11, + 1015, + }, + dictWord{4, 11, 315}, + dictWord{5, 11, 507}, + dictWord{135, 11, 1370}, + dictWord{136, 10, 146}, + dictWord{6, 0, 691}, + dictWord{134, 0, 1503}, + dictWord{ + 4, + 0, + 334, + }, + dictWord{133, 0, 593}, + dictWord{4, 10, 465}, + dictWord{135, 10, 1663}, + dictWord{142, 11, 173}, + dictWord{135, 0, 913}, + dictWord{12, 0, 116}, + dictWord{134, 11, 1722}, + dictWord{134, 0, 1360}, + dictWord{132, 0, 802}, + dictWord{8, 11, 222}, + dictWord{8, 11, 476}, + dictWord{9, 11, 238}, + dictWord{ + 11, + 11, + 516, + }, + dictWord{11, 11, 575}, + dictWord{15, 11, 109}, + dictWord{146, 11, 100}, + dictWord{6, 0, 308}, + dictWord{9, 0, 673}, + dictWord{7, 10, 138}, + dictWord{ + 7, + 10, + 517, + }, + dictWord{139, 10, 238}, + dictWord{132, 0, 709}, + dictWord{6, 0, 1876}, + dictWord{6, 0, 1895}, + dictWord{9, 0, 994}, + dictWord{9, 0, 1006}, + dictWord{ + 12, + 0, + 829, + }, + dictWord{12, 0, 888}, + dictWord{12, 0, 891}, + dictWord{146, 0, 185}, + dictWord{148, 10, 94}, + dictWord{4, 0, 228}, + dictWord{133, 0, 897}, + dictWord{ + 7, + 0, + 1840, + }, + dictWord{5, 10, 495}, + dictWord{7, 10, 834}, + dictWord{9, 10, 733}, + dictWord{139, 10, 378}, + dictWord{133, 10, 559}, + dictWord{6, 10, 21}, + dictWord{ + 6, + 10, + 1737, + }, + dictWord{7, 10, 1444}, + dictWord{136, 10, 224}, + dictWord{4, 0, 608}, + dictWord{133, 0, 497}, + dictWord{6, 11, 40}, + dictWord{135, 11, 1781}, + dictWord{134, 0, 1573}, + dictWord{135, 0, 2039}, + dictWord{6, 0, 540}, + dictWord{136, 0, 136}, + dictWord{4, 0, 897}, + dictWord{5, 0, 786}, + dictWord{133, 10, 519}, + dictWord{6, 0, 1878}, + dictWord{6, 0, 1884}, + dictWord{9, 0, 938}, + dictWord{9, 0, 948}, + dictWord{9, 0, 955}, + dictWord{9, 0, 973}, + dictWord{9, 0, 1012}, + dictWord{ + 12, + 0, + 895, + }, + dictWord{12, 0, 927}, + dictWord{143, 0, 254}, + dictWord{134, 0, 1469}, + dictWord{133, 0, 999}, + dictWord{4, 0, 299}, + dictWord{135, 0, 1004}, + dictWord{ + 4, + 0, + 745, + }, + dictWord{133, 0, 578}, + dictWord{136, 11, 574}, + dictWord{133, 0, 456}, + dictWord{134, 0, 1457}, + dictWord{7, 0, 1679}, + dictWord{132, 10, 402}, + dictWord{7, 0, 693}, + dictWord{8, 0, 180}, + dictWord{12, 0, 163}, + dictWord{8, 10, 323}, + dictWord{136, 10, 479}, + dictWord{11, 10, 580}, + dictWord{142, 10, 201}, + dictWord{5, 10, 59}, + dictWord{135, 10, 672}, + dictWord{132, 11, 354}, + dictWord{146, 10, 34}, + dictWord{4, 0, 755}, + dictWord{135, 11, 1558}, + dictWord{ + 7, + 0, + 1740, + }, + dictWord{146, 0, 48}, + dictWord{4, 10, 85}, + dictWord{135, 10, 549}, + dictWord{139, 0, 338}, + dictWord{133, 10, 94}, + dictWord{134, 0, 1091}, + dictWord{135, 11, 469}, + dictWord{12, 0, 695}, + dictWord{12, 0, 704}, + dictWord{20, 0, 113}, + dictWord{5, 11, 830}, + dictWord{14, 11, 338}, + dictWord{148, 11, 81}, + dictWord{135, 0, 1464}, + dictWord{6, 10, 11}, + dictWord{135, 10, 187}, + dictWord{135, 0, 975}, + dictWord{13, 0, 335}, + dictWord{132, 10, 522}, + dictWord{ + 134, + 0, + 1979, + }, + dictWord{5, 11, 496}, + dictWord{135, 11, 203}, + dictWord{4, 10, 52}, + dictWord{135, 10, 661}, + dictWord{7, 0, 1566}, + dictWord{8, 0, 269}, + dictWord{ + 9, + 0, + 212, + }, + dictWord{9, 0, 718}, + dictWord{14, 0, 15}, + dictWord{14, 0, 132}, + dictWord{142, 0, 227}, + dictWord{4, 0, 890}, + dictWord{5, 0, 805}, + dictWord{5, 0, 819}, + dictWord{ + 5, + 0, + 961, + }, + dictWord{6, 0, 396}, + dictWord{6, 0, 1631}, + dictWord{6, 0, 1678}, + dictWord{7, 0, 1967}, + dictWord{7, 0, 2041}, + dictWord{9, 0, 630}, + dictWord{11, 0, 8}, + dictWord{11, 0, 1019}, + dictWord{12, 0, 176}, + dictWord{13, 0, 225}, + dictWord{14, 0, 292}, + dictWord{21, 0, 24}, + dictWord{4, 10, 383}, + dictWord{133, 10, 520}, + dictWord{134, 11, 547}, + dictWord{135, 11, 1748}, + dictWord{5, 11, 88}, + dictWord{137, 11, 239}, + dictWord{146, 11, 128}, + dictWord{7, 11, 650}, + dictWord{ + 135, + 11, + 1310, + }, + dictWord{4, 10, 281}, + dictWord{5, 10, 38}, + dictWord{7, 10, 194}, + dictWord{7, 10, 668}, + dictWord{7, 10, 1893}, + dictWord{137, 10, 397}, + dictWord{135, 0, 1815}, + dictWord{9, 10, 635}, + dictWord{139, 10, 559}, + dictWord{7, 0, 1505}, + dictWord{10, 0, 190}, + dictWord{10, 0, 634}, + dictWord{11, 0, 792}, + dictWord{12, 0, 358}, + dictWord{140, 0, 447}, + dictWord{5, 0, 0}, + dictWord{6, 0, 536}, + dictWord{7, 0, 604}, + dictWord{13, 0, 445}, + dictWord{145, 0, 126}, + dictWord{ + 7, + 11, + 1076, + }, + dictWord{9, 11, 80}, + dictWord{11, 11, 78}, + dictWord{11, 11, 421}, + dictWord{11, 11, 534}, + dictWord{140, 11, 545}, + dictWord{8, 0, 966}, + dictWord{ + 10, + 0, + 1023, + }, + dictWord{14, 11, 369}, + dictWord{146, 11, 72}, + dictWord{135, 11, 1641}, + dictWord{6, 0, 232}, + dictWord{6, 0, 412}, + dictWord{7, 0, 1074}, + dictWord{ + 8, + 0, + 9, + }, + dictWord{8, 0, 157}, + dictWord{8, 0, 786}, + dictWord{9, 0, 196}, + dictWord{9, 0, 352}, + dictWord{9, 0, 457}, + dictWord{10, 0, 337}, + dictWord{11, 0, 232}, + dictWord{ + 11, + 0, + 877, + }, + dictWord{12, 0, 480}, + dictWord{140, 0, 546}, + dictWord{135, 0, 958}, + dictWord{4, 0, 382}, + dictWord{136, 0, 579}, + dictWord{4, 0, 212}, + dictWord{ + 135, + 0, + 1206, + }, + dictWord{4, 11, 497}, + dictWord{5, 11, 657}, + dictWord{135, 11, 1584}, + dictWord{132, 0, 681}, + dictWord{8, 0, 971}, + dictWord{138, 0, 965}, + dictWord{ + 5, + 10, + 448, + }, + dictWord{136, 10, 535}, + dictWord{14, 0, 16}, + dictWord{146, 0, 44}, + dictWord{11, 0, 584}, + dictWord{11, 0, 616}, + dictWord{14, 0, 275}, + dictWord{ + 11, + 11, + 584, + }, + dictWord{11, 11, 616}, + dictWord{142, 11, 275}, + dictWord{136, 11, 13}, + dictWord{7, 10, 610}, + dictWord{135, 10, 1501}, + dictWord{7, 11, 642}, + dictWord{8, 11, 250}, + dictWord{11, 11, 123}, + dictWord{11, 11, 137}, + dictWord{13, 11, 48}, + dictWord{142, 11, 95}, + dictWord{133, 0, 655}, + dictWord{17, 0, 67}, + dictWord{147, 0, 74}, + dictWord{134, 0, 751}, + dictWord{134, 0, 1967}, + dictWord{6, 0, 231}, + dictWord{136, 0, 423}, + dictWord{5, 0, 300}, + dictWord{138, 0, 1016}, + dictWord{4, 10, 319}, + dictWord{5, 10, 699}, + dictWord{138, 10, 673}, + dictWord{6, 0, 237}, + dictWord{7, 0, 611}, + dictWord{8, 0, 100}, + dictWord{9, 0, 416}, + dictWord{ + 11, + 0, + 335, + }, + dictWord{12, 0, 173}, + dictWord{18, 0, 101}, + dictWord{6, 10, 336}, + dictWord{8, 10, 552}, + dictWord{9, 10, 285}, + dictWord{10, 10, 99}, + dictWord{ + 139, + 10, + 568, + }, + dictWord{134, 0, 1370}, + dictWord{7, 10, 1406}, + dictWord{9, 10, 218}, + dictWord{141, 10, 222}, + dictWord{133, 10, 256}, + dictWord{ + 135, + 0, + 1208, + }, + dictWord{14, 11, 213}, + dictWord{148, 11, 38}, + dictWord{6, 0, 1219}, + dictWord{135, 11, 1642}, + dictWord{13, 0, 417}, + dictWord{14, 0, 129}, + dictWord{143, 0, 15}, + dictWord{10, 11, 545}, + dictWord{140, 11, 301}, + dictWord{17, 10, 39}, + dictWord{148, 10, 36}, + dictWord{133, 0, 199}, + dictWord{4, 11, 904}, + dictWord{133, 11, 794}, + dictWord{12, 0, 427}, + dictWord{146, 0, 38}, + dictWord{134, 0, 949}, + dictWord{8, 0, 665}, + dictWord{135, 10, 634}, + dictWord{ + 132, + 10, + 618, + }, + dictWord{135, 10, 259}, + dictWord{132, 10, 339}, + dictWord{133, 11, 761}, + dictWord{141, 10, 169}, + dictWord{132, 10, 759}, + dictWord{5, 0, 688}, + dictWord{7, 0, 539}, + dictWord{135, 0, 712}, + dictWord{7, 11, 386}, + dictWord{138, 11, 713}, + dictWord{134, 0, 1186}, + dictWord{6, 11, 7}, + dictWord{6, 11, 35}, + dictWord{ + 7, + 11, + 147, + }, + dictWord{7, 11, 1069}, + dictWord{7, 11, 1568}, + dictWord{7, 11, 1575}, + dictWord{7, 11, 1917}, + dictWord{8, 11, 43}, + dictWord{8, 11, 208}, + dictWord{ + 9, + 11, + 128, + }, + dictWord{9, 11, 866}, + dictWord{10, 11, 20}, + dictWord{11, 11, 981}, + dictWord{147, 11, 33}, + dictWord{7, 11, 893}, + dictWord{8, 10, 482}, + dictWord{141, 11, 424}, + dictWord{6, 0, 312}, + dictWord{6, 0, 1715}, + dictWord{10, 0, 584}, + dictWord{11, 0, 546}, + dictWord{11, 0, 692}, + dictWord{12, 0, 259}, + dictWord{ + 12, + 0, + 295, + }, + dictWord{13, 0, 46}, + dictWord{141, 0, 154}, + dictWord{5, 10, 336}, + dictWord{6, 10, 341}, + dictWord{6, 10, 478}, + dictWord{6, 10, 1763}, + dictWord{ + 136, + 10, + 386, + }, + dictWord{137, 0, 151}, + dictWord{132, 0, 588}, + dictWord{152, 0, 4}, + dictWord{6, 11, 322}, + dictWord{9, 11, 552}, + dictWord{11, 11, 274}, + dictWord{ + 13, + 11, + 209, + }, + dictWord{13, 11, 499}, + dictWord{14, 11, 85}, + dictWord{15, 11, 126}, + dictWord{145, 11, 70}, + dictWord{135, 10, 73}, + dictWord{4, 0, 231}, + dictWord{ + 5, + 0, + 61, + }, + dictWord{6, 0, 104}, + dictWord{7, 0, 729}, + dictWord{7, 0, 964}, + dictWord{7, 0, 1658}, + dictWord{140, 0, 414}, + dictWord{6, 0, 263}, + dictWord{138, 0, 757}, + dictWord{135, 10, 1971}, + dictWord{4, 0, 612}, + dictWord{133, 0, 561}, + dictWord{132, 0, 320}, + dictWord{135, 10, 1344}, + dictWord{8, 11, 83}, + dictWord{ + 8, + 11, + 817, + }, + dictWord{9, 11, 28}, + dictWord{9, 11, 29}, + dictWord{9, 11, 885}, + dictWord{10, 11, 387}, + dictWord{11, 11, 633}, + dictWord{11, 11, 740}, + dictWord{ + 13, + 11, + 235, + }, + dictWord{13, 11, 254}, + dictWord{15, 11, 143}, + dictWord{143, 11, 146}, + dictWord{5, 10, 396}, + dictWord{134, 10, 501}, + dictWord{140, 11, 49}, + dictWord{132, 0, 225}, + dictWord{4, 10, 929}, + dictWord{5, 10, 799}, + dictWord{8, 10, 46}, + dictWord{136, 10, 740}, + dictWord{4, 0, 405}, + dictWord{7, 0, 817}, + dictWord{ + 14, + 0, + 58, + }, + dictWord{17, 0, 37}, + dictWord{146, 0, 124}, + dictWord{133, 0, 974}, + dictWord{4, 11, 412}, + dictWord{133, 11, 581}, + dictWord{4, 10, 892}, + dictWord{ + 133, + 10, + 770, + }, + dictWord{4, 0, 996}, + dictWord{134, 0, 2026}, + dictWord{4, 0, 527}, + dictWord{5, 0, 235}, + dictWord{7, 0, 1239}, + dictWord{11, 0, 131}, + dictWord{ + 140, + 0, + 370, + }, + dictWord{9, 0, 16}, + dictWord{13, 0, 386}, + dictWord{135, 11, 421}, + dictWord{7, 0, 956}, + dictWord{7, 0, 1157}, + dictWord{7, 0, 1506}, + dictWord{7, 0, 1606}, + dictWord{7, 0, 1615}, + dictWord{7, 0, 1619}, + dictWord{7, 0, 1736}, + dictWord{7, 0, 1775}, + dictWord{8, 0, 590}, + dictWord{9, 0, 324}, + dictWord{9, 0, 736}, + dictWord{ + 9, + 0, + 774, + }, + dictWord{9, 0, 776}, + dictWord{9, 0, 784}, + dictWord{10, 0, 567}, + dictWord{10, 0, 708}, + dictWord{11, 0, 518}, + dictWord{11, 0, 613}, + dictWord{11, 0, 695}, + dictWord{11, 0, 716}, + dictWord{11, 0, 739}, + dictWord{11, 0, 770}, + dictWord{11, 0, 771}, + dictWord{11, 0, 848}, + dictWord{11, 0, 857}, + dictWord{11, 0, 931}, + dictWord{ + 11, + 0, + 947, + }, + dictWord{12, 0, 326}, + dictWord{12, 0, 387}, + dictWord{12, 0, 484}, + dictWord{12, 0, 528}, + dictWord{12, 0, 552}, + dictWord{12, 0, 613}, + dictWord{ + 13, + 0, + 189, + }, + dictWord{13, 0, 256}, + dictWord{13, 0, 340}, + dictWord{13, 0, 432}, + dictWord{13, 0, 436}, + dictWord{13, 0, 440}, + dictWord{13, 0, 454}, + dictWord{14, 0, 174}, + dictWord{14, 0, 220}, + dictWord{14, 0, 284}, + dictWord{14, 0, 390}, + dictWord{145, 0, 121}, + dictWord{135, 10, 158}, + dictWord{9, 0, 137}, + dictWord{138, 0, 221}, + dictWord{4, 11, 110}, + dictWord{10, 11, 415}, + dictWord{10, 11, 597}, + dictWord{142, 11, 206}, + dictWord{141, 11, 496}, + dictWord{135, 11, 205}, + dictWord{ + 151, + 10, + 25, + }, + dictWord{135, 11, 778}, + dictWord{7, 11, 1656}, + dictWord{7, 10, 2001}, + dictWord{9, 11, 369}, + dictWord{10, 11, 338}, + dictWord{10, 11, 490}, + dictWord{11, 11, 154}, + dictWord{11, 11, 545}, + dictWord{11, 11, 775}, + dictWord{13, 11, 77}, + dictWord{141, 11, 274}, + dictWord{4, 11, 444}, + dictWord{ + 10, + 11, + 146, + }, + dictWord{140, 11, 9}, + dictWord{7, 0, 390}, + dictWord{138, 0, 140}, + dictWord{135, 0, 1144}, + dictWord{134, 0, 464}, + dictWord{7, 10, 1461}, + dictWord{ + 140, + 10, + 91, + }, + dictWord{132, 10, 602}, + dictWord{4, 11, 283}, + dictWord{135, 11, 1194}, + dictWord{5, 0, 407}, + dictWord{11, 0, 204}, + dictWord{11, 0, 243}, + dictWord{ + 11, + 0, + 489, + }, + dictWord{12, 0, 293}, + dictWord{19, 0, 37}, + dictWord{20, 0, 73}, + dictWord{150, 0, 38}, + dictWord{7, 0, 1218}, + dictWord{136, 0, 303}, + dictWord{ + 5, + 0, + 325, + }, + dictWord{8, 0, 5}, + dictWord{8, 0, 227}, + dictWord{9, 0, 105}, + dictWord{10, 0, 585}, + dictWord{12, 0, 614}, + dictWord{4, 10, 13}, + dictWord{5, 10, 567}, + dictWord{ + 7, + 10, + 1498, + }, + dictWord{9, 10, 124}, + dictWord{11, 10, 521}, + dictWord{140, 10, 405}, + dictWord{135, 10, 1006}, + dictWord{7, 0, 800}, + dictWord{10, 0, 12}, + dictWord{134, 11, 1720}, + dictWord{135, 0, 1783}, + dictWord{132, 10, 735}, + dictWord{138, 10, 812}, + dictWord{4, 10, 170}, + dictWord{135, 10, 323}, + dictWord{ + 6, + 0, + 621, + }, + dictWord{13, 0, 504}, + dictWord{144, 0, 89}, + dictWord{5, 10, 304}, + dictWord{135, 10, 1403}, + dictWord{137, 11, 216}, + dictWord{6, 0, 920}, + dictWord{ + 6, + 0, + 1104, + }, + dictWord{9, 11, 183}, + dictWord{139, 11, 286}, + dictWord{4, 0, 376}, + dictWord{133, 10, 742}, + dictWord{134, 0, 218}, + dictWord{8, 0, 641}, + dictWord{ + 11, + 0, + 388, + }, + dictWord{140, 0, 580}, + dictWord{7, 0, 454}, + dictWord{7, 0, 782}, + dictWord{8, 0, 768}, + dictWord{140, 0, 686}, + dictWord{137, 11, 33}, + dictWord{ + 133, + 10, + 111, + }, + dictWord{144, 0, 0}, + dictWord{10, 0, 676}, + dictWord{140, 0, 462}, + dictWord{6, 0, 164}, + dictWord{136, 11, 735}, + dictWord{133, 10, 444}, + dictWord{ + 150, + 0, + 50, + }, + dictWord{7, 11, 1862}, + dictWord{12, 11, 491}, + dictWord{12, 11, 520}, + dictWord{13, 11, 383}, + dictWord{14, 11, 244}, + dictWord{146, 11, 12}, + dictWord{ + 5, + 11, + 132, + }, + dictWord{9, 11, 486}, + dictWord{9, 11, 715}, + dictWord{10, 11, 458}, + dictWord{11, 11, 373}, + dictWord{11, 11, 668}, + dictWord{11, 11, 795}, + dictWord{11, 11, 897}, + dictWord{12, 11, 272}, + dictWord{12, 11, 424}, + dictWord{12, 11, 539}, + dictWord{12, 11, 558}, + dictWord{14, 11, 245}, + dictWord{ + 14, + 11, + 263, + }, + dictWord{14, 11, 264}, + dictWord{14, 11, 393}, + dictWord{142, 11, 403}, + dictWord{8, 10, 123}, + dictWord{15, 10, 6}, + dictWord{144, 10, 7}, + dictWord{ + 6, + 0, + 285, + }, + dictWord{8, 0, 654}, + dictWord{11, 0, 749}, + dictWord{12, 0, 190}, + dictWord{12, 0, 327}, + dictWord{13, 0, 120}, + dictWord{13, 0, 121}, + dictWord{13, 0, 327}, + dictWord{15, 0, 47}, + dictWord{146, 0, 40}, + dictWord{5, 11, 8}, + dictWord{6, 11, 89}, + dictWord{6, 11, 400}, + dictWord{7, 11, 1569}, + dictWord{7, 11, 1623}, + dictWord{ + 7, + 11, + 1850, + }, + dictWord{8, 11, 218}, + dictWord{8, 11, 422}, + dictWord{9, 11, 570}, + dictWord{138, 11, 626}, + dictWord{6, 11, 387}, + dictWord{7, 11, 882}, + dictWord{141, 11, 111}, + dictWord{6, 0, 343}, + dictWord{7, 0, 195}, + dictWord{9, 0, 226}, + dictWord{10, 0, 197}, + dictWord{10, 0, 575}, + dictWord{11, 0, 502}, + dictWord{ + 11, + 0, + 899, + }, + dictWord{6, 11, 224}, + dictWord{7, 11, 877}, + dictWord{137, 11, 647}, + dictWord{5, 10, 937}, + dictWord{135, 10, 100}, + dictWord{135, 11, 790}, + dictWord{150, 0, 29}, + dictWord{147, 0, 8}, + dictWord{134, 0, 1812}, + dictWord{149, 0, 8}, + dictWord{135, 11, 394}, + dictWord{7, 0, 1125}, + dictWord{9, 0, 143}, + dictWord{ + 11, + 0, + 61, + }, + dictWord{14, 0, 405}, + dictWord{150, 0, 21}, + dictWord{10, 11, 755}, + dictWord{147, 11, 29}, + dictWord{9, 11, 378}, + dictWord{141, 11, 162}, + dictWord{135, 10, 922}, + dictWord{5, 10, 619}, + dictWord{133, 10, 698}, + dictWord{134, 0, 1327}, + dictWord{6, 0, 1598}, + dictWord{137, 0, 575}, + dictWord{ + 9, + 11, + 569, + }, + dictWord{12, 11, 12}, + dictWord{12, 11, 81}, + dictWord{12, 11, 319}, + dictWord{13, 11, 69}, + dictWord{14, 11, 259}, + dictWord{16, 11, 87}, + dictWord{ + 17, + 11, + 1, + }, + dictWord{17, 11, 21}, + dictWord{17, 11, 24}, + dictWord{18, 11, 15}, + dictWord{18, 11, 56}, + dictWord{18, 11, 59}, + dictWord{18, 11, 127}, + dictWord{18, 11, 154}, + dictWord{19, 11, 19}, + dictWord{148, 11, 31}, + dictWord{6, 0, 895}, + dictWord{135, 11, 1231}, + dictWord{5, 0, 959}, + dictWord{7, 11, 124}, + dictWord{136, 11, 38}, + dictWord{5, 11, 261}, + dictWord{7, 11, 78}, + dictWord{7, 11, 199}, + dictWord{8, 11, 815}, + dictWord{9, 11, 126}, + dictWord{138, 11, 342}, + dictWord{5, 10, 917}, + dictWord{134, 10, 1659}, + dictWord{7, 0, 1759}, + dictWord{5, 11, 595}, + dictWord{135, 11, 1863}, + dictWord{136, 0, 173}, + dictWord{134, 0, 266}, + dictWord{ + 142, + 0, + 261, + }, + dictWord{132, 11, 628}, + dictWord{5, 10, 251}, + dictWord{5, 10, 956}, + dictWord{8, 10, 268}, + dictWord{9, 10, 214}, + dictWord{146, 10, 142}, + dictWord{ + 7, + 11, + 266, + }, + dictWord{136, 11, 804}, + dictWord{135, 11, 208}, + dictWord{6, 11, 79}, + dictWord{7, 11, 1021}, + dictWord{135, 11, 1519}, + dictWord{11, 11, 704}, + dictWord{141, 11, 396}, + dictWord{5, 10, 346}, + dictWord{5, 10, 711}, + dictWord{136, 10, 390}, + dictWord{136, 11, 741}, + dictWord{134, 11, 376}, + dictWord{ + 134, + 0, + 1427, + }, + dictWord{6, 0, 1033}, + dictWord{6, 0, 1217}, + dictWord{136, 0, 300}, + dictWord{133, 10, 624}, + dictWord{6, 11, 100}, + dictWord{7, 11, 244}, + dictWord{ + 7, + 11, + 632, + }, + dictWord{7, 11, 1609}, + dictWord{8, 11, 178}, + dictWord{8, 11, 638}, + dictWord{141, 11, 58}, + dictWord{6, 0, 584}, + dictWord{5, 10, 783}, + dictWord{ + 7, + 10, + 1998, + }, + dictWord{135, 10, 2047}, + dictWord{5, 0, 427}, + dictWord{5, 0, 734}, + dictWord{7, 0, 478}, + dictWord{136, 0, 52}, + dictWord{7, 0, 239}, + dictWord{ + 11, + 0, + 217, + }, + dictWord{142, 0, 165}, + dictWord{134, 0, 1129}, + dictWord{6, 0, 168}, + dictWord{6, 0, 1734}, + dictWord{7, 0, 20}, + dictWord{7, 0, 1056}, + dictWord{8, 0, 732}, + dictWord{9, 0, 406}, + dictWord{9, 0, 911}, + dictWord{138, 0, 694}, + dictWord{132, 10, 594}, + dictWord{133, 11, 791}, + dictWord{7, 11, 686}, + dictWord{8, 11, 33}, + dictWord{8, 11, 238}, + dictWord{10, 11, 616}, + dictWord{11, 11, 467}, + dictWord{11, 11, 881}, + dictWord{13, 11, 217}, + dictWord{13, 11, 253}, + dictWord{ + 142, + 11, + 268, + }, + dictWord{137, 11, 476}, + dictWord{134, 0, 418}, + dictWord{133, 0, 613}, + dictWord{132, 0, 632}, + dictWord{132, 11, 447}, + dictWord{7, 0, 32}, + dictWord{ + 7, + 0, + 984, + }, + dictWord{8, 0, 85}, + dictWord{8, 0, 709}, + dictWord{9, 0, 579}, + dictWord{9, 0, 847}, + dictWord{9, 0, 856}, + dictWord{10, 0, 799}, + dictWord{11, 0, 258}, + dictWord{ + 11, + 0, + 1007, + }, + dictWord{12, 0, 331}, + dictWord{12, 0, 615}, + dictWord{13, 0, 188}, + dictWord{13, 0, 435}, + dictWord{14, 0, 8}, + dictWord{15, 0, 165}, + dictWord{ + 16, + 0, + 27, + }, + dictWord{20, 0, 40}, + dictWord{144, 11, 35}, + dictWord{4, 11, 128}, + dictWord{5, 11, 415}, + dictWord{6, 11, 462}, + dictWord{7, 11, 294}, + dictWord{7, 11, 578}, + dictWord{10, 11, 710}, + dictWord{139, 11, 86}, + dictWord{5, 0, 694}, + dictWord{136, 0, 909}, + dictWord{7, 0, 1109}, + dictWord{11, 0, 7}, + dictWord{5, 10, 37}, + dictWord{ + 6, + 10, + 39, + }, + dictWord{6, 10, 451}, + dictWord{7, 10, 218}, + dictWord{7, 10, 1166}, + dictWord{7, 10, 1687}, + dictWord{8, 10, 662}, + dictWord{144, 10, 2}, + dictWord{ + 136, + 11, + 587, + }, + dictWord{6, 11, 427}, + dictWord{7, 11, 1018}, + dictWord{138, 11, 692}, + dictWord{4, 11, 195}, + dictWord{6, 10, 508}, + dictWord{135, 11, 802}, + dictWord{4, 0, 167}, + dictWord{135, 0, 82}, + dictWord{5, 0, 62}, + dictWord{6, 0, 24}, + dictWord{6, 0, 534}, + dictWord{7, 0, 74}, + dictWord{7, 0, 678}, + dictWord{7, 0, 684}, + dictWord{ + 7, + 0, + 1043, + }, + dictWord{7, 0, 1072}, + dictWord{8, 0, 280}, + dictWord{8, 0, 541}, + dictWord{8, 0, 686}, + dictWord{9, 0, 258}, + dictWord{10, 0, 519}, + dictWord{11, 0, 252}, + dictWord{140, 0, 282}, + dictWord{138, 0, 33}, + dictWord{4, 0, 359}, + dictWord{133, 11, 738}, + dictWord{7, 0, 980}, + dictWord{9, 0, 328}, + dictWord{13, 0, 186}, + dictWord{13, 0, 364}, + dictWord{7, 10, 635}, + dictWord{7, 10, 796}, + dictWord{8, 10, 331}, + dictWord{9, 10, 330}, + dictWord{9, 10, 865}, + dictWord{10, 10, 119}, + dictWord{ + 10, + 10, + 235, + }, + dictWord{11, 10, 111}, + dictWord{11, 10, 129}, + dictWord{11, 10, 240}, + dictWord{12, 10, 31}, + dictWord{12, 10, 66}, + dictWord{12, 10, 222}, + dictWord{12, 10, 269}, + dictWord{12, 10, 599}, + dictWord{12, 10, 684}, + dictWord{12, 10, 689}, + dictWord{12, 10, 691}, + dictWord{142, 10, 345}, + dictWord{ + 137, + 10, + 527, + }, + dictWord{6, 0, 596}, + dictWord{7, 0, 585}, + dictWord{135, 10, 702}, + dictWord{134, 11, 1683}, + dictWord{133, 0, 211}, + dictWord{6, 0, 145}, + dictWord{ + 141, + 0, + 336, + }, + dictWord{134, 0, 1130}, + dictWord{7, 0, 873}, + dictWord{6, 10, 37}, + dictWord{7, 10, 1666}, + dictWord{8, 10, 195}, + dictWord{8, 10, 316}, + dictWord{ + 9, + 10, + 178, + }, + dictWord{9, 10, 276}, + dictWord{9, 10, 339}, + dictWord{9, 10, 536}, + dictWord{10, 10, 102}, + dictWord{10, 10, 362}, + dictWord{10, 10, 785}, + dictWord{ + 11, + 10, + 55, + }, + dictWord{11, 10, 149}, + dictWord{11, 10, 773}, + dictWord{13, 10, 416}, + dictWord{13, 10, 419}, + dictWord{14, 10, 38}, + dictWord{14, 10, 41}, + dictWord{ + 142, + 10, + 210, + }, + dictWord{8, 0, 840}, + dictWord{136, 0, 841}, + dictWord{132, 0, 263}, + dictWord{5, 11, 3}, + dictWord{8, 11, 578}, + dictWord{9, 11, 118}, + dictWord{ + 10, + 11, + 705, + }, + dictWord{12, 11, 383}, + dictWord{141, 11, 279}, + dictWord{132, 0, 916}, + dictWord{133, 11, 229}, + dictWord{133, 10, 645}, + dictWord{15, 0, 155}, + dictWord{16, 0, 79}, + dictWord{8, 11, 102}, + dictWord{10, 11, 578}, + dictWord{10, 11, 672}, + dictWord{12, 11, 496}, + dictWord{13, 11, 408}, + dictWord{14, 11, 121}, + dictWord{145, 11, 106}, + dictWord{4, 0, 599}, + dictWord{5, 0, 592}, + dictWord{6, 0, 1634}, + dictWord{7, 0, 5}, + dictWord{7, 0, 55}, + dictWord{7, 0, 67}, + dictWord{7, 0, 97}, + dictWord{7, 0, 691}, + dictWord{7, 0, 979}, + dictWord{7, 0, 1600}, + dictWord{7, 0, 1697}, + dictWord{8, 0, 207}, + dictWord{8, 0, 214}, + dictWord{8, 0, 231}, + dictWord{8, 0, 294}, + dictWord{8, 0, 336}, + dictWord{8, 0, 428}, + dictWord{8, 0, 471}, + dictWord{8, 0, 622}, + dictWord{8, 0, 626}, + dictWord{8, 0, 679}, + dictWord{8, 0, 759}, + dictWord{8, 0, 829}, + dictWord{9, 0, 11}, + dictWord{9, 0, 246}, + dictWord{9, 0, 484}, + dictWord{9, 0, 573}, + dictWord{9, 0, 706}, + dictWord{9, 0, 762}, + dictWord{9, 0, 798}, + dictWord{9, 0, 855}, + dictWord{9, 0, 870}, + dictWord{9, 0, 912}, + dictWord{10, 0, 303}, + dictWord{10, 0, 335}, + dictWord{10, 0, 424}, + dictWord{10, 0, 461}, + dictWord{10, 0, 543}, + dictWord{ + 10, + 0, + 759, + }, + dictWord{10, 0, 814}, + dictWord{11, 0, 59}, + dictWord{11, 0, 199}, + dictWord{11, 0, 235}, + dictWord{11, 0, 590}, + dictWord{11, 0, 631}, + dictWord{11, 0, 929}, + dictWord{11, 0, 963}, + dictWord{11, 0, 987}, + dictWord{12, 0, 114}, + dictWord{12, 0, 182}, + dictWord{12, 0, 226}, + dictWord{12, 0, 332}, + dictWord{12, 0, 439}, + dictWord{12, 0, 575}, + dictWord{12, 0, 598}, + dictWord{12, 0, 675}, + dictWord{13, 0, 8}, + dictWord{13, 0, 125}, + dictWord{13, 0, 194}, + dictWord{13, 0, 287}, + dictWord{ + 14, + 0, + 197, + }, + dictWord{14, 0, 383}, + dictWord{15, 0, 53}, + dictWord{17, 0, 63}, + dictWord{19, 0, 46}, + dictWord{19, 0, 98}, + dictWord{19, 0, 106}, + dictWord{148, 0, 85}, + dictWord{ + 7, + 0, + 1356, + }, + dictWord{132, 10, 290}, + dictWord{6, 10, 70}, + dictWord{7, 10, 1292}, + dictWord{10, 10, 762}, + dictWord{139, 10, 288}, + dictWord{150, 11, 55}, + dictWord{4, 0, 593}, + dictWord{8, 11, 115}, + dictWord{8, 11, 350}, + dictWord{9, 11, 489}, + dictWord{10, 11, 128}, + dictWord{11, 11, 306}, + dictWord{12, 11, 373}, + dictWord{14, 11, 30}, + dictWord{17, 11, 79}, + dictWord{147, 11, 80}, + dictWord{135, 11, 1235}, + dictWord{134, 0, 1392}, + dictWord{4, 11, 230}, + dictWord{ + 133, + 11, + 702, + }, + dictWord{147, 0, 126}, + dictWord{7, 10, 131}, + dictWord{7, 10, 422}, + dictWord{8, 10, 210}, + dictWord{140, 10, 573}, + dictWord{134, 0, 1179}, + dictWord{ + 139, + 11, + 435, + }, + dictWord{139, 10, 797}, + dictWord{134, 11, 1728}, + dictWord{4, 0, 162}, + dictWord{18, 11, 26}, + dictWord{19, 11, 42}, + dictWord{20, 11, 43}, + dictWord{21, 11, 0}, + dictWord{23, 11, 27}, + dictWord{152, 11, 14}, + dictWord{132, 10, 936}, + dictWord{6, 0, 765}, + dictWord{5, 10, 453}, + dictWord{134, 10, 441}, + dictWord{133, 0, 187}, + dictWord{135, 0, 1286}, + dictWord{6, 0, 635}, + dictWord{6, 0, 904}, + dictWord{6, 0, 1210}, + dictWord{134, 0, 1489}, + dictWord{4, 0, 215}, + dictWord{ + 8, + 0, + 890, + }, + dictWord{9, 0, 38}, + dictWord{10, 0, 923}, + dictWord{11, 0, 23}, + dictWord{11, 0, 127}, + dictWord{139, 0, 796}, + dictWord{6, 0, 1165}, + dictWord{ + 134, + 0, + 1306, + }, + dictWord{7, 0, 716}, + dictWord{13, 0, 97}, + dictWord{141, 0, 251}, + dictWord{132, 10, 653}, + dictWord{136, 0, 657}, + dictWord{146, 10, 80}, + dictWord{ + 5, + 11, + 622, + }, + dictWord{7, 11, 1032}, + dictWord{11, 11, 26}, + dictWord{11, 11, 213}, + dictWord{11, 11, 707}, + dictWord{12, 11, 380}, + dictWord{13, 11, 226}, + dictWord{141, 11, 355}, + dictWord{6, 0, 299}, + dictWord{5, 11, 70}, + dictWord{6, 11, 334}, + dictWord{9, 11, 171}, + dictWord{11, 11, 637}, + dictWord{12, 11, 202}, + dictWord{14, 11, 222}, + dictWord{145, 11, 42}, + dictWord{142, 0, 134}, + dictWord{4, 11, 23}, + dictWord{5, 11, 313}, + dictWord{5, 11, 1014}, + dictWord{6, 11, 50}, + dictWord{ + 6, + 11, + 51, + }, + dictWord{7, 11, 142}, + dictWord{7, 11, 384}, + dictWord{9, 11, 783}, + dictWord{139, 11, 741}, + dictWord{4, 11, 141}, + dictWord{7, 11, 559}, + dictWord{ + 8, + 11, + 640, + }, + dictWord{9, 11, 460}, + dictWord{12, 11, 183}, + dictWord{141, 11, 488}, + dictWord{136, 11, 614}, + dictWord{7, 10, 1368}, + dictWord{8, 10, 232}, + dictWord{8, 10, 361}, + dictWord{10, 10, 682}, + dictWord{138, 10, 742}, + dictWord{137, 10, 534}, + dictWord{6, 0, 1082}, + dictWord{140, 0, 658}, + dictWord{ + 137, + 10, + 27, + }, + dictWord{135, 0, 2002}, + dictWord{142, 10, 12}, + dictWord{4, 0, 28}, + dictWord{5, 0, 440}, + dictWord{7, 0, 248}, + dictWord{11, 0, 833}, + dictWord{140, 0, 344}, + dictWord{7, 10, 736}, + dictWord{139, 10, 264}, + dictWord{134, 10, 1657}, + dictWord{134, 0, 1654}, + dictWord{138, 0, 531}, + dictWord{5, 11, 222}, + dictWord{ + 9, + 11, + 140, + }, + dictWord{138, 11, 534}, + dictWord{6, 0, 634}, + dictWord{6, 0, 798}, + dictWord{134, 0, 840}, + dictWord{138, 11, 503}, + dictWord{135, 10, 127}, + dictWord{133, 0, 853}, + dictWord{5, 11, 154}, + dictWord{7, 11, 1491}, + dictWord{10, 11, 379}, + dictWord{138, 11, 485}, + dictWord{6, 0, 249}, + dictWord{7, 0, 1234}, + dictWord{139, 0, 573}, + dictWord{133, 11, 716}, + dictWord{7, 11, 1570}, + dictWord{140, 11, 542}, + dictWord{136, 10, 364}, + dictWord{138, 0, 527}, + dictWord{ + 4, + 11, + 91, + }, + dictWord{5, 11, 388}, + dictWord{5, 11, 845}, + dictWord{6, 11, 206}, + dictWord{6, 11, 252}, + dictWord{6, 11, 365}, + dictWord{7, 11, 136}, + dictWord{7, 11, 531}, + dictWord{8, 11, 264}, + dictWord{136, 11, 621}, + dictWord{134, 0, 1419}, + dictWord{135, 11, 1441}, + dictWord{7, 0, 49}, + dictWord{7, 0, 392}, + dictWord{8, 0, 20}, + dictWord{8, 0, 172}, + dictWord{8, 0, 690}, + dictWord{9, 0, 383}, + dictWord{9, 0, 845}, + dictWord{10, 0, 48}, + dictWord{11, 0, 293}, + dictWord{11, 0, 832}, + dictWord{ + 11, + 0, + 920, + }, + dictWord{11, 0, 984}, + dictWord{141, 0, 221}, + dictWord{5, 0, 858}, + dictWord{133, 0, 992}, + dictWord{5, 0, 728}, + dictWord{137, 10, 792}, + dictWord{ + 5, + 10, + 909, + }, + dictWord{9, 10, 849}, + dictWord{138, 10, 805}, + dictWord{7, 0, 525}, + dictWord{7, 0, 1579}, + dictWord{8, 0, 497}, + dictWord{136, 0, 573}, + dictWord{6, 0, 268}, + dictWord{137, 0, 62}, + dictWord{135, 11, 576}, + dictWord{134, 0, 1201}, + dictWord{5, 11, 771}, + dictWord{5, 11, 863}, + dictWord{5, 11, 898}, + dictWord{ + 6, + 11, + 1632, + }, + dictWord{6, 11, 1644}, + dictWord{134, 11, 1780}, + dictWord{133, 11, 331}, + dictWord{7, 0, 193}, + dictWord{7, 0, 1105}, + dictWord{10, 0, 495}, + dictWord{ + 7, + 10, + 397, + }, + dictWord{8, 10, 124}, + dictWord{8, 10, 619}, + dictWord{9, 10, 305}, + dictWord{11, 10, 40}, + dictWord{12, 10, 349}, + dictWord{13, 10, 134}, + dictWord{ + 13, + 10, + 295, + }, + dictWord{14, 10, 155}, + dictWord{15, 10, 120}, + dictWord{146, 10, 105}, + dictWord{138, 0, 106}, + dictWord{6, 0, 859}, + dictWord{5, 11, 107}, + dictWord{ + 7, + 11, + 201, + }, + dictWord{136, 11, 518}, + dictWord{6, 11, 446}, + dictWord{135, 11, 1817}, + dictWord{13, 0, 23}, + dictWord{4, 10, 262}, + dictWord{135, 10, 342}, + dictWord{133, 10, 641}, + dictWord{137, 11, 851}, + dictWord{6, 0, 925}, + dictWord{137, 0, 813}, + dictWord{132, 11, 504}, + dictWord{6, 0, 613}, + dictWord{ + 136, + 0, + 223, + }, + dictWord{4, 10, 99}, + dictWord{6, 10, 250}, + dictWord{6, 10, 346}, + dictWord{8, 10, 127}, + dictWord{138, 10, 81}, + dictWord{136, 0, 953}, + dictWord{ + 132, + 10, + 915, + }, + dictWord{139, 11, 892}, + dictWord{5, 10, 75}, + dictWord{9, 10, 517}, + dictWord{10, 10, 470}, + dictWord{12, 10, 155}, + dictWord{141, 10, 224}, + dictWord{ + 4, + 0, + 666, + }, + dictWord{7, 0, 1017}, + dictWord{7, 11, 996}, + dictWord{138, 11, 390}, + dictWord{5, 11, 883}, + dictWord{133, 11, 975}, + dictWord{14, 10, 83}, + dictWord{ + 142, + 11, + 83, + }, + dictWord{4, 0, 670}, + dictWord{5, 11, 922}, + dictWord{134, 11, 1707}, + dictWord{135, 0, 216}, + dictWord{9, 0, 40}, + dictWord{11, 0, 136}, + dictWord{ + 135, + 11, + 787, + }, + dictWord{5, 10, 954}, + dictWord{5, 11, 993}, + dictWord{7, 11, 515}, + dictWord{137, 11, 91}, + dictWord{139, 0, 259}, + dictWord{7, 0, 1114}, + dictWord{ + 9, + 0, + 310, + }, + dictWord{9, 0, 682}, + dictWord{10, 0, 440}, + dictWord{13, 0, 40}, + dictWord{6, 10, 304}, + dictWord{8, 10, 418}, + dictWord{11, 10, 341}, + dictWord{ + 139, + 10, + 675, + }, + dictWord{14, 0, 296}, + dictWord{9, 10, 410}, + dictWord{139, 10, 425}, + dictWord{10, 11, 377}, + dictWord{12, 11, 363}, + dictWord{13, 11, 68}, + dictWord{ + 13, + 11, + 94, + }, + dictWord{14, 11, 108}, + dictWord{142, 11, 306}, + dictWord{7, 0, 1401}, + dictWord{135, 0, 1476}, + dictWord{4, 0, 296}, + dictWord{6, 0, 475}, + dictWord{ + 7, + 0, + 401, + }, + dictWord{7, 0, 1410}, + dictWord{7, 0, 1594}, + dictWord{7, 0, 1674}, + dictWord{8, 0, 63}, + dictWord{8, 0, 660}, + dictWord{137, 0, 74}, + dictWord{4, 0, 139}, + dictWord{4, 0, 388}, + dictWord{140, 0, 188}, + dictWord{132, 0, 797}, + dictWord{132, 11, 766}, + dictWord{5, 11, 103}, + dictWord{7, 11, 921}, + dictWord{8, 11, 580}, + dictWord{8, 11, 593}, + dictWord{8, 11, 630}, + dictWord{138, 11, 28}, + dictWord{4, 11, 911}, + dictWord{5, 11, 867}, + dictWord{133, 11, 1013}, + dictWord{134, 10, 14}, + dictWord{134, 0, 1572}, + dictWord{134, 10, 1708}, + dictWord{21, 0, 39}, + dictWord{5, 10, 113}, + dictWord{6, 10, 243}, + dictWord{7, 10, 1865}, + dictWord{ + 11, + 10, + 161, + }, + dictWord{16, 10, 37}, + dictWord{145, 10, 99}, + dictWord{7, 11, 1563}, + dictWord{141, 11, 182}, + dictWord{5, 11, 135}, + dictWord{6, 11, 519}, + dictWord{ + 7, + 11, + 1722, + }, + dictWord{10, 11, 271}, + dictWord{11, 11, 261}, + dictWord{145, 11, 54}, + dictWord{132, 10, 274}, + dictWord{134, 0, 1594}, + dictWord{4, 11, 300}, + dictWord{5, 11, 436}, + dictWord{135, 11, 484}, + dictWord{4, 0, 747}, + dictWord{6, 0, 290}, + dictWord{7, 0, 649}, + dictWord{7, 0, 1479}, + dictWord{135, 0, 1583}, + dictWord{133, 11, 535}, + dictWord{147, 11, 82}, + dictWord{133, 0, 232}, + dictWord{137, 0, 887}, + dictWord{135, 10, 166}, + dictWord{136, 0, 521}, + dictWord{4, 0, 14}, + dictWord{7, 0, 472}, + dictWord{7, 0, 1801}, + dictWord{10, 0, 748}, + dictWord{141, 0, 458}, + dictWord{134, 0, 741}, + dictWord{134, 0, 992}, + dictWord{16, 0, 111}, + dictWord{137, 10, 304}, + dictWord{4, 0, 425}, + dictWord{5, 11, 387}, + dictWord{7, 11, 557}, + dictWord{12, 11, 547}, + dictWord{142, 11, 86}, + dictWord{ + 135, + 11, + 1747, + }, + dictWord{5, 10, 654}, + dictWord{135, 11, 1489}, + dictWord{7, 0, 789}, + dictWord{4, 11, 6}, + dictWord{5, 11, 708}, + dictWord{136, 11, 75}, + dictWord{ + 6, + 10, + 273, + }, + dictWord{10, 10, 188}, + dictWord{13, 10, 377}, + dictWord{146, 10, 77}, + dictWord{6, 0, 1593}, + dictWord{4, 11, 303}, + dictWord{7, 11, 619}, + dictWord{ + 10, + 11, + 547, + }, + dictWord{10, 11, 687}, + dictWord{11, 11, 122}, + dictWord{140, 11, 601}, + dictWord{134, 0, 1768}, + dictWord{135, 10, 410}, + dictWord{138, 11, 772}, + dictWord{11, 0, 233}, + dictWord{139, 10, 524}, + dictWord{5, 0, 943}, + dictWord{134, 0, 1779}, + dictWord{134, 10, 1785}, + dictWord{136, 11, 529}, + dictWord{ + 132, + 0, + 955, + }, + dictWord{5, 0, 245}, + dictWord{6, 0, 576}, + dictWord{7, 0, 582}, + dictWord{136, 0, 225}, + dictWord{132, 10, 780}, + dictWord{142, 0, 241}, + dictWord{ + 134, + 0, + 1943, + }, + dictWord{4, 11, 106}, + dictWord{7, 11, 310}, + dictWord{7, 11, 1785}, + dictWord{10, 11, 690}, + dictWord{139, 11, 717}, + dictWord{134, 0, 1284}, + dictWord{5, 11, 890}, + dictWord{133, 11, 988}, + dictWord{6, 11, 626}, + dictWord{142, 11, 431}, + dictWord{10, 11, 706}, + dictWord{145, 11, 32}, + dictWord{ + 137, + 11, + 332, + }, + dictWord{132, 11, 698}, + dictWord{135, 0, 709}, + dictWord{5, 10, 948}, + dictWord{138, 11, 17}, + dictWord{136, 0, 554}, + dictWord{134, 0, 1564}, + dictWord{139, 10, 941}, + dictWord{132, 0, 443}, + dictWord{134, 0, 909}, + dictWord{134, 11, 84}, + dictWord{142, 0, 280}, + dictWord{4, 10, 532}, + dictWord{5, 10, 706}, + dictWord{135, 10, 662}, + dictWord{132, 0, 729}, + dictWord{5, 10, 837}, + dictWord{6, 10, 1651}, + dictWord{139, 10, 985}, + dictWord{135, 10, 1861}, + dictWord{ + 4, + 0, + 348, + }, + dictWord{152, 11, 3}, + dictWord{5, 11, 986}, + dictWord{6, 11, 130}, + dictWord{7, 11, 1582}, + dictWord{8, 11, 458}, + dictWord{10, 11, 101}, + dictWord{ + 10, + 11, + 318, + }, + dictWord{138, 11, 823}, + dictWord{134, 0, 758}, + dictWord{4, 0, 298}, + dictWord{137, 0, 848}, + dictWord{4, 10, 330}, + dictWord{7, 10, 933}, + dictWord{ + 7, + 10, + 2012, + }, + dictWord{136, 10, 292}, + dictWord{7, 11, 1644}, + dictWord{137, 11, 129}, + dictWord{6, 0, 1422}, + dictWord{9, 0, 829}, + dictWord{135, 10, 767}, + dictWord{5, 0, 164}, + dictWord{7, 0, 121}, + dictWord{142, 0, 189}, + dictWord{7, 0, 812}, + dictWord{7, 0, 1261}, + dictWord{7, 0, 1360}, + dictWord{9, 0, 632}, + dictWord{ + 140, + 0, + 352, + }, + dictWord{135, 11, 1788}, + dictWord{139, 0, 556}, + dictWord{135, 11, 997}, + dictWord{145, 10, 114}, + dictWord{4, 0, 172}, + dictWord{9, 0, 611}, + dictWord{10, 0, 436}, + dictWord{12, 0, 673}, + dictWord{13, 0, 255}, + dictWord{137, 10, 883}, + dictWord{11, 0, 530}, + dictWord{138, 10, 274}, + dictWord{133, 0, 844}, + dictWord{134, 0, 984}, + dictWord{13, 0, 232}, + dictWord{18, 0, 35}, + dictWord{4, 10, 703}, + dictWord{135, 10, 207}, + dictWord{132, 10, 571}, + dictWord{9, 0, 263}, + dictWord{10, 0, 147}, + dictWord{138, 0, 492}, + dictWord{7, 11, 1756}, + dictWord{137, 11, 98}, + dictWord{5, 10, 873}, + dictWord{5, 10, 960}, + dictWord{8, 10, 823}, + dictWord{137, 10, 881}, + dictWord{133, 0, 537}, + dictWord{132, 0, 859}, + dictWord{7, 11, 1046}, + dictWord{139, 11, 160}, + dictWord{137, 0, 842}, + dictWord{ + 139, + 10, + 283, + }, + dictWord{5, 10, 33}, + dictWord{6, 10, 470}, + dictWord{139, 10, 424}, + dictWord{6, 11, 45}, + dictWord{7, 11, 433}, + dictWord{8, 11, 129}, + dictWord{ + 9, + 11, + 21, + }, + dictWord{10, 11, 392}, + dictWord{11, 11, 79}, + dictWord{12, 11, 499}, + dictWord{13, 11, 199}, + dictWord{141, 11, 451}, + dictWord{135, 0, 1291}, + dictWord{135, 10, 1882}, + dictWord{7, 11, 558}, + dictWord{136, 11, 353}, + dictWord{134, 0, 1482}, + dictWord{5, 0, 230}, + dictWord{5, 0, 392}, + dictWord{6, 0, 420}, + dictWord{9, 0, 568}, + dictWord{140, 0, 612}, + dictWord{6, 0, 262}, + dictWord{7, 10, 90}, + dictWord{7, 10, 664}, + dictWord{7, 10, 830}, + dictWord{7, 10, 1380}, + dictWord{ + 7, + 10, + 2025, + }, + dictWord{8, 11, 81}, + dictWord{8, 10, 448}, + dictWord{8, 10, 828}, + dictWord{9, 11, 189}, + dictWord{9, 11, 201}, + dictWord{11, 11, 478}, + dictWord{ + 11, + 11, + 712, + }, + dictWord{141, 11, 338}, + dictWord{142, 0, 31}, + dictWord{5, 11, 353}, + dictWord{151, 11, 26}, + dictWord{132, 0, 753}, + dictWord{4, 0, 0}, + dictWord{ + 5, + 0, + 41, + }, + dictWord{7, 0, 1459}, + dictWord{7, 0, 1469}, + dictWord{7, 0, 1859}, + dictWord{9, 0, 549}, + dictWord{139, 0, 905}, + dictWord{9, 10, 417}, + dictWord{ + 137, + 10, + 493, + }, + dictWord{135, 11, 1113}, + dictWord{133, 0, 696}, + dictWord{141, 11, 448}, + dictWord{134, 10, 295}, + dictWord{132, 0, 834}, + dictWord{4, 0, 771}, + dictWord{5, 10, 1019}, + dictWord{6, 11, 25}, + dictWord{7, 11, 855}, + dictWord{7, 11, 1258}, + dictWord{144, 11, 32}, + dictWord{134, 0, 1076}, + dictWord{133, 0, 921}, + dictWord{133, 0, 674}, + dictWord{4, 11, 4}, + dictWord{7, 11, 1118}, + dictWord{7, 11, 1320}, + dictWord{7, 11, 1706}, + dictWord{8, 11, 277}, + dictWord{9, 11, 622}, + dictWord{10, 11, 9}, + dictWord{11, 11, 724}, + dictWord{12, 11, 350}, + dictWord{12, 11, 397}, + dictWord{13, 11, 28}, + dictWord{13, 11, 159}, + dictWord{15, 11, 89}, + dictWord{18, 11, 5}, + dictWord{19, 11, 9}, + dictWord{20, 11, 34}, + dictWord{150, 11, 47}, + dictWord{134, 10, 208}, + dictWord{6, 0, 444}, + dictWord{136, 0, 308}, + dictWord{ + 6, + 0, + 180, + }, + dictWord{7, 0, 1137}, + dictWord{8, 0, 751}, + dictWord{139, 0, 805}, + dictWord{4, 0, 183}, + dictWord{7, 0, 271}, + dictWord{11, 0, 824}, + dictWord{ + 11, + 0, + 952, + }, + dictWord{13, 0, 278}, + dictWord{13, 0, 339}, + dictWord{13, 0, 482}, + dictWord{14, 0, 424}, + dictWord{148, 0, 99}, + dictWord{7, 11, 317}, + dictWord{ + 135, + 11, + 569, + }, + dictWord{4, 0, 19}, + dictWord{5, 0, 477}, + dictWord{5, 0, 596}, + dictWord{6, 0, 505}, + dictWord{7, 0, 1221}, + dictWord{11, 0, 907}, + dictWord{12, 0, 209}, + dictWord{141, 0, 214}, + dictWord{135, 0, 1215}, + dictWord{6, 0, 271}, + dictWord{7, 0, 398}, + dictWord{8, 0, 387}, + dictWord{10, 0, 344}, + dictWord{7, 10, 448}, + dictWord{ + 7, + 10, + 1629, + }, + dictWord{7, 10, 1813}, + dictWord{8, 10, 442}, + dictWord{9, 10, 710}, + dictWord{10, 10, 282}, + dictWord{138, 10, 722}, + dictWord{11, 10, 844}, + dictWord{12, 10, 104}, + dictWord{140, 10, 625}, + dictWord{134, 11, 255}, + dictWord{133, 10, 787}, + dictWord{134, 0, 1645}, + dictWord{11, 11, 956}, + dictWord{ + 151, + 11, + 3, + }, + dictWord{6, 0, 92}, + dictWord{6, 0, 188}, + dictWord{7, 0, 209}, + dictWord{7, 0, 1269}, + dictWord{7, 0, 1524}, + dictWord{7, 0, 1876}, + dictWord{8, 0, 661}, + dictWord{10, 0, 42}, + dictWord{10, 0, 228}, + dictWord{11, 0, 58}, + dictWord{11, 0, 1020}, + dictWord{12, 0, 58}, + dictWord{12, 0, 118}, + dictWord{141, 0, 32}, + dictWord{ + 4, + 0, + 459, + }, + dictWord{133, 0, 966}, + dictWord{4, 11, 536}, + dictWord{7, 11, 1141}, + dictWord{10, 11, 723}, + dictWord{139, 11, 371}, + dictWord{140, 0, 330}, + dictWord{134, 0, 1557}, + dictWord{7, 11, 285}, + dictWord{135, 11, 876}, + dictWord{136, 10, 491}, + dictWord{135, 11, 560}, + dictWord{6, 0, 18}, + dictWord{7, 0, 179}, + dictWord{7, 0, 932}, + dictWord{8, 0, 548}, + dictWord{8, 0, 757}, + dictWord{9, 0, 54}, + dictWord{9, 0, 65}, + dictWord{9, 0, 532}, + dictWord{9, 0, 844}, + dictWord{10, 0, 113}, + dictWord{10, 0, 117}, + dictWord{10, 0, 315}, + dictWord{10, 0, 560}, + dictWord{10, 0, 622}, + dictWord{10, 0, 798}, + dictWord{11, 0, 153}, + dictWord{11, 0, 351}, + dictWord{ + 11, + 0, + 375, + }, + dictWord{12, 0, 78}, + dictWord{12, 0, 151}, + dictWord{12, 0, 392}, + dictWord{12, 0, 666}, + dictWord{14, 0, 248}, + dictWord{143, 0, 23}, + dictWord{ + 6, + 0, + 1742, + }, + dictWord{132, 11, 690}, + dictWord{4, 10, 403}, + dictWord{5, 10, 441}, + dictWord{7, 10, 450}, + dictWord{10, 10, 840}, + dictWord{11, 10, 101}, + dictWord{ + 12, + 10, + 193, + }, + dictWord{141, 10, 430}, + dictWord{133, 0, 965}, + dictWord{134, 0, 182}, + dictWord{10, 0, 65}, + dictWord{10, 0, 488}, + dictWord{138, 0, 497}, + dictWord{135, 11, 1346}, + dictWord{6, 0, 973}, + dictWord{6, 0, 1158}, + dictWord{10, 11, 200}, + dictWord{19, 11, 2}, + dictWord{151, 11, 22}, + dictWord{4, 11, 190}, + dictWord{133, 11, 554}, + dictWord{133, 10, 679}, + dictWord{7, 0, 328}, + dictWord{137, 10, 326}, + dictWord{133, 11, 1001}, + dictWord{9, 0, 588}, + dictWord{ + 138, + 0, + 260, + }, + dictWord{133, 11, 446}, + dictWord{135, 10, 1128}, + dictWord{135, 10, 1796}, + dictWord{147, 11, 119}, + dictWord{134, 0, 1786}, + dictWord{ + 6, + 0, + 1328, + }, + dictWord{6, 0, 1985}, + dictWord{8, 0, 962}, + dictWord{138, 0, 1017}, + dictWord{135, 0, 308}, + dictWord{11, 0, 508}, + dictWord{4, 10, 574}, + dictWord{ + 7, + 10, + 350, + }, + dictWord{7, 10, 1024}, + dictWord{8, 10, 338}, + dictWord{9, 10, 677}, + dictWord{138, 10, 808}, + dictWord{138, 11, 752}, + dictWord{135, 10, 1081}, + dictWord{137, 11, 96}, + dictWord{7, 10, 1676}, + dictWord{135, 10, 2037}, + dictWord{136, 0, 588}, + dictWord{132, 11, 304}, + dictWord{133, 0, 614}, + dictWord{ + 140, + 0, + 793, + }, + dictWord{136, 0, 287}, + dictWord{137, 10, 297}, + dictWord{141, 10, 37}, + dictWord{6, 11, 53}, + dictWord{6, 11, 199}, + dictWord{7, 11, 1408}, + dictWord{ + 8, + 11, + 32, + }, + dictWord{8, 11, 93}, + dictWord{9, 11, 437}, + dictWord{10, 11, 397}, + dictWord{10, 11, 629}, + dictWord{11, 11, 593}, + dictWord{11, 11, 763}, + dictWord{ + 13, + 11, + 326, + }, + dictWord{145, 11, 35}, + dictWord{134, 11, 105}, + dictWord{9, 11, 320}, + dictWord{10, 11, 506}, + dictWord{138, 11, 794}, + dictWord{5, 11, 114}, + dictWord{5, 11, 255}, + dictWord{141, 11, 285}, + dictWord{140, 0, 290}, + dictWord{7, 11, 2035}, + dictWord{8, 11, 19}, + dictWord{9, 11, 89}, + dictWord{138, 11, 831}, + dictWord{134, 0, 1136}, + dictWord{7, 0, 719}, + dictWord{8, 0, 796}, + dictWord{8, 0, 809}, + dictWord{8, 0, 834}, + dictWord{6, 10, 306}, + dictWord{7, 10, 1140}, + dictWord{ + 7, + 10, + 1340, + }, + dictWord{8, 10, 133}, + dictWord{138, 10, 449}, + dictWord{139, 10, 1011}, + dictWord{5, 0, 210}, + dictWord{6, 0, 213}, + dictWord{7, 0, 60}, + dictWord{ + 10, + 0, + 364, + }, + dictWord{139, 0, 135}, + dictWord{5, 0, 607}, + dictWord{8, 0, 326}, + dictWord{136, 0, 490}, + dictWord{138, 11, 176}, + dictWord{132, 0, 701}, + dictWord{ + 5, + 0, + 472, + }, + dictWord{7, 0, 380}, + dictWord{137, 0, 758}, + dictWord{135, 0, 1947}, + dictWord{6, 0, 1079}, + dictWord{138, 0, 278}, + dictWord{138, 11, 391}, + dictWord{ + 5, + 10, + 329, + }, + dictWord{8, 10, 260}, + dictWord{139, 11, 156}, + dictWord{4, 0, 386}, + dictWord{7, 0, 41}, + dictWord{8, 0, 405}, + dictWord{8, 0, 728}, + dictWord{9, 0, 497}, + dictWord{11, 0, 110}, + dictWord{11, 0, 360}, + dictWord{15, 0, 37}, + dictWord{144, 0, 84}, + dictWord{5, 0, 46}, + dictWord{7, 0, 1452}, + dictWord{7, 0, 1480}, + dictWord{ + 8, + 0, + 634, + }, + dictWord{140, 0, 472}, + dictWord{136, 0, 961}, + dictWord{4, 0, 524}, + dictWord{136, 0, 810}, + dictWord{10, 0, 238}, + dictWord{141, 0, 33}, + dictWord{ + 132, + 10, + 657, + }, + dictWord{152, 10, 7}, + dictWord{133, 0, 532}, + dictWord{5, 0, 997}, + dictWord{135, 10, 1665}, + dictWord{7, 11, 594}, + dictWord{7, 11, 851}, + dictWord{ + 7, + 11, + 1858, + }, + dictWord{9, 11, 411}, + dictWord{9, 11, 574}, + dictWord{9, 11, 666}, + dictWord{9, 11, 737}, + dictWord{10, 11, 346}, + dictWord{10, 11, 712}, + dictWord{ + 11, + 11, + 246, + }, + dictWord{11, 11, 432}, + dictWord{11, 11, 517}, + dictWord{11, 11, 647}, + dictWord{11, 11, 679}, + dictWord{11, 11, 727}, + dictWord{12, 11, 304}, + dictWord{12, 11, 305}, + dictWord{12, 11, 323}, + dictWord{12, 11, 483}, + dictWord{12, 11, 572}, + dictWord{12, 11, 593}, + dictWord{12, 11, 602}, + dictWord{ + 13, + 11, + 95, + }, + dictWord{13, 11, 101}, + dictWord{13, 11, 171}, + dictWord{13, 11, 315}, + dictWord{13, 11, 378}, + dictWord{13, 11, 425}, + dictWord{13, 11, 475}, + dictWord{ + 14, + 11, + 63, + }, + dictWord{14, 11, 380}, + dictWord{14, 11, 384}, + dictWord{15, 11, 133}, + dictWord{18, 11, 112}, + dictWord{148, 11, 72}, + dictWord{5, 11, 955}, + dictWord{136, 11, 814}, + dictWord{134, 0, 1301}, + dictWord{5, 10, 66}, + dictWord{7, 10, 1896}, + dictWord{136, 10, 288}, + dictWord{133, 11, 56}, + dictWord{ + 134, + 10, + 1643, + }, + dictWord{6, 0, 1298}, + dictWord{148, 11, 100}, + dictWord{5, 0, 782}, + dictWord{5, 0, 829}, + dictWord{6, 0, 671}, + dictWord{6, 0, 1156}, + dictWord{6, 0, 1738}, + dictWord{137, 11, 621}, + dictWord{4, 0, 306}, + dictWord{5, 0, 570}, + dictWord{7, 0, 1347}, + dictWord{5, 10, 91}, + dictWord{5, 10, 648}, + dictWord{5, 10, 750}, + dictWord{ + 5, + 10, + 781, + }, + dictWord{6, 10, 54}, + dictWord{6, 10, 112}, + dictWord{6, 10, 402}, + dictWord{6, 10, 1732}, + dictWord{7, 10, 315}, + dictWord{7, 10, 749}, + dictWord{ + 7, + 10, + 1900, + }, + dictWord{9, 10, 78}, + dictWord{9, 10, 508}, + dictWord{10, 10, 611}, + dictWord{10, 10, 811}, + dictWord{11, 10, 510}, + dictWord{11, 10, 728}, + dictWord{ + 13, + 10, + 36, + }, + dictWord{14, 10, 39}, + dictWord{16, 10, 83}, + dictWord{17, 10, 124}, + dictWord{148, 10, 30}, + dictWord{8, 10, 570}, + dictWord{9, 11, 477}, + dictWord{ + 141, + 11, + 78, + }, + dictWord{4, 11, 639}, + dictWord{10, 11, 4}, + dictWord{10, 10, 322}, + dictWord{10, 10, 719}, + dictWord{11, 10, 407}, + dictWord{11, 11, 638}, + dictWord{ + 12, + 11, + 177, + }, + dictWord{148, 11, 57}, + dictWord{7, 0, 1823}, + dictWord{139, 0, 693}, + dictWord{7, 0, 759}, + dictWord{5, 11, 758}, + dictWord{8, 10, 125}, + dictWord{ + 8, + 10, + 369, + }, + dictWord{8, 10, 524}, + dictWord{10, 10, 486}, + dictWord{11, 10, 13}, + dictWord{11, 10, 381}, + dictWord{11, 10, 736}, + dictWord{11, 10, 766}, + dictWord{ + 11, + 10, + 845, + }, + dictWord{13, 10, 114}, + dictWord{13, 10, 292}, + dictWord{142, 10, 47}, + dictWord{7, 0, 1932}, + dictWord{6, 10, 1684}, + dictWord{6, 10, 1731}, + dictWord{7, 10, 356}, + dictWord{8, 10, 54}, + dictWord{8, 10, 221}, + dictWord{9, 10, 225}, + dictWord{9, 10, 356}, + dictWord{10, 10, 77}, + dictWord{10, 10, 446}, + dictWord{ + 10, + 10, + 731, + }, + dictWord{12, 10, 404}, + dictWord{141, 10, 491}, + dictWord{135, 11, 552}, + dictWord{135, 11, 1112}, + dictWord{4, 0, 78}, + dictWord{5, 0, 96}, + dictWord{ + 5, + 0, + 182, + }, + dictWord{6, 0, 1257}, + dictWord{7, 0, 1724}, + dictWord{7, 0, 1825}, + dictWord{10, 0, 394}, + dictWord{10, 0, 471}, + dictWord{11, 0, 532}, + dictWord{ + 14, + 0, + 340, + }, + dictWord{145, 0, 88}, + dictWord{139, 11, 328}, + dictWord{135, 0, 1964}, + dictWord{132, 10, 411}, + dictWord{4, 10, 80}, + dictWord{5, 10, 44}, + dictWord{ + 137, + 11, + 133, + }, + dictWord{5, 11, 110}, + dictWord{6, 11, 169}, + dictWord{6, 11, 1702}, + dictWord{7, 11, 400}, + dictWord{8, 11, 538}, + dictWord{9, 11, 184}, + dictWord{ + 9, + 11, + 524, + }, + dictWord{140, 11, 218}, + dictWord{4, 0, 521}, + dictWord{5, 10, 299}, + dictWord{7, 10, 1083}, + dictWord{140, 11, 554}, + dictWord{6, 11, 133}, + dictWord{ + 9, + 11, + 353, + }, + dictWord{12, 11, 628}, + dictWord{146, 11, 79}, + dictWord{6, 0, 215}, + dictWord{7, 0, 584}, + dictWord{7, 0, 1028}, + dictWord{7, 0, 1473}, + dictWord{ + 7, + 0, + 1721, + }, + dictWord{9, 0, 424}, + dictWord{138, 0, 779}, + dictWord{7, 0, 857}, + dictWord{7, 0, 1209}, + dictWord{7, 10, 1713}, + dictWord{9, 10, 537}, + dictWord{ + 10, + 10, + 165, + }, + dictWord{12, 10, 219}, + dictWord{140, 10, 561}, + dictWord{4, 10, 219}, + dictWord{6, 11, 93}, + dictWord{7, 11, 1422}, + dictWord{7, 10, 1761}, + dictWord{ + 7, + 11, + 1851, + }, + dictWord{8, 11, 673}, + dictWord{9, 10, 86}, + dictWord{9, 11, 529}, + dictWord{140, 11, 43}, + dictWord{137, 11, 371}, + dictWord{136, 0, 671}, + dictWord{ + 5, + 0, + 328, + }, + dictWord{135, 0, 918}, + dictWord{132, 0, 529}, + dictWord{9, 11, 25}, + dictWord{10, 11, 467}, + dictWord{138, 11, 559}, + dictWord{4, 11, 335}, + dictWord{ + 135, + 11, + 942, + }, + dictWord{134, 0, 716}, + dictWord{134, 0, 1509}, + dictWord{6, 0, 67}, + dictWord{7, 0, 258}, + dictWord{7, 0, 1630}, + dictWord{9, 0, 354}, + dictWord{ + 9, + 0, + 675, + }, + dictWord{10, 0, 830}, + dictWord{14, 0, 80}, + dictWord{17, 0, 80}, + dictWord{140, 10, 428}, + dictWord{134, 0, 1112}, + dictWord{6, 0, 141}, + dictWord{7, 0, 225}, + dictWord{9, 0, 59}, + dictWord{9, 0, 607}, + dictWord{10, 0, 312}, + dictWord{11, 0, 687}, + dictWord{12, 0, 555}, + dictWord{13, 0, 373}, + dictWord{13, 0, 494}, + dictWord{ + 148, + 0, + 58, + }, + dictWord{133, 10, 514}, + dictWord{8, 11, 39}, + dictWord{10, 11, 773}, + dictWord{11, 11, 84}, + dictWord{12, 11, 205}, + dictWord{142, 11, 1}, + dictWord{ + 8, + 0, + 783, + }, + dictWord{5, 11, 601}, + dictWord{133, 11, 870}, + dictWord{136, 11, 594}, + dictWord{4, 10, 55}, + dictWord{5, 10, 301}, + dictWord{6, 10, 571}, + dictWord{ + 14, + 10, + 49, + }, + dictWord{146, 10, 102}, + dictWord{132, 11, 181}, + dictWord{134, 11, 1652}, + dictWord{133, 10, 364}, + dictWord{4, 11, 97}, + dictWord{5, 11, 147}, + dictWord{6, 11, 286}, + dictWord{7, 11, 1362}, + dictWord{141, 11, 176}, + dictWord{4, 10, 76}, + dictWord{7, 10, 1550}, + dictWord{9, 10, 306}, + dictWord{9, 10, 430}, + dictWord{9, 10, 663}, + dictWord{10, 10, 683}, + dictWord{11, 10, 427}, + dictWord{11, 10, 753}, + dictWord{12, 10, 334}, + dictWord{12, 10, 442}, + dictWord{ + 14, + 10, + 258, + }, + dictWord{14, 10, 366}, + dictWord{143, 10, 131}, + dictWord{137, 10, 52}, + dictWord{6, 0, 955}, + dictWord{134, 0, 1498}, + dictWord{6, 11, 375}, + dictWord{ + 7, + 11, + 169, + }, + dictWord{7, 11, 254}, + dictWord{136, 11, 780}, + dictWord{7, 0, 430}, + dictWord{11, 0, 46}, + dictWord{14, 0, 343}, + dictWord{142, 11, 343}, + dictWord{ + 135, + 0, + 1183, + }, + dictWord{5, 0, 602}, + dictWord{7, 0, 2018}, + dictWord{9, 0, 418}, + dictWord{9, 0, 803}, + dictWord{135, 11, 1447}, + dictWord{8, 0, 677}, + dictWord{ + 135, + 11, + 1044, + }, + dictWord{139, 11, 285}, + dictWord{4, 10, 656}, + dictWord{135, 10, 779}, + dictWord{135, 10, 144}, + dictWord{5, 11, 629}, + dictWord{ + 135, + 11, + 1549, + }, + dictWord{135, 10, 1373}, + dictWord{138, 11, 209}, + dictWord{7, 10, 554}, + dictWord{7, 10, 605}, + dictWord{141, 10, 10}, + dictWord{5, 10, 838}, + dictWord{ + 5, + 10, + 841, + }, + dictWord{134, 10, 1649}, + dictWord{133, 10, 1012}, + dictWord{6, 0, 1357}, + dictWord{134, 0, 1380}, + dictWord{144, 0, 53}, + dictWord{6, 0, 590}, + dictWord{7, 10, 365}, + dictWord{7, 10, 1357}, + dictWord{7, 10, 1497}, + dictWord{8, 10, 154}, + dictWord{141, 10, 281}, + dictWord{133, 10, 340}, + dictWord{ + 132, + 11, + 420, + }, + dictWord{135, 0, 329}, + dictWord{147, 11, 32}, + dictWord{4, 0, 469}, + dictWord{10, 11, 429}, + dictWord{139, 10, 495}, + dictWord{8, 10, 261}, + dictWord{ + 9, + 10, + 144, + }, + dictWord{9, 10, 466}, + dictWord{10, 10, 370}, + dictWord{12, 10, 470}, + dictWord{13, 10, 144}, + dictWord{142, 10, 348}, + dictWord{142, 0, 460}, + dictWord{4, 11, 325}, + dictWord{9, 10, 897}, + dictWord{138, 11, 125}, + dictWord{6, 0, 1743}, + dictWord{6, 10, 248}, + dictWord{9, 10, 546}, + dictWord{10, 10, 535}, + dictWord{11, 10, 681}, + dictWord{141, 10, 135}, + dictWord{4, 0, 990}, + dictWord{5, 0, 929}, + dictWord{6, 0, 340}, + dictWord{8, 0, 376}, + dictWord{8, 0, 807}, + dictWord{ + 8, + 0, + 963, + }, + dictWord{8, 0, 980}, + dictWord{138, 0, 1007}, + dictWord{134, 0, 1603}, + dictWord{140, 0, 250}, + dictWord{4, 11, 714}, + dictWord{133, 11, 469}, + dictWord{134, 10, 567}, + dictWord{136, 10, 445}, + dictWord{5, 0, 218}, + dictWord{7, 0, 1610}, + dictWord{8, 0, 646}, + dictWord{10, 0, 83}, + dictWord{11, 11, 138}, + dictWord{140, 11, 40}, + dictWord{7, 0, 1512}, + dictWord{135, 0, 1794}, + dictWord{135, 11, 1216}, + dictWord{11, 0, 0}, + dictWord{16, 0, 78}, + dictWord{132, 11, 718}, + dictWord{133, 0, 571}, + dictWord{132, 0, 455}, + dictWord{134, 0, 1012}, + dictWord{5, 11, 124}, + dictWord{5, 11, 144}, + dictWord{6, 11, 548}, + dictWord{7, 11, 15}, + dictWord{7, 11, 153}, + dictWord{137, 11, 629}, + dictWord{142, 11, 10}, + dictWord{6, 11, 75}, + dictWord{7, 11, 1531}, + dictWord{8, 11, 416}, + dictWord{9, 11, 240}, + dictWord{9, 11, 275}, + dictWord{10, 11, 100}, + dictWord{11, 11, 658}, + dictWord{11, 11, 979}, + dictWord{12, 11, 86}, + dictWord{13, 11, 468}, + dictWord{14, 11, 66}, + dictWord{14, 11, 207}, + dictWord{15, 11, 20}, + dictWord{15, 11, 25}, + dictWord{144, 11, 58}, + dictWord{132, 10, 577}, + dictWord{5, 11, 141}, + dictWord{ + 5, + 11, + 915, + }, + dictWord{6, 11, 1783}, + dictWord{7, 11, 211}, + dictWord{7, 11, 698}, + dictWord{7, 11, 1353}, + dictWord{9, 11, 83}, + dictWord{9, 11, 281}, + dictWord{ + 10, + 11, + 376, + }, + dictWord{10, 11, 431}, + dictWord{11, 11, 543}, + dictWord{12, 11, 664}, + dictWord{13, 11, 280}, + dictWord{13, 11, 428}, + dictWord{14, 11, 61}, + dictWord{ + 14, + 11, + 128, + }, + dictWord{17, 11, 52}, + dictWord{145, 11, 81}, + dictWord{6, 0, 161}, + dictWord{7, 0, 372}, + dictWord{137, 0, 597}, + dictWord{132, 0, 349}, + dictWord{ + 10, + 11, + 702, + }, + dictWord{139, 11, 245}, + dictWord{134, 0, 524}, + dictWord{134, 10, 174}, + dictWord{6, 0, 432}, + dictWord{9, 0, 751}, + dictWord{139, 0, 322}, + dictWord{147, 11, 94}, + dictWord{4, 11, 338}, + dictWord{133, 11, 400}, + dictWord{5, 0, 468}, + dictWord{10, 0, 325}, + dictWord{11, 0, 856}, + dictWord{12, 0, 345}, + dictWord{143, 0, 104}, + dictWord{133, 0, 223}, + dictWord{132, 0, 566}, + dictWord{4, 11, 221}, + dictWord{5, 11, 659}, + dictWord{5, 11, 989}, + dictWord{7, 11, 697}, + dictWord{7, 11, 1211}, + dictWord{138, 11, 284}, + dictWord{135, 11, 1070}, + dictWord{4, 0, 59}, + dictWord{135, 0, 1394}, + dictWord{6, 0, 436}, + dictWord{11, 0, 481}, + dictWord{5, 10, 878}, + dictWord{133, 10, 972}, + dictWord{4, 0, 48}, + dictWord{5, 0, 271}, + dictWord{135, 0, 953}, + dictWord{5, 0, 610}, + dictWord{136, 0, 457}, + dictWord{ + 4, + 0, + 773, + }, + dictWord{5, 0, 618}, + dictWord{137, 0, 756}, + dictWord{133, 0, 755}, + dictWord{135, 0, 1217}, + dictWord{138, 11, 507}, + dictWord{132, 10, 351}, + dictWord{132, 0, 197}, + dictWord{143, 11, 78}, + dictWord{4, 11, 188}, + dictWord{7, 11, 805}, + dictWord{11, 11, 276}, + dictWord{142, 11, 293}, + dictWord{ + 5, + 11, + 884, + }, + dictWord{139, 11, 991}, + dictWord{132, 10, 286}, + dictWord{10, 0, 259}, + dictWord{10, 0, 428}, + dictWord{7, 10, 438}, + dictWord{7, 10, 627}, + dictWord{ + 7, + 10, + 1516, + }, + dictWord{8, 10, 40}, + dictWord{9, 10, 56}, + dictWord{9, 10, 294}, + dictWord{11, 10, 969}, + dictWord{11, 10, 995}, + dictWord{146, 10, 148}, + dictWord{ + 4, + 0, + 356, + }, + dictWord{5, 0, 217}, + dictWord{5, 0, 492}, + dictWord{5, 0, 656}, + dictWord{8, 0, 544}, + dictWord{136, 11, 544}, + dictWord{5, 0, 259}, + dictWord{6, 0, 1230}, + dictWord{7, 0, 414}, + dictWord{7, 0, 854}, + dictWord{142, 0, 107}, + dictWord{132, 0, 1007}, + dictWord{15, 0, 14}, + dictWord{144, 0, 5}, + dictWord{6, 0, 1580}, + dictWord{ + 132, + 10, + 738, + }, + dictWord{132, 11, 596}, + dictWord{132, 0, 673}, + dictWord{133, 10, 866}, + dictWord{6, 0, 1843}, + dictWord{135, 11, 1847}, + dictWord{4, 0, 165}, + dictWord{7, 0, 1398}, + dictWord{135, 0, 1829}, + dictWord{135, 11, 1634}, + dictWord{147, 11, 65}, + dictWord{6, 0, 885}, + dictWord{6, 0, 1009}, + dictWord{ + 137, + 0, + 809, + }, + dictWord{133, 10, 116}, + dictWord{132, 10, 457}, + dictWord{136, 11, 770}, + dictWord{9, 0, 498}, + dictWord{12, 0, 181}, + dictWord{10, 11, 361}, + dictWord{142, 11, 316}, + dictWord{134, 11, 595}, + dictWord{5, 0, 9}, + dictWord{7, 0, 297}, + dictWord{7, 0, 966}, + dictWord{140, 0, 306}, + dictWord{4, 11, 89}, + dictWord{ + 5, + 11, + 489, + }, + dictWord{6, 11, 315}, + dictWord{7, 11, 553}, + dictWord{7, 11, 1745}, + dictWord{138, 11, 243}, + dictWord{134, 0, 1487}, + dictWord{132, 0, 437}, + dictWord{ + 5, + 0, + 146, + }, + dictWord{6, 0, 411}, + dictWord{138, 0, 721}, + dictWord{5, 10, 527}, + dictWord{6, 10, 189}, + dictWord{135, 10, 859}, + dictWord{11, 10, 104}, + dictWord{ + 11, + 10, + 554, + }, + dictWord{15, 10, 60}, + dictWord{143, 10, 125}, + dictWord{6, 11, 1658}, + dictWord{9, 11, 3}, + dictWord{10, 11, 154}, + dictWord{11, 11, 641}, + dictWord{13, 11, 85}, + dictWord{13, 11, 201}, + dictWord{141, 11, 346}, + dictWord{6, 0, 177}, + dictWord{135, 0, 467}, + dictWord{134, 0, 1377}, + dictWord{ + 134, + 10, + 116, + }, + dictWord{136, 11, 645}, + dictWord{4, 11, 166}, + dictWord{5, 11, 505}, + dictWord{6, 11, 1670}, + dictWord{137, 11, 110}, + dictWord{133, 10, 487}, + dictWord{ + 4, + 10, + 86, + }, + dictWord{5, 10, 667}, + dictWord{5, 10, 753}, + dictWord{6, 10, 316}, + dictWord{6, 10, 455}, + dictWord{135, 10, 946}, + dictWord{133, 0, 200}, + dictWord{132, 0, 959}, + dictWord{6, 0, 1928}, + dictWord{134, 0, 1957}, + dictWord{139, 11, 203}, + dictWord{150, 10, 45}, + dictWord{4, 10, 79}, + dictWord{7, 10, 1773}, + dictWord{10, 10, 450}, + dictWord{11, 10, 589}, + dictWord{13, 10, 332}, + dictWord{13, 10, 493}, + dictWord{14, 10, 183}, + dictWord{14, 10, 334}, + dictWord{ + 14, + 10, + 362, + }, + dictWord{14, 10, 368}, + dictWord{14, 10, 376}, + dictWord{14, 10, 379}, + dictWord{19, 10, 90}, + dictWord{19, 10, 103}, + dictWord{19, 10, 127}, + dictWord{148, 10, 90}, + dictWord{6, 0, 1435}, + dictWord{135, 11, 1275}, + dictWord{134, 0, 481}, + dictWord{7, 11, 445}, + dictWord{8, 11, 307}, + dictWord{8, 11, 704}, + dictWord{10, 11, 41}, + dictWord{10, 11, 439}, + dictWord{11, 11, 237}, + dictWord{11, 11, 622}, + dictWord{140, 11, 201}, + dictWord{135, 11, 869}, + dictWord{ + 4, + 0, + 84, + }, + dictWord{7, 0, 1482}, + dictWord{10, 0, 76}, + dictWord{138, 0, 142}, + dictWord{11, 11, 277}, + dictWord{144, 11, 14}, + dictWord{135, 11, 1977}, + dictWord{ + 4, + 11, + 189, + }, + dictWord{5, 11, 713}, + dictWord{136, 11, 57}, + dictWord{133, 0, 1015}, + dictWord{138, 11, 371}, + dictWord{4, 0, 315}, + dictWord{5, 0, 507}, + dictWord{ + 135, + 0, + 1370, + }, + dictWord{4, 11, 552}, + dictWord{142, 10, 381}, + dictWord{9, 0, 759}, + dictWord{16, 0, 31}, + dictWord{16, 0, 39}, + dictWord{16, 0, 75}, + dictWord{18, 0, 24}, + dictWord{20, 0, 42}, + dictWord{152, 0, 1}, + dictWord{134, 0, 712}, + dictWord{134, 0, 1722}, + dictWord{133, 10, 663}, + dictWord{133, 10, 846}, + dictWord{ + 8, + 0, + 222, + }, + dictWord{8, 0, 476}, + dictWord{9, 0, 238}, + dictWord{11, 0, 516}, + dictWord{11, 0, 575}, + dictWord{15, 0, 109}, + dictWord{146, 0, 100}, + dictWord{7, 0, 1402}, + dictWord{7, 0, 1414}, + dictWord{12, 0, 456}, + dictWord{5, 10, 378}, + dictWord{8, 10, 465}, + dictWord{9, 10, 286}, + dictWord{10, 10, 185}, + dictWord{10, 10, 562}, + dictWord{10, 10, 635}, + dictWord{11, 10, 31}, + dictWord{11, 10, 393}, + dictWord{13, 10, 312}, + dictWord{18, 10, 65}, + dictWord{18, 10, 96}, + dictWord{147, 10, 89}, + dictWord{4, 0, 986}, + dictWord{6, 0, 1958}, + dictWord{6, 0, 2032}, + dictWord{8, 0, 934}, + dictWord{138, 0, 985}, + dictWord{7, 10, 1880}, + dictWord{9, 10, 680}, + dictWord{139, 10, 798}, + dictWord{134, 10, 1770}, + dictWord{145, 11, 49}, + dictWord{132, 11, 614}, + dictWord{132, 10, 648}, + dictWord{5, 10, 945}, + dictWord{ + 6, + 10, + 1656, + }, + dictWord{6, 10, 1787}, + dictWord{7, 10, 167}, + dictWord{8, 10, 824}, + dictWord{9, 10, 391}, + dictWord{10, 10, 375}, + dictWord{139, 10, 185}, + dictWord{138, 11, 661}, + dictWord{7, 0, 1273}, + dictWord{135, 11, 1945}, + dictWord{7, 0, 706}, + dictWord{7, 0, 1058}, + dictWord{138, 0, 538}, + dictWord{7, 10, 1645}, + dictWord{8, 10, 352}, + dictWord{137, 10, 249}, + dictWord{132, 10, 152}, + dictWord{11, 0, 92}, + dictWord{11, 0, 196}, + dictWord{11, 0, 409}, + dictWord{11, 0, 450}, + dictWord{11, 0, 666}, + dictWord{11, 0, 777}, + dictWord{12, 0, 262}, + dictWord{13, 0, 385}, + dictWord{13, 0, 393}, + dictWord{15, 0, 115}, + dictWord{16, 0, 45}, + dictWord{145, 0, 82}, + dictWord{133, 10, 1006}, + dictWord{6, 0, 40}, + dictWord{135, 0, 1781}, + dictWord{9, 11, 614}, + dictWord{139, 11, 327}, + dictWord{5, 10, 420}, + dictWord{135, 10, 1449}, + dictWord{135, 0, 431}, + dictWord{10, 0, 97}, + dictWord{135, 10, 832}, + dictWord{6, 0, 423}, + dictWord{7, 0, 665}, + dictWord{ + 135, + 0, + 1210, + }, + dictWord{7, 0, 237}, + dictWord{8, 0, 664}, + dictWord{9, 0, 42}, + dictWord{9, 0, 266}, + dictWord{9, 0, 380}, + dictWord{9, 0, 645}, + dictWord{10, 0, 177}, + dictWord{ + 138, + 0, + 276, + }, + dictWord{7, 0, 264}, + dictWord{133, 10, 351}, + dictWord{8, 0, 213}, + dictWord{5, 10, 40}, + dictWord{7, 10, 598}, + dictWord{7, 10, 1638}, + dictWord{ + 9, + 10, + 166, + }, + dictWord{9, 10, 640}, + dictWord{9, 10, 685}, + dictWord{9, 10, 773}, + dictWord{11, 10, 215}, + dictWord{13, 10, 65}, + dictWord{14, 10, 172}, + dictWord{ + 14, + 10, + 317, + }, + dictWord{145, 10, 6}, + dictWord{5, 11, 84}, + dictWord{134, 11, 163}, + dictWord{8, 10, 60}, + dictWord{9, 10, 343}, + dictWord{139, 10, 769}, + dictWord{ + 137, + 0, + 455, + }, + dictWord{133, 11, 410}, + dictWord{8, 0, 906}, + dictWord{12, 0, 700}, + dictWord{12, 0, 706}, + dictWord{140, 0, 729}, + dictWord{21, 11, 33}, + dictWord{ + 150, + 11, + 40, + }, + dictWord{7, 10, 1951}, + dictWord{8, 10, 765}, + dictWord{8, 10, 772}, + dictWord{140, 10, 671}, + dictWord{7, 10, 108}, + dictWord{8, 10, 219}, + dictWord{ + 8, + 10, + 388, + }, + dictWord{9, 10, 639}, + dictWord{9, 10, 775}, + dictWord{11, 10, 275}, + dictWord{140, 10, 464}, + dictWord{5, 11, 322}, + dictWord{7, 11, 1941}, + dictWord{ + 8, + 11, + 186, + }, + dictWord{9, 11, 262}, + dictWord{10, 11, 187}, + dictWord{14, 11, 208}, + dictWord{146, 11, 130}, + dictWord{139, 0, 624}, + dictWord{8, 0, 574}, + dictWord{ + 5, + 11, + 227, + }, + dictWord{140, 11, 29}, + dictWord{7, 11, 1546}, + dictWord{11, 11, 299}, + dictWord{142, 11, 407}, + dictWord{5, 10, 15}, + dictWord{6, 10, 56}, + dictWord{ + 7, + 10, + 1758, + }, + dictWord{8, 10, 500}, + dictWord{9, 10, 730}, + dictWord{11, 10, 331}, + dictWord{13, 10, 150}, + dictWord{142, 10, 282}, + dictWord{7, 11, 1395}, + dictWord{8, 11, 486}, + dictWord{9, 11, 236}, + dictWord{9, 11, 878}, + dictWord{10, 11, 218}, + dictWord{11, 11, 95}, + dictWord{19, 11, 17}, + dictWord{147, 11, 31}, + dictWord{135, 11, 2043}, + dictWord{4, 0, 354}, + dictWord{146, 11, 4}, + dictWord{140, 11, 80}, + dictWord{135, 0, 1558}, + dictWord{134, 10, 1886}, + dictWord{ + 5, + 10, + 205, + }, + dictWord{6, 10, 438}, + dictWord{137, 10, 711}, + dictWord{133, 11, 522}, + dictWord{133, 10, 534}, + dictWord{7, 0, 235}, + dictWord{7, 0, 1475}, + dictWord{ + 15, + 0, + 68, + }, + dictWord{146, 0, 120}, + dictWord{137, 10, 691}, + dictWord{4, 0, 942}, + dictWord{6, 0, 1813}, + dictWord{8, 0, 917}, + dictWord{10, 0, 884}, + dictWord{ + 12, + 0, + 696, + }, + dictWord{12, 0, 717}, + dictWord{12, 0, 723}, + dictWord{12, 0, 738}, + dictWord{12, 0, 749}, + dictWord{12, 0, 780}, + dictWord{16, 0, 97}, + dictWord{146, 0, 169}, + dictWord{6, 10, 443}, + dictWord{8, 11, 562}, + dictWord{9, 10, 237}, + dictWord{9, 10, 571}, + dictWord{9, 10, 695}, + dictWord{10, 10, 139}, + dictWord{11, 10, 715}, + dictWord{12, 10, 417}, + dictWord{141, 10, 421}, + dictWord{135, 0, 957}, + dictWord{133, 0, 830}, + dictWord{134, 11, 1771}, + dictWord{146, 0, 23}, + dictWord{ + 5, + 0, + 496, + }, + dictWord{6, 0, 694}, + dictWord{7, 0, 203}, + dictWord{7, 11, 1190}, + dictWord{137, 11, 620}, + dictWord{137, 11, 132}, + dictWord{6, 0, 547}, + dictWord{ + 134, + 0, + 1549, + }, + dictWord{8, 11, 258}, + dictWord{9, 11, 208}, + dictWord{137, 11, 359}, + dictWord{4, 0, 864}, + dictWord{5, 0, 88}, + dictWord{137, 0, 239}, + dictWord{ + 135, + 11, + 493, + }, + dictWord{4, 11, 317}, + dictWord{135, 11, 1279}, + dictWord{132, 11, 477}, + dictWord{4, 10, 578}, + dictWord{5, 11, 63}, + dictWord{133, 11, 509}, + dictWord{ + 7, + 0, + 650, + }, + dictWord{135, 0, 1310}, + dictWord{7, 0, 1076}, + dictWord{9, 0, 80}, + dictWord{11, 0, 78}, + dictWord{11, 0, 421}, + dictWord{11, 0, 534}, + dictWord{ + 140, + 0, + 545, + }, + dictWord{132, 11, 288}, + dictWord{12, 0, 553}, + dictWord{14, 0, 118}, + dictWord{133, 10, 923}, + dictWord{7, 0, 274}, + dictWord{11, 0, 479}, + dictWord{ + 139, + 0, + 507, + }, + dictWord{8, 11, 89}, + dictWord{8, 11, 620}, + dictWord{9, 11, 49}, + dictWord{10, 11, 774}, + dictWord{11, 11, 628}, + dictWord{12, 11, 322}, + dictWord{ + 143, + 11, + 124, + }, + dictWord{4, 0, 497}, + dictWord{135, 0, 1584}, + dictWord{7, 0, 261}, + dictWord{7, 0, 1115}, + dictWord{7, 0, 1354}, + dictWord{7, 0, 1404}, + dictWord{ + 7, + 0, + 1588, + }, + dictWord{7, 0, 1705}, + dictWord{7, 0, 1902}, + dictWord{9, 0, 465}, + dictWord{10, 0, 248}, + dictWord{10, 0, 349}, + dictWord{10, 0, 647}, + dictWord{11, 0, 527}, + dictWord{11, 0, 660}, + dictWord{11, 0, 669}, + dictWord{12, 0, 529}, + dictWord{13, 0, 305}, + dictWord{132, 10, 924}, + dictWord{133, 10, 665}, + dictWord{ + 136, + 0, + 13, + }, + dictWord{6, 0, 791}, + dictWord{138, 11, 120}, + dictWord{7, 0, 642}, + dictWord{8, 0, 250}, + dictWord{11, 0, 123}, + dictWord{11, 0, 137}, + dictWord{13, 0, 48}, + dictWord{142, 0, 95}, + dictWord{4, 10, 265}, + dictWord{7, 10, 807}, + dictWord{135, 10, 950}, + dictWord{5, 10, 93}, + dictWord{140, 10, 267}, + dictWord{135, 0, 1429}, + dictWord{4, 0, 949}, + dictWord{10, 0, 885}, + dictWord{10, 0, 891}, + dictWord{10, 0, 900}, + dictWord{10, 0, 939}, + dictWord{12, 0, 760}, + dictWord{142, 0, 449}, + dictWord{139, 11, 366}, + dictWord{132, 0, 818}, + dictWord{134, 11, 85}, + dictWord{135, 10, 994}, + dictWord{7, 0, 330}, + dictWord{5, 10, 233}, + dictWord{5, 10, 320}, + dictWord{6, 10, 140}, + dictWord{136, 10, 295}, + dictWord{4, 0, 1004}, + dictWord{8, 0, 982}, + dictWord{136, 0, 993}, + dictWord{133, 10, 978}, + dictWord{4, 10, 905}, + dictWord{6, 10, 1701}, + dictWord{137, 10, 843}, + dictWord{10, 0, 545}, + dictWord{140, 0, 301}, + dictWord{6, 0, 947}, + dictWord{134, 0, 1062}, + dictWord{ + 134, + 0, + 1188, + }, + dictWord{4, 0, 904}, + dictWord{5, 0, 794}, + dictWord{152, 10, 6}, + dictWord{134, 0, 1372}, + dictWord{135, 11, 608}, + dictWord{5, 11, 279}, + dictWord{ + 6, + 11, + 235, + }, + dictWord{7, 11, 468}, + dictWord{8, 11, 446}, + dictWord{9, 11, 637}, + dictWord{10, 11, 717}, + dictWord{11, 11, 738}, + dictWord{140, 11, 514}, + dictWord{ + 132, + 10, + 509, + }, + dictWord{5, 11, 17}, + dictWord{6, 11, 371}, + dictWord{137, 11, 528}, + dictWord{132, 0, 693}, + dictWord{4, 11, 115}, + dictWord{5, 11, 669}, + dictWord{ + 6, + 11, + 407, + }, + dictWord{8, 11, 311}, + dictWord{11, 11, 10}, + dictWord{141, 11, 5}, + dictWord{11, 0, 377}, + dictWord{7, 10, 273}, + dictWord{137, 11, 381}, + dictWord{ + 135, + 0, + 695, + }, + dictWord{7, 0, 386}, + dictWord{138, 0, 713}, + dictWord{135, 10, 1041}, + dictWord{134, 0, 1291}, + dictWord{6, 0, 7}, + dictWord{6, 0, 35}, + dictWord{ + 7, + 0, + 147, + }, + dictWord{7, 0, 1069}, + dictWord{7, 0, 1568}, + dictWord{7, 0, 1575}, + dictWord{7, 0, 1917}, + dictWord{8, 0, 43}, + dictWord{8, 0, 208}, + dictWord{9, 0, 128}, + dictWord{ + 9, + 0, + 866, + }, + dictWord{10, 0, 20}, + dictWord{11, 0, 981}, + dictWord{147, 0, 33}, + dictWord{7, 0, 893}, + dictWord{141, 0, 424}, + dictWord{139, 10, 234}, + dictWord{ + 150, + 11, + 56, + }, + dictWord{5, 11, 779}, + dictWord{5, 11, 807}, + dictWord{6, 11, 1655}, + dictWord{134, 11, 1676}, + dictWord{5, 10, 802}, + dictWord{7, 10, 2021}, + dictWord{136, 10, 805}, + dictWord{4, 11, 196}, + dictWord{5, 10, 167}, + dictWord{5, 11, 558}, + dictWord{5, 10, 899}, + dictWord{5, 11, 949}, + dictWord{6, 10, 410}, + dictWord{137, 10, 777}, + dictWord{137, 10, 789}, + dictWord{134, 10, 1705}, + dictWord{8, 0, 904}, + dictWord{140, 0, 787}, + dictWord{6, 0, 322}, + dictWord{9, 0, 552}, + dictWord{11, 0, 274}, + dictWord{13, 0, 209}, + dictWord{13, 0, 499}, + dictWord{14, 0, 85}, + dictWord{15, 0, 126}, + dictWord{145, 0, 70}, + dictWord{135, 10, 10}, + dictWord{ + 5, + 10, + 11, + }, + dictWord{6, 10, 117}, + dictWord{6, 10, 485}, + dictWord{7, 10, 1133}, + dictWord{9, 10, 582}, + dictWord{9, 10, 594}, + dictWord{11, 10, 21}, + dictWord{ + 11, + 10, + 818, + }, + dictWord{12, 10, 535}, + dictWord{141, 10, 86}, + dictWord{4, 10, 264}, + dictWord{7, 10, 1067}, + dictWord{8, 10, 204}, + dictWord{8, 10, 385}, + dictWord{139, 10, 953}, + dictWord{132, 11, 752}, + dictWord{138, 10, 56}, + dictWord{133, 10, 470}, + dictWord{6, 0, 1808}, + dictWord{8, 0, 83}, + dictWord{8, 0, 742}, + dictWord{8, 0, 817}, + dictWord{9, 0, 28}, + dictWord{9, 0, 29}, + dictWord{9, 0, 885}, + dictWord{10, 0, 387}, + dictWord{11, 0, 633}, + dictWord{11, 0, 740}, + dictWord{13, 0, 235}, + dictWord{13, 0, 254}, + dictWord{15, 0, 143}, + dictWord{143, 0, 146}, + dictWord{140, 0, 49}, + dictWord{134, 0, 1832}, + dictWord{4, 11, 227}, + dictWord{5, 11, 159}, + dictWord{5, 11, 409}, + dictWord{7, 11, 80}, + dictWord{10, 11, 294}, + dictWord{10, 11, 479}, + dictWord{12, 11, 418}, + dictWord{14, 11, 50}, + dictWord{14, 11, 249}, + dictWord{142, 11, 295}, + dictWord{7, 11, 1470}, + dictWord{8, 11, 66}, + dictWord{8, 11, 137}, + dictWord{8, 11, 761}, + dictWord{9, 11, 638}, + dictWord{11, 11, 80}, + dictWord{11, 11, 212}, + dictWord{11, 11, 368}, + dictWord{11, 11, 418}, + dictWord{12, 11, 8}, + dictWord{13, 11, 15}, + dictWord{16, 11, 61}, + dictWord{17, 11, 59}, + dictWord{19, 11, 28}, + dictWord{148, 11, 84}, + dictWord{139, 10, 1015}, + dictWord{138, 11, 468}, + dictWord{135, 0, 421}, + dictWord{6, 0, 415}, + dictWord{ + 7, + 0, + 1049, + }, + dictWord{137, 0, 442}, + dictWord{6, 11, 38}, + dictWord{7, 11, 1220}, + dictWord{8, 11, 185}, + dictWord{8, 11, 256}, + dictWord{9, 11, 22}, + dictWord{ + 9, + 11, + 331, + }, + dictWord{10, 11, 738}, + dictWord{11, 11, 205}, + dictWord{11, 11, 540}, + dictWord{11, 11, 746}, + dictWord{13, 11, 399}, + dictWord{13, 11, 465}, + dictWord{ + 14, + 11, + 88, + }, + dictWord{142, 11, 194}, + dictWord{139, 0, 289}, + dictWord{133, 10, 715}, + dictWord{4, 0, 110}, + dictWord{10, 0, 415}, + dictWord{10, 0, 597}, + dictWord{142, 0, 206}, + dictWord{4, 11, 159}, + dictWord{6, 11, 115}, + dictWord{7, 11, 252}, + dictWord{7, 11, 257}, + dictWord{7, 11, 1928}, + dictWord{8, 11, 69}, + dictWord{ + 9, + 11, + 384, + }, + dictWord{10, 11, 91}, + dictWord{10, 11, 615}, + dictWord{12, 11, 375}, + dictWord{14, 11, 235}, + dictWord{18, 11, 117}, + dictWord{147, 11, 123}, + dictWord{5, 11, 911}, + dictWord{136, 11, 278}, + dictWord{7, 0, 205}, + dictWord{7, 0, 2000}, + dictWord{8, 10, 794}, + dictWord{9, 10, 400}, + dictWord{10, 10, 298}, + dictWord{142, 10, 228}, + dictWord{135, 11, 1774}, + dictWord{4, 11, 151}, + dictWord{7, 11, 1567}, + dictWord{8, 11, 351}, + dictWord{137, 11, 322}, + dictWord{ + 136, + 10, + 724, + }, + dictWord{133, 11, 990}, + dictWord{7, 0, 1539}, + dictWord{11, 0, 512}, + dictWord{13, 0, 205}, + dictWord{19, 0, 30}, + dictWord{22, 0, 36}, + dictWord{23, 0, 19}, + dictWord{135, 11, 1539}, + dictWord{5, 11, 194}, + dictWord{7, 11, 1662}, + dictWord{9, 11, 90}, + dictWord{140, 11, 180}, + dictWord{6, 10, 190}, + dictWord{ + 7, + 10, + 768, + }, + dictWord{135, 10, 1170}, + dictWord{134, 0, 1340}, + dictWord{4, 0, 283}, + dictWord{135, 0, 1194}, + dictWord{133, 11, 425}, + dictWord{133, 11, 971}, + dictWord{12, 0, 549}, + dictWord{14, 10, 67}, + dictWord{147, 10, 60}, + dictWord{135, 10, 1023}, + dictWord{134, 0, 1720}, + dictWord{138, 11, 587}, + dictWord{ + 5, + 11, + 72, + }, + dictWord{6, 11, 264}, + dictWord{7, 11, 21}, + dictWord{7, 11, 46}, + dictWord{7, 11, 2013}, + dictWord{8, 11, 215}, + dictWord{8, 11, 513}, + dictWord{10, 11, 266}, + dictWord{139, 11, 22}, + dictWord{5, 0, 319}, + dictWord{135, 0, 534}, + dictWord{6, 10, 137}, + dictWord{9, 10, 75}, + dictWord{9, 10, 253}, + dictWord{10, 10, 194}, + dictWord{138, 10, 444}, + dictWord{7, 0, 1180}, + dictWord{20, 0, 112}, + dictWord{6, 11, 239}, + dictWord{7, 11, 118}, + dictWord{10, 11, 95}, + dictWord{11, 11, 603}, + dictWord{13, 11, 443}, + dictWord{14, 11, 160}, + dictWord{143, 11, 4}, + dictWord{134, 11, 431}, + dictWord{5, 11, 874}, + dictWord{6, 11, 1677}, + dictWord{ + 11, + 10, + 643, + }, + dictWord{12, 10, 115}, + dictWord{143, 11, 0}, + dictWord{134, 0, 967}, + dictWord{6, 11, 65}, + dictWord{7, 11, 939}, + dictWord{7, 11, 1172}, + dictWord{ + 7, + 11, + 1671, + }, + dictWord{9, 11, 540}, + dictWord{10, 11, 696}, + dictWord{11, 11, 265}, + dictWord{11, 11, 732}, + dictWord{11, 11, 928}, + dictWord{11, 11, 937}, + dictWord{ + 12, + 11, + 399, + }, + dictWord{13, 11, 438}, + dictWord{149, 11, 19}, + dictWord{137, 11, 200}, + dictWord{135, 0, 1940}, + dictWord{5, 10, 760}, + dictWord{7, 10, 542}, + dictWord{8, 10, 135}, + dictWord{136, 10, 496}, + dictWord{140, 11, 44}, + dictWord{7, 11, 1655}, + dictWord{136, 11, 305}, + dictWord{7, 10, 319}, + dictWord{ + 7, + 10, + 355, + }, + dictWord{7, 10, 763}, + dictWord{10, 10, 389}, + dictWord{145, 10, 43}, + dictWord{136, 0, 735}, + dictWord{138, 10, 786}, + dictWord{137, 11, 19}, + dictWord{132, 11, 696}, + dictWord{5, 0, 132}, + dictWord{9, 0, 486}, + dictWord{9, 0, 715}, + dictWord{10, 0, 458}, + dictWord{11, 0, 373}, + dictWord{11, 0, 668}, + dictWord{ + 11, + 0, + 795, + }, + dictWord{11, 0, 897}, + dictWord{12, 0, 272}, + dictWord{12, 0, 424}, + dictWord{12, 0, 539}, + dictWord{12, 0, 558}, + dictWord{14, 0, 245}, + dictWord{ + 14, + 0, + 263, + }, + dictWord{14, 0, 264}, + dictWord{14, 0, 393}, + dictWord{142, 0, 403}, + dictWord{10, 0, 38}, + dictWord{139, 0, 784}, + dictWord{132, 0, 838}, + dictWord{ + 4, + 11, + 302, + }, + dictWord{135, 11, 1766}, + dictWord{133, 0, 379}, + dictWord{5, 0, 8}, + dictWord{6, 0, 89}, + dictWord{6, 0, 400}, + dictWord{7, 0, 1569}, + dictWord{7, 0, 1623}, + dictWord{7, 0, 1850}, + dictWord{8, 0, 218}, + dictWord{8, 0, 422}, + dictWord{9, 0, 570}, + dictWord{10, 0, 626}, + dictWord{4, 11, 726}, + dictWord{133, 11, 630}, + dictWord{ + 4, + 0, + 1017, + }, + dictWord{138, 0, 660}, + dictWord{6, 0, 387}, + dictWord{7, 0, 882}, + dictWord{141, 0, 111}, + dictWord{6, 0, 224}, + dictWord{7, 0, 877}, + dictWord{ + 137, + 0, + 647, + }, + dictWord{4, 10, 58}, + dictWord{5, 10, 286}, + dictWord{6, 10, 319}, + dictWord{7, 10, 402}, + dictWord{7, 10, 1254}, + dictWord{7, 10, 1903}, + dictWord{ + 8, + 10, + 356, + }, + dictWord{140, 10, 408}, + dictWord{135, 0, 790}, + dictWord{9, 0, 510}, + dictWord{10, 0, 53}, + dictWord{4, 10, 389}, + dictWord{9, 10, 181}, + dictWord{ + 10, + 10, + 29, + }, + dictWord{10, 10, 816}, + dictWord{11, 10, 311}, + dictWord{11, 10, 561}, + dictWord{12, 10, 67}, + dictWord{141, 10, 181}, + dictWord{142, 0, 458}, + dictWord{ + 6, + 11, + 118, + }, + dictWord{7, 11, 215}, + dictWord{7, 11, 1521}, + dictWord{140, 11, 11}, + dictWord{134, 0, 954}, + dictWord{135, 0, 394}, + dictWord{134, 0, 1367}, + dictWord{5, 11, 225}, + dictWord{133, 10, 373}, + dictWord{132, 0, 882}, + dictWord{7, 0, 1409}, + dictWord{135, 10, 1972}, + dictWord{135, 10, 1793}, + dictWord{ + 4, + 11, + 370, + }, + dictWord{5, 11, 756}, + dictWord{135, 11, 1326}, + dictWord{150, 11, 13}, + dictWord{7, 11, 354}, + dictWord{10, 11, 410}, + dictWord{139, 11, 815}, + dictWord{6, 11, 1662}, + dictWord{7, 11, 48}, + dictWord{8, 11, 771}, + dictWord{10, 11, 116}, + dictWord{13, 11, 104}, + dictWord{14, 11, 105}, + dictWord{14, 11, 184}, + dictWord{15, 11, 168}, + dictWord{19, 11, 92}, + dictWord{148, 11, 68}, + dictWord{7, 0, 124}, + dictWord{136, 0, 38}, + dictWord{5, 0, 261}, + dictWord{7, 0, 78}, + dictWord{ + 7, + 0, + 199, + }, + dictWord{8, 0, 815}, + dictWord{9, 0, 126}, + dictWord{10, 0, 342}, + dictWord{140, 0, 647}, + dictWord{4, 0, 628}, + dictWord{140, 0, 724}, + dictWord{7, 0, 266}, + dictWord{8, 0, 804}, + dictWord{7, 10, 1651}, + dictWord{145, 10, 89}, + dictWord{135, 0, 208}, + dictWord{134, 0, 1178}, + dictWord{6, 0, 79}, + dictWord{135, 0, 1519}, + dictWord{132, 10, 672}, + dictWord{133, 10, 737}, + dictWord{136, 0, 741}, + dictWord{132, 11, 120}, + dictWord{4, 0, 710}, + dictWord{6, 0, 376}, + dictWord{ + 134, + 0, + 606, + }, + dictWord{134, 0, 1347}, + dictWord{134, 0, 1494}, + dictWord{6, 0, 850}, + dictWord{6, 0, 1553}, + dictWord{137, 0, 821}, + dictWord{5, 10, 145}, + dictWord{ + 134, + 11, + 593, + }, + dictWord{7, 0, 1311}, + dictWord{140, 0, 135}, + dictWord{4, 0, 467}, + dictWord{5, 0, 405}, + dictWord{134, 0, 544}, + dictWord{5, 11, 820}, + dictWord{ + 135, + 11, + 931, + }, + dictWord{6, 0, 100}, + dictWord{7, 0, 244}, + dictWord{7, 0, 632}, + dictWord{7, 0, 1609}, + dictWord{8, 0, 178}, + dictWord{8, 0, 638}, + dictWord{141, 0, 58}, + dictWord{4, 10, 387}, + dictWord{135, 10, 1288}, + dictWord{6, 11, 151}, + dictWord{6, 11, 1675}, + dictWord{7, 11, 383}, + dictWord{151, 11, 10}, + dictWord{ + 132, + 0, + 481, + }, + dictWord{135, 10, 550}, + dictWord{134, 0, 1378}, + dictWord{6, 11, 1624}, + dictWord{11, 11, 11}, + dictWord{12, 11, 422}, + dictWord{13, 11, 262}, + dictWord{142, 11, 360}, + dictWord{133, 0, 791}, + dictWord{4, 11, 43}, + dictWord{5, 11, 344}, + dictWord{133, 11, 357}, + dictWord{7, 0, 1227}, + dictWord{140, 0, 978}, + dictWord{7, 0, 686}, + dictWord{8, 0, 33}, + dictWord{8, 0, 238}, + dictWord{10, 0, 616}, + dictWord{11, 0, 467}, + dictWord{11, 0, 881}, + dictWord{13, 0, 217}, + dictWord{ + 13, + 0, + 253, + }, + dictWord{142, 0, 268}, + dictWord{137, 0, 857}, + dictWord{8, 0, 467}, + dictWord{8, 0, 1006}, + dictWord{7, 11, 148}, + dictWord{8, 11, 284}, + dictWord{ + 141, + 11, + 63, + }, + dictWord{4, 10, 576}, + dictWord{135, 10, 1263}, + dictWord{133, 11, 888}, + dictWord{5, 10, 919}, + dictWord{134, 10, 1673}, + dictWord{20, 10, 37}, + dictWord{148, 11, 37}, + dictWord{132, 0, 447}, + dictWord{132, 11, 711}, + dictWord{4, 0, 128}, + dictWord{5, 0, 415}, + dictWord{6, 0, 462}, + dictWord{7, 0, 294}, + dictWord{ + 7, + 0, + 578, + }, + dictWord{10, 0, 710}, + dictWord{139, 0, 86}, + dictWord{4, 10, 82}, + dictWord{5, 10, 333}, + dictWord{5, 10, 904}, + dictWord{6, 10, 207}, + dictWord{7, 10, 325}, + dictWord{7, 10, 1726}, + dictWord{8, 10, 101}, + dictWord{10, 10, 778}, + dictWord{139, 10, 220}, + dictWord{136, 0, 587}, + dictWord{137, 11, 440}, + dictWord{ + 133, + 10, + 903, + }, + dictWord{6, 0, 427}, + dictWord{7, 0, 1018}, + dictWord{138, 0, 692}, + dictWord{4, 0, 195}, + dictWord{135, 0, 802}, + dictWord{140, 10, 147}, + dictWord{ + 134, + 0, + 1546, + }, + dictWord{134, 0, 684}, + dictWord{132, 10, 705}, + dictWord{136, 0, 345}, + dictWord{11, 11, 678}, + dictWord{140, 11, 307}, + dictWord{ + 133, + 0, + 365, + }, + dictWord{134, 0, 1683}, + dictWord{4, 11, 65}, + dictWord{5, 11, 479}, + dictWord{5, 11, 1004}, + dictWord{7, 11, 1913}, + dictWord{8, 11, 317}, + dictWord{ + 9, + 11, + 302, + }, + dictWord{10, 11, 612}, + dictWord{141, 11, 22}, + dictWord{138, 0, 472}, + dictWord{4, 11, 261}, + dictWord{135, 11, 510}, + dictWord{134, 10, 90}, + dictWord{142, 0, 433}, + dictWord{151, 0, 28}, + dictWord{4, 11, 291}, + dictWord{7, 11, 101}, + dictWord{9, 11, 515}, + dictWord{12, 11, 152}, + dictWord{12, 11, 443}, + dictWord{13, 11, 392}, + dictWord{142, 11, 357}, + dictWord{140, 0, 997}, + dictWord{5, 0, 3}, + dictWord{8, 0, 578}, + dictWord{9, 0, 118}, + dictWord{10, 0, 705}, + dictWord{ + 141, + 0, + 279, + }, + dictWord{135, 11, 1266}, + dictWord{7, 10, 813}, + dictWord{12, 10, 497}, + dictWord{141, 10, 56}, + dictWord{133, 0, 229}, + dictWord{6, 10, 125}, + dictWord{135, 10, 1277}, + dictWord{8, 0, 102}, + dictWord{10, 0, 578}, + dictWord{10, 0, 672}, + dictWord{12, 0, 496}, + dictWord{13, 0, 408}, + dictWord{14, 0, 121}, + dictWord{17, 0, 106}, + dictWord{151, 10, 12}, + dictWord{6, 0, 866}, + dictWord{134, 0, 1080}, + dictWord{136, 0, 1022}, + dictWord{4, 11, 130}, + dictWord{135, 11, 843}, + dictWord{5, 11, 42}, + dictWord{5, 11, 879}, + dictWord{7, 11, 245}, + dictWord{7, 11, 324}, + dictWord{7, 11, 1532}, + dictWord{11, 11, 463}, + dictWord{11, 11, 472}, + dictWord{13, 11, 363}, + dictWord{144, 11, 52}, + dictWord{150, 0, 55}, + dictWord{8, 0, 115}, + dictWord{8, 0, 350}, + dictWord{9, 0, 489}, + dictWord{10, 0, 128}, + dictWord{ + 11, + 0, + 306, + }, + dictWord{12, 0, 373}, + dictWord{14, 0, 30}, + dictWord{17, 0, 79}, + dictWord{19, 0, 80}, + dictWord{4, 11, 134}, + dictWord{133, 11, 372}, + dictWord{ + 134, + 0, + 657, + }, + dictWord{134, 0, 933}, + dictWord{135, 11, 1147}, + dictWord{4, 0, 230}, + dictWord{133, 0, 702}, + dictWord{134, 0, 1728}, + dictWord{4, 0, 484}, + dictWord{ + 18, + 0, + 26, + }, + dictWord{19, 0, 42}, + dictWord{20, 0, 43}, + dictWord{21, 0, 0}, + dictWord{23, 0, 27}, + dictWord{152, 0, 14}, + dictWord{7, 0, 185}, + dictWord{135, 0, 703}, + dictWord{ + 6, + 0, + 417, + }, + dictWord{10, 0, 618}, + dictWord{7, 10, 1106}, + dictWord{9, 10, 770}, + dictWord{11, 10, 112}, + dictWord{140, 10, 413}, + dictWord{134, 0, 803}, + dictWord{132, 11, 644}, + dictWord{134, 0, 1262}, + dictWord{7, 11, 540}, + dictWord{12, 10, 271}, + dictWord{145, 10, 109}, + dictWord{135, 11, 123}, + dictWord{ + 132, + 0, + 633, + }, + dictWord{134, 11, 623}, + dictWord{4, 11, 908}, + dictWord{5, 11, 359}, + dictWord{5, 11, 508}, + dictWord{6, 11, 1723}, + dictWord{7, 11, 343}, + dictWord{ + 7, + 11, + 1996, + }, + dictWord{135, 11, 2026}, + dictWord{135, 0, 479}, + dictWord{10, 0, 262}, + dictWord{7, 10, 304}, + dictWord{9, 10, 646}, + dictWord{9, 10, 862}, + dictWord{ + 11, + 10, + 696, + }, + dictWord{12, 10, 208}, + dictWord{15, 10, 79}, + dictWord{147, 10, 108}, + dictWord{4, 11, 341}, + dictWord{135, 11, 480}, + dictWord{134, 0, 830}, + dictWord{5, 0, 70}, + dictWord{5, 0, 622}, + dictWord{6, 0, 334}, + dictWord{7, 0, 1032}, + dictWord{9, 0, 171}, + dictWord{11, 0, 26}, + dictWord{11, 0, 213}, + dictWord{ + 11, + 0, + 637, + }, + dictWord{11, 0, 707}, + dictWord{12, 0, 202}, + dictWord{12, 0, 380}, + dictWord{13, 0, 226}, + dictWord{13, 0, 355}, + dictWord{14, 0, 222}, + dictWord{145, 0, 42}, + dictWord{135, 10, 981}, + dictWord{143, 0, 217}, + dictWord{137, 11, 114}, + dictWord{4, 0, 23}, + dictWord{4, 0, 141}, + dictWord{5, 0, 313}, + dictWord{5, 0, 1014}, + dictWord{6, 0, 50}, + dictWord{6, 0, 51}, + dictWord{7, 0, 142}, + dictWord{7, 0, 384}, + dictWord{7, 0, 559}, + dictWord{8, 0, 640}, + dictWord{9, 0, 460}, + dictWord{9, 0, 783}, + dictWord{11, 0, 741}, + dictWord{12, 0, 183}, + dictWord{141, 0, 488}, + dictWord{141, 0, 360}, + dictWord{7, 0, 1586}, + dictWord{7, 11, 1995}, + dictWord{8, 11, 299}, + dictWord{11, 11, 890}, + dictWord{140, 11, 674}, + dictWord{132, 10, 434}, + dictWord{7, 0, 652}, + dictWord{134, 10, 550}, + dictWord{7, 0, 766}, + dictWord{5, 10, 553}, + dictWord{138, 10, 824}, + dictWord{7, 0, 737}, + dictWord{8, 0, 298}, + dictWord{136, 10, 452}, + dictWord{4, 11, 238}, + dictWord{5, 11, 503}, + dictWord{6, 11, 179}, + dictWord{7, 11, 2003}, + dictWord{8, 11, 381}, + dictWord{8, 11, 473}, + dictWord{9, 11, 149}, + dictWord{10, 11, 183}, + dictWord{15, 11, 45}, + dictWord{143, 11, 86}, + dictWord{133, 10, 292}, + dictWord{5, 0, 222}, + dictWord{9, 0, 655}, + dictWord{138, 0, 534}, + dictWord{138, 10, 135}, + dictWord{4, 11, 121}, + dictWord{5, 11, 156}, + dictWord{5, 11, 349}, + dictWord{9, 11, 136}, + dictWord{10, 11, 605}, + dictWord{14, 11, 342}, + dictWord{147, 11, 107}, + dictWord{137, 0, 906}, + dictWord{6, 0, 1013}, + dictWord{134, 0, 1250}, + dictWord{6, 0, 1956}, + dictWord{6, 0, 2009}, + dictWord{8, 0, 991}, + dictWord{144, 0, 120}, + dictWord{135, 11, 1192}, + dictWord{ + 138, + 0, + 503, + }, + dictWord{5, 0, 154}, + dictWord{7, 0, 1491}, + dictWord{10, 0, 379}, + dictWord{138, 0, 485}, + dictWord{6, 0, 1867}, + dictWord{6, 0, 1914}, + dictWord{6, 0, 1925}, + dictWord{9, 0, 917}, + dictWord{9, 0, 925}, + dictWord{9, 0, 932}, + dictWord{9, 0, 951}, + dictWord{9, 0, 1007}, + dictWord{9, 0, 1013}, + dictWord{12, 0, 806}, + dictWord{ + 12, + 0, + 810, + }, + dictWord{12, 0, 814}, + dictWord{12, 0, 816}, + dictWord{12, 0, 824}, + dictWord{12, 0, 832}, + dictWord{12, 0, 837}, + dictWord{12, 0, 863}, + dictWord{ + 12, + 0, + 868, + }, + dictWord{12, 0, 870}, + dictWord{12, 0, 889}, + dictWord{12, 0, 892}, + dictWord{12, 0, 900}, + dictWord{12, 0, 902}, + dictWord{12, 0, 908}, + dictWord{12, 0, 933}, + dictWord{12, 0, 942}, + dictWord{12, 0, 949}, + dictWord{12, 0, 954}, + dictWord{15, 0, 175}, + dictWord{15, 0, 203}, + dictWord{15, 0, 213}, + dictWord{15, 0, 218}, + dictWord{15, 0, 225}, + dictWord{15, 0, 231}, + dictWord{15, 0, 239}, + dictWord{15, 0, 248}, + dictWord{15, 0, 252}, + dictWord{18, 0, 190}, + dictWord{18, 0, 204}, + dictWord{ + 18, + 0, + 215, + }, + dictWord{18, 0, 216}, + dictWord{18, 0, 222}, + dictWord{18, 0, 225}, + dictWord{18, 0, 230}, + dictWord{18, 0, 239}, + dictWord{18, 0, 241}, + dictWord{ + 21, + 0, + 42, + }, + dictWord{21, 0, 43}, + dictWord{21, 0, 44}, + dictWord{21, 0, 45}, + dictWord{21, 0, 46}, + dictWord{21, 0, 53}, + dictWord{24, 0, 27}, + dictWord{152, 0, 31}, + dictWord{ + 133, + 0, + 716, + }, + dictWord{135, 0, 844}, + dictWord{4, 0, 91}, + dictWord{5, 0, 388}, + dictWord{5, 0, 845}, + dictWord{6, 0, 206}, + dictWord{6, 0, 252}, + dictWord{6, 0, 365}, + dictWord{ + 7, + 0, + 136, + }, + dictWord{7, 0, 531}, + dictWord{136, 0, 621}, + dictWord{7, 10, 393}, + dictWord{10, 10, 603}, + dictWord{139, 10, 206}, + dictWord{6, 11, 80}, + dictWord{ + 6, + 11, + 1694, + }, + dictWord{7, 11, 173}, + dictWord{7, 11, 1974}, + dictWord{9, 11, 547}, + dictWord{10, 11, 730}, + dictWord{14, 11, 18}, + dictWord{150, 11, 39}, + dictWord{137, 0, 748}, + dictWord{4, 11, 923}, + dictWord{134, 11, 1711}, + dictWord{4, 10, 912}, + dictWord{137, 10, 232}, + dictWord{7, 10, 98}, + dictWord{7, 10, 1973}, + dictWord{136, 10, 716}, + dictWord{14, 0, 103}, + dictWord{133, 10, 733}, + dictWord{132, 11, 595}, + dictWord{12, 0, 158}, + dictWord{18, 0, 8}, + dictWord{19, 0, 62}, + dictWord{20, 0, 6}, + dictWord{22, 0, 4}, + dictWord{23, 0, 2}, + dictWord{23, 0, 9}, + dictWord{5, 11, 240}, + dictWord{6, 11, 459}, + dictWord{7, 11, 12}, + dictWord{7, 11, 114}, + dictWord{7, 11, 502}, + dictWord{7, 11, 1751}, + dictWord{7, 11, 1753}, + dictWord{7, 11, 1805}, + dictWord{8, 11, 658}, + dictWord{9, 11, 1}, + dictWord{11, 11, 959}, + dictWord{13, 11, 446}, + dictWord{142, 11, 211}, + dictWord{135, 0, 576}, + dictWord{5, 0, 771}, + dictWord{5, 0, 863}, + dictWord{5, 0, 898}, + dictWord{6, 0, 648}, + dictWord{ + 6, + 0, + 1632, + }, + dictWord{6, 0, 1644}, + dictWord{134, 0, 1780}, + dictWord{133, 0, 331}, + dictWord{7, 11, 633}, + dictWord{7, 11, 905}, + dictWord{7, 11, 909}, + dictWord{ + 7, + 11, + 1538, + }, + dictWord{9, 11, 767}, + dictWord{140, 11, 636}, + dictWord{140, 0, 632}, + dictWord{5, 0, 107}, + dictWord{7, 0, 201}, + dictWord{136, 0, 518}, + dictWord{ + 6, + 0, + 446, + }, + dictWord{7, 0, 1817}, + dictWord{134, 11, 490}, + dictWord{9, 0, 851}, + dictWord{141, 0, 510}, + dictWord{7, 11, 250}, + dictWord{8, 11, 506}, + dictWord{ + 136, + 11, + 507, + }, + dictWord{4, 0, 504}, + dictWord{137, 10, 72}, + dictWord{132, 11, 158}, + dictWord{4, 11, 140}, + dictWord{7, 11, 362}, + dictWord{8, 11, 209}, + dictWord{ + 9, + 11, + 10, + }, + dictWord{9, 11, 160}, + dictWord{9, 11, 503}, + dictWord{10, 11, 689}, + dictWord{11, 11, 350}, + dictWord{11, 11, 553}, + dictWord{11, 11, 725}, + dictWord{ + 12, + 11, + 252, + }, + dictWord{12, 11, 583}, + dictWord{13, 11, 192}, + dictWord{13, 11, 352}, + dictWord{14, 11, 269}, + dictWord{14, 11, 356}, + dictWord{148, 11, 50}, + dictWord{6, 11, 597}, + dictWord{135, 11, 1318}, + dictWord{135, 10, 1454}, + dictWord{5, 0, 883}, + dictWord{5, 0, 975}, + dictWord{8, 0, 392}, + dictWord{148, 0, 7}, + dictWord{6, 11, 228}, + dictWord{7, 11, 1341}, + dictWord{9, 11, 408}, + dictWord{138, 11, 343}, + dictWord{11, 11, 348}, + dictWord{11, 10, 600}, + dictWord{12, 11, 99}, + dictWord{13, 10, 245}, + dictWord{18, 11, 1}, + dictWord{18, 11, 11}, + dictWord{147, 11, 4}, + dictWord{134, 11, 296}, + dictWord{5, 0, 922}, + dictWord{134, 0, 1707}, + dictWord{132, 11, 557}, + dictWord{4, 11, 548}, + dictWord{7, 10, 164}, + dictWord{7, 10, 1571}, + dictWord{9, 10, 107}, + dictWord{140, 10, 225}, + dictWord{ + 7, + 11, + 197, + }, + dictWord{8, 11, 142}, + dictWord{8, 11, 325}, + dictWord{9, 11, 150}, + dictWord{9, 11, 596}, + dictWord{10, 11, 350}, + dictWord{10, 11, 353}, + dictWord{ + 11, + 11, + 74, + }, + dictWord{11, 11, 315}, + dictWord{14, 11, 423}, + dictWord{143, 11, 141}, + dictWord{5, 0, 993}, + dictWord{7, 0, 515}, + dictWord{137, 0, 91}, + dictWord{4, 0, 131}, + dictWord{8, 0, 200}, + dictWord{5, 10, 484}, + dictWord{5, 10, 510}, + dictWord{6, 10, 434}, + dictWord{7, 10, 1000}, + dictWord{7, 10, 1098}, + dictWord{136, 10, 2}, + dictWord{152, 0, 10}, + dictWord{4, 11, 62}, + dictWord{5, 11, 83}, + dictWord{6, 11, 399}, + dictWord{6, 11, 579}, + dictWord{7, 11, 692}, + dictWord{7, 11, 846}, + dictWord{ + 7, + 11, + 1015, + }, + dictWord{7, 11, 1799}, + dictWord{8, 11, 403}, + dictWord{9, 11, 394}, + dictWord{10, 11, 133}, + dictWord{12, 11, 4}, + dictWord{12, 11, 297}, + dictWord{ + 12, + 11, + 452, + }, + dictWord{16, 11, 81}, + dictWord{18, 11, 19}, + dictWord{18, 11, 25}, + dictWord{21, 11, 14}, + dictWord{22, 11, 12}, + dictWord{151, 11, 18}, + dictWord{ + 140, + 11, + 459, + }, + dictWord{132, 11, 177}, + dictWord{7, 0, 1433}, + dictWord{9, 0, 365}, + dictWord{137, 11, 365}, + dictWord{132, 10, 460}, + dictWord{5, 0, 103}, + dictWord{ + 6, + 0, + 2004, + }, + dictWord{7, 0, 921}, + dictWord{8, 0, 580}, + dictWord{8, 0, 593}, + dictWord{8, 0, 630}, + dictWord{10, 0, 28}, + dictWord{5, 11, 411}, + dictWord{ + 135, + 11, + 653, + }, + dictWord{4, 10, 932}, + dictWord{133, 10, 891}, + dictWord{4, 0, 911}, + dictWord{5, 0, 867}, + dictWord{5, 0, 1013}, + dictWord{7, 0, 2034}, + dictWord{8, 0, 798}, + dictWord{136, 0, 813}, + dictWord{7, 11, 439}, + dictWord{10, 11, 727}, + dictWord{11, 11, 260}, + dictWord{139, 11, 684}, + dictWord{136, 10, 625}, + dictWord{ + 5, + 11, + 208, + }, + dictWord{7, 11, 753}, + dictWord{135, 11, 1528}, + dictWord{5, 0, 461}, + dictWord{7, 0, 1925}, + dictWord{12, 0, 39}, + dictWord{13, 0, 265}, + dictWord{ + 13, + 0, + 439, + }, + dictWord{134, 10, 76}, + dictWord{6, 0, 853}, + dictWord{8, 10, 92}, + dictWord{137, 10, 221}, + dictWord{5, 0, 135}, + dictWord{6, 0, 519}, + dictWord{7, 0, 1722}, + dictWord{10, 0, 271}, + dictWord{11, 0, 261}, + dictWord{145, 0, 54}, + dictWord{139, 11, 814}, + dictWord{14, 0, 338}, + dictWord{148, 0, 81}, + dictWord{4, 0, 300}, + dictWord{133, 0, 436}, + dictWord{5, 0, 419}, + dictWord{5, 0, 687}, + dictWord{7, 0, 864}, + dictWord{9, 0, 470}, + dictWord{135, 11, 864}, + dictWord{9, 0, 836}, + dictWord{ + 133, + 11, + 242, + }, + dictWord{134, 0, 1937}, + dictWord{4, 10, 763}, + dictWord{133, 11, 953}, + dictWord{132, 10, 622}, + dictWord{132, 0, 393}, + dictWord{ + 133, + 10, + 253, + }, + dictWord{8, 0, 357}, + dictWord{10, 0, 745}, + dictWord{14, 0, 426}, + dictWord{17, 0, 94}, + dictWord{19, 0, 57}, + dictWord{135, 10, 546}, + dictWord{5, 11, 615}, + dictWord{146, 11, 37}, + dictWord{9, 10, 73}, + dictWord{10, 10, 110}, + dictWord{14, 10, 185}, + dictWord{145, 10, 119}, + dictWord{11, 0, 703}, + dictWord{7, 10, 624}, + dictWord{7, 10, 916}, + dictWord{10, 10, 256}, + dictWord{139, 10, 87}, + dictWord{133, 11, 290}, + dictWord{5, 10, 212}, + dictWord{12, 10, 35}, + dictWord{ + 141, + 10, + 382, + }, + dictWord{132, 11, 380}, + dictWord{5, 11, 52}, + dictWord{7, 11, 277}, + dictWord{9, 11, 368}, + dictWord{139, 11, 791}, + dictWord{133, 0, 387}, + dictWord{ + 10, + 11, + 138, + }, + dictWord{139, 11, 476}, + dictWord{4, 0, 6}, + dictWord{5, 0, 708}, + dictWord{136, 0, 75}, + dictWord{7, 0, 1351}, + dictWord{9, 0, 581}, + dictWord{10, 0, 639}, + dictWord{11, 0, 453}, + dictWord{140, 0, 584}, + dictWord{132, 0, 303}, + dictWord{138, 0, 772}, + dictWord{135, 10, 1175}, + dictWord{4, 0, 749}, + dictWord{ + 5, + 10, + 816, + }, + dictWord{6, 11, 256}, + dictWord{7, 11, 307}, + dictWord{7, 11, 999}, + dictWord{7, 11, 1481}, + dictWord{7, 11, 1732}, + dictWord{7, 11, 1738}, + dictWord{ + 8, + 11, + 265, + }, + dictWord{9, 11, 414}, + dictWord{11, 11, 316}, + dictWord{12, 11, 52}, + dictWord{13, 11, 420}, + dictWord{147, 11, 100}, + dictWord{135, 11, 1296}, + dictWord{ + 6, + 0, + 1065, + }, + dictWord{5, 10, 869}, + dictWord{5, 10, 968}, + dictWord{6, 10, 1626}, + dictWord{8, 10, 734}, + dictWord{136, 10, 784}, + dictWord{4, 10, 542}, + dictWord{ + 6, + 10, + 1716, + }, + dictWord{6, 10, 1727}, + dictWord{7, 10, 1082}, + dictWord{7, 10, 1545}, + dictWord{8, 10, 56}, + dictWord{8, 10, 118}, + dictWord{8, 10, 412}, + dictWord{ + 8, + 10, + 564, + }, + dictWord{9, 10, 888}, + dictWord{9, 10, 908}, + dictWord{10, 10, 50}, + dictWord{10, 10, 423}, + dictWord{11, 10, 685}, + dictWord{11, 10, 697}, + dictWord{11, 10, 933}, + dictWord{12, 10, 299}, + dictWord{13, 10, 126}, + dictWord{13, 10, 136}, + dictWord{13, 10, 170}, + dictWord{141, 10, 190}, + dictWord{ + 134, + 0, + 226, + }, + dictWord{4, 0, 106}, + dictWord{7, 0, 310}, + dictWord{11, 0, 717}, + dictWord{133, 11, 723}, + dictWord{5, 0, 890}, + dictWord{5, 0, 988}, + dictWord{4, 10, 232}, + dictWord{9, 10, 202}, + dictWord{10, 10, 474}, + dictWord{140, 10, 433}, + dictWord{6, 0, 626}, + dictWord{142, 0, 431}, + dictWord{10, 0, 706}, + dictWord{150, 0, 44}, + dictWord{13, 0, 51}, + dictWord{6, 10, 108}, + dictWord{7, 10, 1003}, + dictWord{7, 10, 1181}, + dictWord{8, 10, 111}, + dictWord{136, 10, 343}, + dictWord{132, 0, 698}, + dictWord{5, 11, 109}, + dictWord{6, 11, 1784}, + dictWord{7, 11, 1895}, + dictWord{12, 11, 296}, + dictWord{140, 11, 302}, + dictWord{134, 0, 828}, + dictWord{ + 134, + 10, + 1712, + }, + dictWord{138, 0, 17}, + dictWord{7, 0, 1929}, + dictWord{4, 10, 133}, + dictWord{5, 11, 216}, + dictWord{7, 10, 711}, + dictWord{7, 10, 1298}, + dictWord{ + 7, + 10, + 1585, + }, + dictWord{7, 11, 1879}, + dictWord{9, 11, 141}, + dictWord{9, 11, 270}, + dictWord{9, 11, 679}, + dictWord{10, 11, 159}, + dictWord{10, 11, 553}, + dictWord{ + 11, + 11, + 197, + }, + dictWord{11, 11, 438}, + dictWord{12, 11, 538}, + dictWord{12, 11, 559}, + dictWord{13, 11, 193}, + dictWord{13, 11, 423}, + dictWord{14, 11, 144}, + dictWord{14, 11, 166}, + dictWord{14, 11, 167}, + dictWord{15, 11, 67}, + dictWord{147, 11, 84}, + dictWord{141, 11, 127}, + dictWord{7, 11, 1872}, + dictWord{ + 137, + 11, + 81, + }, + dictWord{6, 10, 99}, + dictWord{7, 10, 1808}, + dictWord{145, 10, 57}, + dictWord{134, 11, 391}, + dictWord{5, 0, 689}, + dictWord{6, 0, 84}, + dictWord{7, 0, 1250}, + dictWord{6, 10, 574}, + dictWord{7, 10, 428}, + dictWord{10, 10, 669}, + dictWord{11, 10, 485}, + dictWord{11, 10, 840}, + dictWord{12, 10, 300}, + dictWord{ + 142, + 10, + 250, + }, + dictWord{7, 11, 322}, + dictWord{136, 11, 249}, + dictWord{7, 11, 432}, + dictWord{135, 11, 1649}, + dictWord{135, 10, 1871}, + dictWord{137, 10, 252}, + dictWord{6, 11, 155}, + dictWord{140, 11, 234}, + dictWord{7, 0, 871}, + dictWord{19, 0, 27}, + dictWord{147, 11, 27}, + dictWord{140, 0, 498}, + dictWord{5, 0, 986}, + dictWord{6, 0, 130}, + dictWord{138, 0, 823}, + dictWord{6, 0, 1793}, + dictWord{7, 0, 1582}, + dictWord{8, 0, 458}, + dictWord{10, 0, 101}, + dictWord{10, 0, 318}, + dictWord{ + 10, + 0, + 945, + }, + dictWord{12, 0, 734}, + dictWord{16, 0, 104}, + dictWord{18, 0, 177}, + dictWord{6, 10, 323}, + dictWord{135, 10, 1564}, + dictWord{5, 11, 632}, + dictWord{ + 138, + 11, + 526, + }, + dictWord{10, 0, 435}, + dictWord{7, 10, 461}, + dictWord{136, 10, 775}, + dictWord{6, 11, 144}, + dictWord{7, 11, 948}, + dictWord{7, 11, 1042}, + dictWord{ + 7, + 11, + 1857, + }, + dictWord{8, 11, 235}, + dictWord{8, 11, 461}, + dictWord{9, 11, 453}, + dictWord{9, 11, 530}, + dictWord{10, 11, 354}, + dictWord{17, 11, 77}, + dictWord{ + 19, + 11, + 99, + }, + dictWord{148, 11, 79}, + dictWord{138, 0, 966}, + dictWord{7, 0, 1644}, + dictWord{137, 0, 129}, + dictWord{135, 0, 997}, + dictWord{136, 0, 502}, + dictWord{ + 5, + 11, + 196, + }, + dictWord{6, 11, 486}, + dictWord{7, 11, 212}, + dictWord{8, 11, 309}, + dictWord{136, 11, 346}, + dictWord{7, 10, 727}, + dictWord{146, 10, 73}, + dictWord{132, 0, 823}, + dictWord{132, 11, 686}, + dictWord{135, 0, 1927}, + dictWord{4, 0, 762}, + dictWord{7, 0, 1756}, + dictWord{137, 0, 98}, + dictWord{136, 10, 577}, + dictWord{24, 0, 8}, + dictWord{4, 11, 30}, + dictWord{5, 11, 43}, + dictWord{152, 11, 8}, + dictWord{7, 0, 1046}, + dictWord{139, 0, 160}, + dictWord{7, 0, 492}, + dictWord{ + 4, + 10, + 413, + }, + dictWord{5, 10, 677}, + dictWord{7, 11, 492}, + dictWord{8, 10, 432}, + dictWord{140, 10, 280}, + dictWord{6, 0, 45}, + dictWord{7, 0, 433}, + dictWord{8, 0, 129}, + dictWord{9, 0, 21}, + dictWord{10, 0, 392}, + dictWord{11, 0, 79}, + dictWord{12, 0, 499}, + dictWord{13, 0, 199}, + dictWord{141, 0, 451}, + dictWord{7, 0, 558}, + dictWord{ + 136, + 0, + 353, + }, + dictWord{4, 11, 220}, + dictWord{7, 11, 1535}, + dictWord{9, 11, 93}, + dictWord{139, 11, 474}, + dictWord{7, 10, 646}, + dictWord{7, 10, 1730}, + dictWord{ + 11, + 10, + 446, + }, + dictWord{141, 10, 178}, + dictWord{133, 0, 785}, + dictWord{134, 0, 1145}, + dictWord{8, 0, 81}, + dictWord{9, 0, 189}, + dictWord{9, 0, 201}, + dictWord{ + 11, + 0, + 478, + }, + dictWord{11, 0, 712}, + dictWord{141, 0, 338}, + dictWord{5, 0, 353}, + dictWord{151, 0, 26}, + dictWord{11, 0, 762}, + dictWord{132, 10, 395}, + dictWord{ + 134, + 0, + 2024, + }, + dictWord{4, 0, 611}, + dictWord{133, 0, 606}, + dictWord{9, 10, 174}, + dictWord{10, 10, 164}, + dictWord{11, 10, 440}, + dictWord{11, 10, 841}, + dictWord{ + 143, + 10, + 98, + }, + dictWord{134, 10, 426}, + dictWord{10, 10, 608}, + dictWord{139, 10, 1002}, + dictWord{138, 10, 250}, + dictWord{6, 0, 25}, + dictWord{7, 0, 855}, + dictWord{7, 0, 1258}, + dictWord{144, 0, 32}, + dictWord{7, 11, 1725}, + dictWord{138, 11, 393}, + dictWord{5, 11, 263}, + dictWord{134, 11, 414}, + dictWord{6, 0, 2011}, + dictWord{133, 10, 476}, + dictWord{4, 0, 4}, + dictWord{7, 0, 1118}, + dictWord{7, 0, 1320}, + dictWord{7, 0, 1706}, + dictWord{8, 0, 277}, + dictWord{9, 0, 622}, + dictWord{ + 10, + 0, + 9, + }, + dictWord{11, 0, 724}, + dictWord{12, 0, 350}, + dictWord{12, 0, 397}, + dictWord{13, 0, 28}, + dictWord{13, 0, 159}, + dictWord{15, 0, 89}, + dictWord{18, 0, 5}, + dictWord{ + 19, + 0, + 9, + }, + dictWord{20, 0, 34}, + dictWord{22, 0, 47}, + dictWord{6, 11, 178}, + dictWord{6, 11, 1750}, + dictWord{8, 11, 251}, + dictWord{9, 11, 690}, + dictWord{ + 10, + 11, + 155, + }, + dictWord{10, 11, 196}, + dictWord{10, 11, 373}, + dictWord{11, 11, 698}, + dictWord{13, 11, 155}, + dictWord{148, 11, 93}, + dictWord{5, 11, 97}, + dictWord{ + 137, + 11, + 393, + }, + dictWord{7, 0, 764}, + dictWord{11, 0, 461}, + dictWord{12, 0, 172}, + dictWord{5, 10, 76}, + dictWord{6, 10, 458}, + dictWord{6, 10, 497}, + dictWord{ + 7, + 10, + 868, + }, + dictWord{9, 10, 658}, + dictWord{10, 10, 594}, + dictWord{11, 10, 566}, + dictWord{12, 10, 338}, + dictWord{141, 10, 200}, + dictWord{134, 0, 1449}, + dictWord{138, 11, 40}, + dictWord{134, 11, 1639}, + dictWord{134, 0, 1445}, + dictWord{6, 0, 1168}, + dictWord{4, 10, 526}, + dictWord{7, 10, 1029}, + dictWord{ + 135, + 10, + 1054, + }, + dictWord{4, 11, 191}, + dictWord{7, 11, 934}, + dictWord{8, 11, 647}, + dictWord{145, 11, 97}, + dictWord{132, 10, 636}, + dictWord{6, 0, 233}, + dictWord{ + 7, + 10, + 660, + }, + dictWord{7, 10, 1124}, + dictWord{17, 10, 31}, + dictWord{19, 10, 22}, + dictWord{151, 10, 14}, + dictWord{6, 10, 1699}, + dictWord{136, 11, 110}, + dictWord{ + 12, + 11, + 246, + }, + dictWord{15, 11, 162}, + dictWord{19, 11, 64}, + dictWord{20, 11, 8}, + dictWord{20, 11, 95}, + dictWord{22, 11, 24}, + dictWord{152, 11, 17}, + dictWord{ + 5, + 11, + 165, + }, + dictWord{9, 11, 346}, + dictWord{138, 11, 655}, + dictWord{5, 11, 319}, + dictWord{135, 11, 534}, + dictWord{134, 0, 255}, + dictWord{9, 0, 216}, + dictWord{ + 8, + 11, + 128, + }, + dictWord{139, 11, 179}, + dictWord{9, 0, 183}, + dictWord{139, 0, 286}, + dictWord{11, 0, 956}, + dictWord{151, 0, 3}, + dictWord{4, 0, 536}, + dictWord{ + 7, + 0, + 1141, + }, + dictWord{10, 0, 723}, + dictWord{139, 0, 371}, + dictWord{4, 10, 279}, + dictWord{7, 10, 301}, + dictWord{137, 10, 362}, + dictWord{7, 0, 285}, + dictWord{ + 5, + 11, + 57, + }, + dictWord{6, 11, 101}, + dictWord{6, 11, 1663}, + dictWord{7, 11, 132}, + dictWord{7, 11, 1048}, + dictWord{7, 11, 1154}, + dictWord{7, 11, 1415}, + dictWord{ + 7, + 11, + 1507, + }, + dictWord{12, 11, 493}, + dictWord{15, 11, 105}, + dictWord{151, 11, 15}, + dictWord{5, 11, 459}, + dictWord{7, 11, 1073}, + dictWord{7, 10, 1743}, + dictWord{ + 8, + 11, + 241, + }, + dictWord{136, 11, 334}, + dictWord{4, 10, 178}, + dictWord{133, 10, 399}, + dictWord{135, 0, 560}, + dictWord{132, 0, 690}, + dictWord{135, 0, 1246}, + dictWord{18, 0, 157}, + dictWord{147, 0, 63}, + dictWord{10, 0, 599}, + dictWord{11, 0, 33}, + dictWord{12, 0, 571}, + dictWord{149, 0, 1}, + dictWord{6, 11, 324}, + dictWord{ + 6, + 11, + 520, + }, + dictWord{7, 11, 338}, + dictWord{7, 11, 1616}, + dictWord{7, 11, 1729}, + dictWord{8, 11, 228}, + dictWord{9, 11, 69}, + dictWord{139, 11, 750}, + dictWord{ + 7, + 0, + 1862, + }, + dictWord{12, 0, 491}, + dictWord{12, 0, 520}, + dictWord{13, 0, 383}, + dictWord{142, 0, 244}, + dictWord{135, 11, 734}, + dictWord{134, 10, 1692}, + dictWord{10, 0, 448}, + dictWord{11, 0, 630}, + dictWord{17, 0, 117}, + dictWord{6, 10, 202}, + dictWord{7, 11, 705}, + dictWord{12, 10, 360}, + dictWord{17, 10, 118}, + dictWord{18, 10, 27}, + dictWord{148, 10, 67}, + dictWord{4, 11, 73}, + dictWord{6, 11, 612}, + dictWord{7, 11, 927}, + dictWord{7, 11, 1822}, + dictWord{8, 11, 217}, + dictWord{ + 9, + 11, + 472, + }, + dictWord{9, 11, 765}, + dictWord{9, 11, 766}, + dictWord{10, 11, 408}, + dictWord{11, 11, 51}, + dictWord{11, 11, 793}, + dictWord{12, 11, 266}, + dictWord{ + 15, + 11, + 158, + }, + dictWord{20, 11, 89}, + dictWord{150, 11, 32}, + dictWord{4, 0, 190}, + dictWord{133, 0, 554}, + dictWord{133, 0, 1001}, + dictWord{5, 11, 389}, + dictWord{ + 8, + 11, + 636, + }, + dictWord{137, 11, 229}, + dictWord{5, 0, 446}, + dictWord{7, 10, 872}, + dictWord{10, 10, 516}, + dictWord{139, 10, 167}, + dictWord{137, 10, 313}, + dictWord{132, 10, 224}, + dictWord{134, 0, 1313}, + dictWord{5, 10, 546}, + dictWord{7, 10, 35}, + dictWord{8, 10, 11}, + dictWord{8, 10, 12}, + dictWord{9, 10, 315}, + dictWord{9, 10, 533}, + dictWord{10, 10, 802}, + dictWord{11, 10, 166}, + dictWord{12, 10, 525}, + dictWord{142, 10, 243}, + dictWord{6, 0, 636}, + dictWord{137, 0, 837}, + dictWord{5, 10, 241}, + dictWord{8, 10, 242}, + dictWord{9, 10, 451}, + dictWord{10, 10, 667}, + dictWord{11, 10, 598}, + dictWord{140, 10, 429}, + dictWord{22, 10, 46}, + dictWord{150, 11, 46}, + dictWord{136, 11, 472}, + dictWord{11, 0, 278}, + dictWord{142, 0, 73}, + dictWord{141, 11, 185}, + dictWord{132, 0, 868}, + dictWord{ + 134, + 0, + 972, + }, + dictWord{4, 10, 366}, + dictWord{137, 10, 516}, + dictWord{138, 0, 1010}, + dictWord{5, 11, 189}, + dictWord{6, 10, 1736}, + dictWord{7, 11, 442}, + dictWord{ + 7, + 11, + 443, + }, + dictWord{8, 11, 281}, + dictWord{12, 11, 174}, + dictWord{13, 11, 83}, + dictWord{141, 11, 261}, + dictWord{139, 11, 384}, + dictWord{6, 11, 2}, + dictWord{ + 7, + 11, + 191, + }, + dictWord{7, 11, 446}, + dictWord{7, 11, 758}, + dictWord{7, 11, 1262}, + dictWord{7, 11, 1737}, + dictWord{8, 11, 22}, + dictWord{8, 11, 270}, + dictWord{ + 8, + 11, + 612, + }, + dictWord{9, 11, 4}, + dictWord{9, 11, 167}, + dictWord{9, 11, 312}, + dictWord{9, 11, 436}, + dictWord{10, 11, 156}, + dictWord{10, 11, 216}, + dictWord{ + 10, + 11, + 311, + }, + dictWord{10, 11, 623}, + dictWord{11, 11, 72}, + dictWord{11, 11, 330}, + dictWord{11, 11, 455}, + dictWord{12, 11, 101}, + dictWord{12, 11, 321}, + dictWord{ + 12, + 11, + 504, + }, + dictWord{12, 11, 530}, + dictWord{12, 11, 543}, + dictWord{13, 11, 17}, + dictWord{13, 11, 156}, + dictWord{13, 11, 334}, + dictWord{14, 11, 48}, + dictWord{15, 11, 70}, + dictWord{17, 11, 60}, + dictWord{148, 11, 64}, + dictWord{6, 10, 331}, + dictWord{136, 10, 623}, + dictWord{135, 0, 1231}, + dictWord{132, 0, 304}, + dictWord{6, 11, 60}, + dictWord{7, 11, 670}, + dictWord{7, 11, 1327}, + dictWord{8, 11, 411}, + dictWord{8, 11, 435}, + dictWord{9, 11, 653}, + dictWord{9, 11, 740}, + dictWord{10, 11, 385}, + dictWord{11, 11, 222}, + dictWord{11, 11, 324}, + dictWord{11, 11, 829}, + dictWord{140, 11, 611}, + dictWord{7, 0, 506}, + dictWord{6, 11, 166}, + dictWord{7, 11, 374}, + dictWord{135, 11, 1174}, + dictWord{14, 11, 43}, + dictWord{146, 11, 21}, + dictWord{135, 11, 1694}, + dictWord{135, 10, 1888}, + dictWord{ + 5, + 11, + 206, + }, + dictWord{134, 11, 398}, + dictWord{135, 11, 50}, + dictWord{150, 0, 26}, + dictWord{6, 0, 53}, + dictWord{6, 0, 199}, + dictWord{7, 0, 1408}, + dictWord{ + 8, + 0, + 32, + }, + dictWord{8, 0, 93}, + dictWord{10, 0, 397}, + dictWord{10, 0, 629}, + dictWord{11, 0, 593}, + dictWord{11, 0, 763}, + dictWord{13, 0, 326}, + dictWord{145, 0, 35}, + dictWord{134, 0, 105}, + dictWord{132, 10, 394}, + dictWord{4, 0, 843}, + dictWord{138, 0, 794}, + dictWord{11, 0, 704}, + dictWord{141, 0, 396}, + dictWord{5, 0, 114}, + dictWord{5, 0, 255}, + dictWord{141, 0, 285}, + dictWord{6, 0, 619}, + dictWord{7, 0, 898}, + dictWord{7, 0, 1092}, + dictWord{8, 0, 485}, + dictWord{18, 0, 28}, + dictWord{ + 19, + 0, + 116, + }, + dictWord{135, 10, 1931}, + dictWord{9, 0, 145}, + dictWord{7, 10, 574}, + dictWord{135, 10, 1719}, + dictWord{7, 0, 2035}, + dictWord{8, 0, 19}, + dictWord{ + 9, + 0, + 89, + }, + dictWord{138, 0, 831}, + dictWord{132, 10, 658}, + dictWord{6, 11, 517}, + dictWord{7, 11, 1159}, + dictWord{10, 11, 621}, + dictWord{139, 11, 192}, + dictWord{ + 7, + 0, + 1933, + }, + dictWord{7, 11, 1933}, + dictWord{9, 10, 781}, + dictWord{10, 10, 144}, + dictWord{11, 10, 385}, + dictWord{13, 10, 161}, + dictWord{13, 10, 228}, + dictWord{13, 10, 268}, + dictWord{148, 10, 107}, + dictWord{136, 10, 374}, + dictWord{10, 11, 223}, + dictWord{139, 11, 645}, + dictWord{135, 0, 1728}, + dictWord{ + 7, + 11, + 64, + }, + dictWord{7, 11, 289}, + dictWord{136, 11, 245}, + dictWord{4, 10, 344}, + dictWord{6, 10, 498}, + dictWord{139, 10, 323}, + dictWord{136, 0, 746}, + dictWord{ + 135, + 10, + 1063, + }, + dictWord{137, 10, 155}, + dictWord{4, 0, 987}, + dictWord{6, 0, 1964}, + dictWord{6, 0, 1974}, + dictWord{6, 0, 1990}, + dictWord{136, 0, 995}, + dictWord{133, 11, 609}, + dictWord{133, 10, 906}, + dictWord{134, 0, 1550}, + dictWord{134, 0, 874}, + dictWord{5, 11, 129}, + dictWord{6, 11, 61}, + dictWord{ + 135, + 11, + 947, + }, + dictWord{4, 0, 1018}, + dictWord{6, 0, 1938}, + dictWord{6, 0, 2021}, + dictWord{134, 0, 2039}, + dictWord{132, 0, 814}, + dictWord{11, 0, 126}, + dictWord{ + 139, + 0, + 287, + }, + dictWord{134, 0, 1264}, + dictWord{5, 0, 955}, + dictWord{136, 0, 814}, + dictWord{141, 11, 506}, + dictWord{132, 11, 314}, + dictWord{6, 0, 981}, + dictWord{139, 11, 1000}, + dictWord{5, 0, 56}, + dictWord{8, 0, 892}, + dictWord{8, 0, 915}, + dictWord{140, 0, 776}, + dictWord{148, 0, 100}, + dictWord{10, 0, 4}, + dictWord{ + 10, + 0, + 13, + }, + dictWord{11, 0, 638}, + dictWord{148, 0, 57}, + dictWord{148, 11, 74}, + dictWord{5, 0, 738}, + dictWord{132, 10, 616}, + dictWord{133, 11, 637}, + dictWord{ + 136, + 10, + 692, + }, + dictWord{133, 0, 758}, + dictWord{132, 10, 305}, + dictWord{137, 11, 590}, + dictWord{5, 11, 280}, + dictWord{135, 11, 1226}, + dictWord{ + 134, + 11, + 494, + }, + dictWord{135, 0, 1112}, + dictWord{133, 11, 281}, + dictWord{13, 0, 44}, + dictWord{14, 0, 214}, + dictWord{5, 10, 214}, + dictWord{7, 10, 603}, + dictWord{ + 8, + 10, + 611, + }, + dictWord{9, 10, 686}, + dictWord{10, 10, 88}, + dictWord{11, 10, 459}, + dictWord{11, 10, 496}, + dictWord{12, 10, 463}, + dictWord{140, 10, 590}, + dictWord{ + 139, + 0, + 328, + }, + dictWord{135, 11, 1064}, + dictWord{137, 0, 133}, + dictWord{7, 0, 168}, + dictWord{13, 0, 196}, + dictWord{141, 0, 237}, + dictWord{134, 10, 1703}, + dictWord{134, 0, 1152}, + dictWord{135, 0, 1245}, + dictWord{5, 0, 110}, + dictWord{6, 0, 169}, + dictWord{6, 0, 1702}, + dictWord{7, 0, 400}, + dictWord{8, 0, 538}, + dictWord{ + 9, + 0, + 184, + }, + dictWord{9, 0, 524}, + dictWord{140, 0, 218}, + dictWord{6, 0, 1816}, + dictWord{10, 0, 871}, + dictWord{12, 0, 769}, + dictWord{140, 0, 785}, + dictWord{ + 132, + 11, + 630, + }, + dictWord{7, 11, 33}, + dictWord{7, 11, 120}, + dictWord{8, 11, 489}, + dictWord{9, 11, 319}, + dictWord{10, 11, 820}, + dictWord{11, 11, 1004}, + dictWord{ + 12, + 11, + 379, + }, + dictWord{13, 11, 117}, + dictWord{13, 11, 412}, + dictWord{14, 11, 25}, + dictWord{15, 11, 52}, + dictWord{15, 11, 161}, + dictWord{16, 11, 47}, + dictWord{149, 11, 2}, + dictWord{6, 0, 133}, + dictWord{8, 0, 413}, + dictWord{9, 0, 353}, + dictWord{139, 0, 993}, + dictWord{145, 10, 19}, + dictWord{4, 11, 937}, + dictWord{ + 133, + 11, + 801, + }, + dictWord{134, 0, 978}, + dictWord{6, 0, 93}, + dictWord{6, 0, 1508}, + dictWord{7, 0, 1422}, + dictWord{7, 0, 1851}, + dictWord{8, 0, 673}, + dictWord{9, 0, 529}, + dictWord{140, 0, 43}, + dictWord{6, 0, 317}, + dictWord{10, 0, 512}, + dictWord{4, 10, 737}, + dictWord{11, 10, 294}, + dictWord{12, 10, 60}, + dictWord{12, 10, 437}, + dictWord{13, 10, 64}, + dictWord{13, 10, 380}, + dictWord{142, 10, 430}, + dictWord{9, 0, 371}, + dictWord{7, 11, 1591}, + dictWord{144, 11, 43}, + dictWord{6, 10, 1758}, + dictWord{8, 10, 520}, + dictWord{9, 10, 345}, + dictWord{9, 10, 403}, + dictWord{142, 10, 350}, + dictWord{5, 0, 526}, + dictWord{10, 10, 242}, + dictWord{ + 138, + 10, + 579, + }, + dictWord{9, 0, 25}, + dictWord{10, 0, 467}, + dictWord{138, 0, 559}, + dictWord{5, 10, 139}, + dictWord{7, 10, 1168}, + dictWord{138, 10, 539}, + dictWord{ + 4, + 0, + 335, + }, + dictWord{135, 0, 942}, + dictWord{140, 0, 754}, + dictWord{132, 11, 365}, + dictWord{11, 0, 182}, + dictWord{142, 0, 195}, + dictWord{142, 11, 29}, + dictWord{ + 5, + 11, + 7, + }, + dictWord{139, 11, 774}, + dictWord{4, 11, 746}, + dictWord{135, 11, 1090}, + dictWord{8, 0, 39}, + dictWord{10, 0, 773}, + dictWord{11, 0, 84}, + dictWord{ + 12, + 0, + 205, + }, + dictWord{142, 0, 1}, + dictWord{5, 0, 601}, + dictWord{5, 0, 870}, + dictWord{5, 11, 360}, + dictWord{136, 11, 237}, + dictWord{132, 0, 181}, + dictWord{ + 136, + 0, + 370, + }, + dictWord{134, 0, 1652}, + dictWord{8, 0, 358}, + dictWord{4, 10, 107}, + dictWord{7, 10, 613}, + dictWord{8, 10, 439}, + dictWord{8, 10, 504}, + dictWord{ + 9, + 10, + 501, + }, + dictWord{10, 10, 383}, + dictWord{139, 10, 477}, + dictWord{132, 10, 229}, + dictWord{137, 11, 785}, + dictWord{4, 0, 97}, + dictWord{5, 0, 147}, + dictWord{ + 6, + 0, + 286, + }, + dictWord{7, 0, 1362}, + dictWord{141, 0, 176}, + dictWord{6, 0, 537}, + dictWord{7, 0, 788}, + dictWord{7, 0, 1816}, + dictWord{132, 10, 903}, + dictWord{ + 140, + 10, + 71, + }, + dictWord{6, 0, 743}, + dictWord{134, 0, 1223}, + dictWord{6, 0, 375}, + dictWord{7, 0, 169}, + dictWord{7, 0, 254}, + dictWord{8, 0, 780}, + dictWord{135, 11, 1493}, + dictWord{7, 0, 1714}, + dictWord{4, 10, 47}, + dictWord{6, 10, 373}, + dictWord{7, 10, 452}, + dictWord{7, 10, 543}, + dictWord{7, 10, 1856}, + dictWord{9, 10, 6}, + dictWord{ + 11, + 10, + 257, + }, + dictWord{139, 10, 391}, + dictWord{6, 0, 896}, + dictWord{136, 0, 1003}, + dictWord{135, 0, 1447}, + dictWord{137, 11, 341}, + dictWord{5, 10, 980}, + dictWord{134, 10, 1754}, + dictWord{145, 11, 22}, + dictWord{4, 11, 277}, + dictWord{5, 11, 608}, + dictWord{6, 11, 493}, + dictWord{7, 11, 457}, + dictWord{ + 140, + 11, + 384, + }, + dictWord{7, 10, 536}, + dictWord{7, 10, 1331}, + dictWord{136, 10, 143}, + dictWord{140, 0, 744}, + dictWord{7, 11, 27}, + dictWord{135, 11, 316}, + dictWord{ + 18, + 0, + 126, + }, + dictWord{5, 10, 19}, + dictWord{134, 10, 533}, + dictWord{4, 0, 788}, + dictWord{11, 0, 41}, + dictWord{5, 11, 552}, + dictWord{5, 11, 586}, + dictWord{ + 5, + 11, + 676, + }, + dictWord{6, 11, 448}, + dictWord{8, 11, 244}, + dictWord{11, 11, 1}, + dictWord{11, 11, 41}, + dictWord{13, 11, 3}, + dictWord{16, 11, 54}, + dictWord{17, 11, 4}, + dictWord{146, 11, 13}, + dictWord{4, 0, 985}, + dictWord{6, 0, 1801}, + dictWord{4, 11, 401}, + dictWord{137, 11, 264}, + dictWord{5, 10, 395}, + dictWord{5, 10, 951}, + dictWord{134, 10, 1776}, + dictWord{5, 0, 629}, + dictWord{135, 0, 1549}, + dictWord{11, 10, 663}, + dictWord{12, 10, 210}, + dictWord{13, 10, 166}, + dictWord{ + 13, + 10, + 310, + }, + dictWord{14, 10, 373}, + dictWord{147, 10, 43}, + dictWord{9, 11, 543}, + dictWord{10, 11, 524}, + dictWord{11, 11, 30}, + dictWord{12, 11, 524}, + dictWord{ + 14, + 11, + 315, + }, + dictWord{16, 11, 18}, + dictWord{20, 11, 26}, + dictWord{148, 11, 65}, + dictWord{4, 11, 205}, + dictWord{5, 11, 623}, + dictWord{7, 11, 104}, + dictWord{ + 136, + 11, + 519, + }, + dictWord{5, 0, 293}, + dictWord{134, 0, 601}, + dictWord{7, 11, 579}, + dictWord{9, 11, 41}, + dictWord{9, 11, 244}, + dictWord{9, 11, 669}, + dictWord{ + 10, + 11, + 5, + }, + dictWord{11, 11, 861}, + dictWord{11, 11, 951}, + dictWord{139, 11, 980}, + dictWord{132, 11, 717}, + dictWord{132, 10, 695}, + dictWord{7, 10, 497}, + dictWord{ + 9, + 10, + 387, + }, + dictWord{147, 10, 81}, + dictWord{132, 0, 420}, + dictWord{142, 0, 37}, + dictWord{6, 0, 1134}, + dictWord{6, 0, 1900}, + dictWord{12, 0, 830}, + dictWord{ + 12, + 0, + 878, + }, + dictWord{12, 0, 894}, + dictWord{15, 0, 221}, + dictWord{143, 0, 245}, + dictWord{132, 11, 489}, + dictWord{7, 0, 1570}, + dictWord{140, 0, 542}, + dictWord{ + 8, + 0, + 933, + }, + dictWord{136, 0, 957}, + dictWord{6, 0, 1371}, + dictWord{7, 0, 31}, + dictWord{8, 0, 373}, + dictWord{5, 10, 284}, + dictWord{6, 10, 49}, + dictWord{6, 10, 350}, + dictWord{7, 10, 377}, + dictWord{7, 10, 1693}, + dictWord{8, 10, 678}, + dictWord{9, 10, 161}, + dictWord{9, 10, 585}, + dictWord{9, 10, 671}, + dictWord{9, 10, 839}, + dictWord{11, 10, 912}, + dictWord{141, 10, 427}, + dictWord{135, 11, 892}, + dictWord{4, 0, 325}, + dictWord{138, 0, 125}, + dictWord{139, 11, 47}, + dictWord{ + 132, + 10, + 597, + }, + dictWord{138, 0, 323}, + dictWord{6, 0, 1547}, + dictWord{7, 11, 1605}, + dictWord{9, 11, 473}, + dictWord{11, 11, 962}, + dictWord{146, 11, 139}, + dictWord{ + 139, + 10, + 908, + }, + dictWord{7, 11, 819}, + dictWord{9, 11, 26}, + dictWord{9, 11, 392}, + dictWord{10, 11, 152}, + dictWord{10, 11, 226}, + dictWord{11, 11, 19}, + dictWord{ + 12, + 11, + 276, + }, + dictWord{12, 11, 426}, + dictWord{12, 11, 589}, + dictWord{13, 11, 460}, + dictWord{15, 11, 97}, + dictWord{19, 11, 48}, + dictWord{148, 11, 104}, + dictWord{135, 11, 51}, + dictWord{4, 0, 718}, + dictWord{135, 0, 1216}, + dictWord{6, 0, 1896}, + dictWord{6, 0, 1905}, + dictWord{6, 0, 1912}, + dictWord{9, 0, 947}, + dictWord{ + 9, + 0, + 974, + }, + dictWord{12, 0, 809}, + dictWord{12, 0, 850}, + dictWord{12, 0, 858}, + dictWord{12, 0, 874}, + dictWord{12, 0, 887}, + dictWord{12, 0, 904}, + dictWord{ + 12, + 0, + 929, + }, + dictWord{12, 0, 948}, + dictWord{12, 0, 952}, + dictWord{15, 0, 198}, + dictWord{15, 0, 206}, + dictWord{15, 0, 220}, + dictWord{15, 0, 227}, + dictWord{15, 0, 247}, + dictWord{18, 0, 188}, + dictWord{21, 0, 48}, + dictWord{21, 0, 50}, + dictWord{24, 0, 25}, + dictWord{24, 0, 29}, + dictWord{7, 11, 761}, + dictWord{7, 11, 1051}, + dictWord{ + 137, + 11, + 545, + }, + dictWord{5, 0, 124}, + dictWord{5, 0, 144}, + dictWord{6, 0, 548}, + dictWord{7, 0, 15}, + dictWord{7, 0, 153}, + dictWord{137, 0, 629}, + dictWord{ + 135, + 11, + 606, + }, + dictWord{135, 10, 2014}, + dictWord{7, 10, 2007}, + dictWord{9, 11, 46}, + dictWord{9, 10, 101}, + dictWord{9, 10, 450}, + dictWord{10, 10, 66}, + dictWord{ + 10, + 10, + 842, + }, + dictWord{11, 10, 536}, + dictWord{140, 10, 587}, + dictWord{6, 0, 75}, + dictWord{7, 0, 1531}, + dictWord{8, 0, 416}, + dictWord{9, 0, 240}, + dictWord{9, 0, 275}, + dictWord{10, 0, 100}, + dictWord{11, 0, 658}, + dictWord{11, 0, 979}, + dictWord{12, 0, 86}, + dictWord{14, 0, 207}, + dictWord{15, 0, 20}, + dictWord{143, 0, 25}, + dictWord{ + 5, + 0, + 141, + }, + dictWord{5, 0, 915}, + dictWord{6, 0, 1783}, + dictWord{7, 0, 211}, + dictWord{7, 0, 698}, + dictWord{7, 0, 1353}, + dictWord{9, 0, 83}, + dictWord{9, 0, 281}, + dictWord{ + 10, + 0, + 376, + }, + dictWord{10, 0, 431}, + dictWord{11, 0, 543}, + dictWord{12, 0, 664}, + dictWord{13, 0, 280}, + dictWord{13, 0, 428}, + dictWord{14, 0, 61}, + dictWord{ + 14, + 0, + 128, + }, + dictWord{17, 0, 52}, + dictWord{145, 0, 81}, + dictWord{132, 11, 674}, + dictWord{135, 0, 533}, + dictWord{149, 0, 6}, + dictWord{132, 11, 770}, + dictWord{ + 133, + 0, + 538, + }, + dictWord{5, 11, 79}, + dictWord{7, 11, 1027}, + dictWord{7, 11, 1477}, + dictWord{139, 11, 52}, + dictWord{139, 10, 62}, + dictWord{4, 0, 338}, + dictWord{ + 133, + 0, + 400, + }, + dictWord{5, 11, 789}, + dictWord{134, 11, 195}, + dictWord{4, 11, 251}, + dictWord{4, 11, 688}, + dictWord{7, 11, 513}, + dictWord{7, 11, 1284}, + dictWord{ + 9, + 11, + 87, + }, + dictWord{138, 11, 365}, + dictWord{134, 10, 1766}, + dictWord{6, 0, 0}, + dictWord{7, 0, 84}, + dictWord{11, 0, 895}, + dictWord{145, 0, 11}, + dictWord{ + 139, + 0, + 892, + }, + dictWord{4, 0, 221}, + dictWord{5, 0, 659}, + dictWord{7, 0, 697}, + dictWord{7, 0, 1211}, + dictWord{138, 0, 284}, + dictWord{133, 0, 989}, + dictWord{ + 133, + 11, + 889, + }, + dictWord{4, 11, 160}, + dictWord{5, 11, 330}, + dictWord{7, 11, 1434}, + dictWord{136, 11, 174}, + dictWord{6, 10, 1665}, + dictWord{7, 10, 256}, + dictWord{ + 7, + 10, + 1388, + }, + dictWord{10, 10, 499}, + dictWord{139, 10, 670}, + dictWord{7, 0, 848}, + dictWord{4, 10, 22}, + dictWord{5, 10, 10}, + dictWord{136, 10, 97}, + dictWord{ + 138, + 0, + 507, + }, + dictWord{133, 10, 481}, + dictWord{4, 0, 188}, + dictWord{135, 0, 805}, + dictWord{5, 0, 884}, + dictWord{6, 0, 732}, + dictWord{139, 0, 991}, + dictWord{ + 135, + 11, + 968, + }, + dictWord{11, 11, 636}, + dictWord{15, 11, 145}, + dictWord{17, 11, 34}, + dictWord{19, 11, 50}, + dictWord{151, 11, 20}, + dictWord{7, 0, 959}, + dictWord{ + 16, + 0, + 60, + }, + dictWord{6, 10, 134}, + dictWord{7, 10, 437}, + dictWord{9, 10, 37}, + dictWord{14, 10, 285}, + dictWord{142, 10, 371}, + dictWord{7, 10, 486}, + dictWord{ + 8, + 10, + 155, + }, + dictWord{11, 10, 93}, + dictWord{140, 10, 164}, + dictWord{134, 0, 1653}, + dictWord{7, 0, 337}, + dictWord{133, 10, 591}, + dictWord{6, 0, 1989}, + dictWord{ + 8, + 0, + 922, + }, + dictWord{8, 0, 978}, + dictWord{133, 11, 374}, + dictWord{132, 0, 638}, + dictWord{138, 0, 500}, + dictWord{133, 11, 731}, + dictWord{5, 10, 380}, + dictWord{ + 5, + 10, + 650, + }, + dictWord{136, 10, 310}, + dictWord{138, 11, 381}, + dictWord{4, 10, 364}, + dictWord{7, 10, 1156}, + dictWord{7, 10, 1187}, + dictWord{137, 10, 409}, + dictWord{137, 11, 224}, + dictWord{140, 0, 166}, + dictWord{134, 10, 482}, + dictWord{4, 11, 626}, + dictWord{5, 11, 642}, + dictWord{6, 11, 425}, + dictWord{ + 10, + 11, + 202, + }, + dictWord{139, 11, 141}, + dictWord{4, 10, 781}, + dictWord{6, 10, 487}, + dictWord{7, 10, 926}, + dictWord{8, 10, 263}, + dictWord{139, 10, 500}, + dictWord{ + 135, + 0, + 418, + }, + dictWord{4, 10, 94}, + dictWord{135, 10, 1265}, + dictWord{136, 0, 760}, + dictWord{132, 10, 417}, + dictWord{136, 11, 835}, + dictWord{5, 10, 348}, + dictWord{134, 10, 522}, + dictWord{6, 0, 1277}, + dictWord{134, 0, 1538}, + dictWord{139, 11, 541}, + dictWord{135, 11, 1597}, + dictWord{5, 11, 384}, + dictWord{ + 8, + 11, + 455, + }, + dictWord{140, 11, 48}, + dictWord{136, 0, 770}, + dictWord{5, 11, 264}, + dictWord{134, 11, 184}, + dictWord{4, 0, 89}, + dictWord{5, 0, 489}, + dictWord{ + 6, + 0, + 315, + }, + dictWord{7, 0, 553}, + dictWord{7, 0, 1745}, + dictWord{138, 0, 243}, + dictWord{4, 10, 408}, + dictWord{4, 10, 741}, + dictWord{135, 10, 500}, + dictWord{ + 134, + 0, + 1396, + }, + dictWord{133, 0, 560}, + dictWord{6, 0, 1658}, + dictWord{9, 0, 3}, + dictWord{10, 0, 154}, + dictWord{11, 0, 641}, + dictWord{13, 0, 85}, + dictWord{13, 0, 201}, + dictWord{141, 0, 346}, + dictWord{135, 11, 1595}, + dictWord{5, 11, 633}, + dictWord{6, 11, 28}, + dictWord{7, 11, 219}, + dictWord{135, 11, 1323}, + dictWord{ + 9, + 11, + 769, + }, + dictWord{140, 11, 185}, + dictWord{135, 11, 785}, + dictWord{7, 11, 359}, + dictWord{8, 11, 243}, + dictWord{140, 11, 175}, + dictWord{138, 0, 586}, + dictWord{ + 7, + 0, + 1271, + }, + dictWord{134, 10, 73}, + dictWord{132, 11, 105}, + dictWord{4, 0, 166}, + dictWord{5, 0, 505}, + dictWord{134, 0, 1670}, + dictWord{133, 10, 576}, + dictWord{4, 11, 324}, + dictWord{138, 11, 104}, + dictWord{142, 10, 231}, + dictWord{6, 0, 637}, + dictWord{7, 10, 1264}, + dictWord{7, 10, 1678}, + dictWord{ + 11, + 10, + 945, + }, + dictWord{12, 10, 341}, + dictWord{12, 10, 471}, + dictWord{12, 10, 569}, + dictWord{23, 11, 21}, + dictWord{151, 11, 23}, + dictWord{8, 11, 559}, + dictWord{ + 141, + 11, + 109, + }, + dictWord{134, 0, 1947}, + dictWord{7, 0, 445}, + dictWord{8, 0, 307}, + dictWord{8, 0, 704}, + dictWord{10, 0, 41}, + dictWord{10, 0, 439}, + dictWord{ + 11, + 0, + 237, + }, + dictWord{11, 0, 622}, + dictWord{140, 0, 201}, + dictWord{135, 11, 963}, + dictWord{135, 0, 1977}, + dictWord{4, 0, 189}, + dictWord{5, 0, 713}, + dictWord{ + 136, + 0, + 57, + }, + dictWord{138, 0, 371}, + dictWord{135, 10, 538}, + dictWord{132, 0, 552}, + dictWord{6, 0, 883}, + dictWord{133, 10, 413}, + dictWord{6, 0, 923}, + dictWord{ + 132, + 11, + 758, + }, + dictWord{138, 11, 215}, + dictWord{136, 10, 495}, + dictWord{7, 10, 54}, + dictWord{8, 10, 312}, + dictWord{10, 10, 191}, + dictWord{10, 10, 614}, + dictWord{140, 10, 567}, + dictWord{7, 11, 351}, + dictWord{139, 11, 128}, + dictWord{7, 0, 875}, + dictWord{6, 10, 468}, + dictWord{7, 10, 1478}, + dictWord{8, 10, 530}, + dictWord{142, 10, 290}, + dictWord{135, 0, 1788}, + dictWord{17, 0, 49}, + dictWord{133, 11, 918}, + dictWord{12, 11, 398}, + dictWord{20, 11, 39}, + dictWord{ + 21, + 11, + 11, + }, + dictWord{150, 11, 41}, + dictWord{10, 0, 661}, + dictWord{6, 10, 484}, + dictWord{135, 10, 822}, + dictWord{135, 0, 1945}, + dictWord{134, 0, 794}, + dictWord{ + 137, + 10, + 900, + }, + dictWord{135, 10, 1335}, + dictWord{6, 10, 1724}, + dictWord{135, 10, 2022}, + dictWord{132, 11, 340}, + dictWord{134, 0, 1135}, + dictWord{ + 4, + 0, + 784, + }, + dictWord{133, 0, 745}, + dictWord{5, 0, 84}, + dictWord{134, 0, 163}, + dictWord{133, 0, 410}, + dictWord{4, 0, 976}, + dictWord{5, 11, 985}, + dictWord{7, 11, 509}, + dictWord{7, 11, 529}, + dictWord{145, 11, 96}, + dictWord{132, 10, 474}, + dictWord{134, 0, 703}, + dictWord{135, 11, 1919}, + dictWord{5, 0, 322}, + dictWord{ + 8, + 0, + 186, + }, + dictWord{9, 0, 262}, + dictWord{10, 0, 187}, + dictWord{142, 0, 208}, + dictWord{135, 10, 1504}, + dictWord{133, 0, 227}, + dictWord{9, 0, 560}, + dictWord{ + 13, + 0, + 208, + }, + dictWord{133, 10, 305}, + dictWord{132, 11, 247}, + dictWord{7, 0, 1395}, + dictWord{8, 0, 486}, + dictWord{9, 0, 236}, + dictWord{9, 0, 878}, + dictWord{ + 10, + 0, + 218, + }, + dictWord{11, 0, 95}, + dictWord{19, 0, 17}, + dictWord{147, 0, 31}, + dictWord{7, 0, 2043}, + dictWord{8, 0, 672}, + dictWord{141, 0, 448}, + dictWord{4, 11, 184}, + dictWord{5, 11, 390}, + dictWord{6, 11, 337}, + dictWord{7, 11, 23}, + dictWord{7, 11, 494}, + dictWord{7, 11, 618}, + dictWord{7, 11, 1456}, + dictWord{8, 11, 27}, + dictWord{ + 8, + 11, + 599, + }, + dictWord{10, 11, 153}, + dictWord{139, 11, 710}, + dictWord{135, 0, 466}, + dictWord{135, 10, 1236}, + dictWord{6, 0, 167}, + dictWord{7, 0, 186}, + dictWord{7, 0, 656}, + dictWord{10, 0, 643}, + dictWord{4, 10, 480}, + dictWord{6, 10, 302}, + dictWord{6, 10, 1642}, + dictWord{7, 10, 837}, + dictWord{7, 10, 1547}, + dictWord{ + 7, + 10, + 1657, + }, + dictWord{8, 10, 429}, + dictWord{9, 10, 228}, + dictWord{13, 10, 289}, + dictWord{13, 10, 343}, + dictWord{147, 10, 101}, + dictWord{134, 0, 1428}, + dictWord{134, 0, 1440}, + dictWord{5, 0, 412}, + dictWord{7, 10, 278}, + dictWord{10, 10, 739}, + dictWord{11, 10, 708}, + dictWord{141, 10, 348}, + dictWord{ + 134, + 0, + 1118, + }, + dictWord{136, 0, 562}, + dictWord{148, 11, 46}, + dictWord{9, 0, 316}, + dictWord{139, 0, 256}, + dictWord{134, 0, 1771}, + dictWord{135, 0, 1190}, + dictWord{137, 0, 132}, + dictWord{10, 11, 227}, + dictWord{11, 11, 497}, + dictWord{11, 11, 709}, + dictWord{140, 11, 415}, + dictWord{143, 0, 66}, + dictWord{6, 11, 360}, + dictWord{7, 11, 1664}, + dictWord{136, 11, 478}, + dictWord{144, 10, 28}, + dictWord{4, 0, 317}, + dictWord{135, 0, 1279}, + dictWord{5, 0, 63}, + dictWord{ + 133, + 0, + 509, + }, + dictWord{136, 11, 699}, + dictWord{145, 10, 36}, + dictWord{134, 0, 1475}, + dictWord{11, 11, 343}, + dictWord{142, 11, 127}, + dictWord{132, 11, 739}, + dictWord{132, 0, 288}, + dictWord{135, 11, 1757}, + dictWord{8, 0, 89}, + dictWord{8, 0, 620}, + dictWord{9, 0, 608}, + dictWord{11, 0, 628}, + dictWord{12, 0, 322}, + dictWord{143, 0, 124}, + dictWord{134, 0, 1225}, + dictWord{7, 0, 1189}, + dictWord{4, 11, 67}, + dictWord{5, 11, 422}, + dictWord{6, 10, 363}, + dictWord{7, 11, 1037}, + dictWord{7, 11, 1289}, + dictWord{7, 11, 1555}, + dictWord{7, 10, 1955}, + dictWord{8, 10, 725}, + dictWord{9, 11, 741}, + dictWord{145, 11, 108}, + dictWord{ + 134, + 0, + 1468, + }, + dictWord{6, 0, 689}, + dictWord{134, 0, 1451}, + dictWord{138, 0, 120}, + dictWord{151, 0, 1}, + dictWord{137, 10, 805}, + dictWord{142, 0, 329}, + dictWord{ + 5, + 10, + 813, + }, + dictWord{135, 10, 2046}, + dictWord{135, 0, 226}, + dictWord{138, 11, 96}, + dictWord{7, 0, 1855}, + dictWord{5, 10, 712}, + dictWord{11, 10, 17}, + dictWord{13, 10, 321}, + dictWord{144, 10, 67}, + dictWord{9, 0, 461}, + dictWord{6, 10, 320}, + dictWord{7, 10, 781}, + dictWord{7, 10, 1921}, + dictWord{9, 10, 55}, + dictWord{ + 10, + 10, + 186, + }, + dictWord{10, 10, 273}, + dictWord{10, 10, 664}, + dictWord{10, 10, 801}, + dictWord{11, 10, 996}, + dictWord{11, 10, 997}, + dictWord{13, 10, 157}, + dictWord{142, 10, 170}, + dictWord{8, 11, 203}, + dictWord{8, 10, 271}, + dictWord{11, 11, 823}, + dictWord{11, 11, 846}, + dictWord{12, 11, 482}, + dictWord{ + 13, + 11, + 133, + }, + dictWord{13, 11, 277}, + dictWord{13, 11, 302}, + dictWord{13, 11, 464}, + dictWord{14, 11, 205}, + dictWord{142, 11, 221}, + dictWord{135, 0, 1346}, + dictWord{4, 11, 449}, + dictWord{133, 11, 718}, + dictWord{134, 0, 85}, + dictWord{14, 0, 299}, + dictWord{7, 10, 103}, + dictWord{7, 10, 863}, + dictWord{11, 10, 184}, + dictWord{145, 10, 62}, + dictWord{4, 11, 355}, + dictWord{6, 11, 311}, + dictWord{9, 11, 256}, + dictWord{138, 11, 404}, + dictWord{137, 10, 659}, + dictWord{ + 138, + 11, + 758, + }, + dictWord{133, 11, 827}, + dictWord{5, 11, 64}, + dictWord{140, 11, 581}, + dictWord{134, 0, 1171}, + dictWord{4, 11, 442}, + dictWord{7, 11, 1047}, + dictWord{ + 7, + 11, + 1352, + }, + dictWord{135, 11, 1643}, + dictWord{132, 0, 980}, + dictWord{5, 11, 977}, + dictWord{6, 11, 288}, + dictWord{7, 11, 528}, + dictWord{135, 11, 1065}, + dictWord{5, 0, 279}, + dictWord{6, 0, 235}, + dictWord{7, 0, 468}, + dictWord{8, 0, 446}, + dictWord{9, 0, 637}, + dictWord{10, 0, 717}, + dictWord{11, 0, 738}, + dictWord{ + 140, + 0, + 514, + }, + dictWord{132, 0, 293}, + dictWord{11, 10, 337}, + dictWord{142, 10, 303}, + dictWord{136, 11, 285}, + dictWord{5, 0, 17}, + dictWord{6, 0, 371}, + dictWord{ + 9, + 0, + 528, + }, + dictWord{12, 0, 364}, + dictWord{132, 11, 254}, + dictWord{5, 10, 77}, + dictWord{7, 10, 1455}, + dictWord{10, 10, 843}, + dictWord{147, 10, 73}, + dictWord{ + 150, + 0, + 5, + }, + dictWord{132, 10, 458}, + dictWord{6, 11, 12}, + dictWord{7, 11, 1219}, + dictWord{145, 11, 73}, + dictWord{135, 10, 1420}, + dictWord{6, 10, 109}, + dictWord{138, 10, 382}, + dictWord{135, 11, 125}, + dictWord{6, 10, 330}, + dictWord{7, 10, 1084}, + dictWord{139, 10, 142}, + dictWord{6, 11, 369}, + dictWord{ + 6, + 11, + 502, + }, + dictWord{7, 11, 1036}, + dictWord{8, 11, 348}, + dictWord{9, 11, 452}, + dictWord{10, 11, 26}, + dictWord{11, 11, 224}, + dictWord{11, 11, 387}, + dictWord{ + 11, + 11, + 772, + }, + dictWord{12, 11, 95}, + dictWord{12, 11, 629}, + dictWord{13, 11, 195}, + dictWord{13, 11, 207}, + dictWord{13, 11, 241}, + dictWord{14, 11, 260}, + dictWord{ + 14, + 11, + 270, + }, + dictWord{143, 11, 140}, + dictWord{132, 11, 269}, + dictWord{5, 11, 480}, + dictWord{7, 11, 532}, + dictWord{7, 11, 1197}, + dictWord{7, 11, 1358}, + dictWord{8, 11, 291}, + dictWord{11, 11, 349}, + dictWord{142, 11, 396}, + dictWord{150, 0, 48}, + dictWord{10, 0, 601}, + dictWord{13, 0, 353}, + dictWord{141, 0, 376}, + dictWord{5, 0, 779}, + dictWord{5, 0, 807}, + dictWord{6, 0, 1655}, + dictWord{134, 0, 1676}, + dictWord{142, 11, 223}, + dictWord{4, 0, 196}, + dictWord{5, 0, 558}, + dictWord{133, 0, 949}, + dictWord{148, 11, 15}, + dictWord{135, 11, 1764}, + dictWord{134, 0, 1322}, + dictWord{132, 0, 752}, + dictWord{139, 0, 737}, + dictWord{ + 135, + 11, + 657, + }, + dictWord{136, 11, 533}, + dictWord{135, 0, 412}, + dictWord{4, 0, 227}, + dictWord{5, 0, 159}, + dictWord{5, 0, 409}, + dictWord{7, 0, 80}, + dictWord{8, 0, 556}, + dictWord{10, 0, 479}, + dictWord{12, 0, 418}, + dictWord{14, 0, 50}, + dictWord{14, 0, 123}, + dictWord{14, 0, 192}, + dictWord{14, 0, 249}, + dictWord{14, 0, 295}, + dictWord{143, 0, 27}, + dictWord{7, 0, 1470}, + dictWord{8, 0, 66}, + dictWord{8, 0, 137}, + dictWord{8, 0, 761}, + dictWord{9, 0, 638}, + dictWord{11, 0, 80}, + dictWord{11, 0, 212}, + dictWord{11, 0, 368}, + dictWord{11, 0, 418}, + dictWord{12, 0, 8}, + dictWord{13, 0, 15}, + dictWord{16, 0, 61}, + dictWord{17, 0, 59}, + dictWord{19, 0, 28}, + dictWord{ + 148, + 0, + 84, + }, + dictWord{135, 10, 1985}, + dictWord{4, 11, 211}, + dictWord{4, 11, 332}, + dictWord{5, 11, 335}, + dictWord{6, 11, 238}, + dictWord{7, 11, 269}, + dictWord{ + 7, + 11, + 811, + }, + dictWord{7, 11, 1797}, + dictWord{8, 10, 122}, + dictWord{8, 11, 836}, + dictWord{9, 11, 507}, + dictWord{141, 11, 242}, + dictWord{6, 0, 683}, + dictWord{ + 134, + 0, + 1252, + }, + dictWord{4, 0, 873}, + dictWord{132, 10, 234}, + dictWord{134, 0, 835}, + dictWord{6, 0, 38}, + dictWord{7, 0, 1220}, + dictWord{8, 0, 185}, + dictWord{8, 0, 256}, + dictWord{9, 0, 22}, + dictWord{9, 0, 331}, + dictWord{10, 0, 738}, + dictWord{11, 0, 205}, + dictWord{11, 0, 540}, + dictWord{11, 0, 746}, + dictWord{13, 0, 465}, + dictWord{ + 14, + 0, + 88, + }, + dictWord{142, 0, 194}, + dictWord{138, 0, 986}, + dictWord{5, 11, 1009}, + dictWord{12, 11, 582}, + dictWord{146, 11, 131}, + dictWord{4, 0, 159}, + dictWord{ + 6, + 0, + 115, + }, + dictWord{7, 0, 252}, + dictWord{7, 0, 257}, + dictWord{7, 0, 1928}, + dictWord{8, 0, 69}, + dictWord{9, 0, 384}, + dictWord{10, 0, 91}, + dictWord{10, 0, 615}, + dictWord{ + 12, + 0, + 375, + }, + dictWord{14, 0, 235}, + dictWord{18, 0, 117}, + dictWord{147, 0, 123}, + dictWord{133, 0, 911}, + dictWord{136, 0, 278}, + dictWord{5, 10, 430}, + dictWord{ + 5, + 10, + 932, + }, + dictWord{6, 10, 131}, + dictWord{7, 10, 417}, + dictWord{9, 10, 522}, + dictWord{11, 10, 314}, + dictWord{141, 10, 390}, + dictWord{14, 10, 149}, + dictWord{14, 10, 399}, + dictWord{143, 10, 57}, + dictWord{4, 0, 151}, + dictWord{7, 0, 1567}, + dictWord{136, 0, 749}, + dictWord{5, 11, 228}, + dictWord{6, 11, 203}, + dictWord{ + 7, + 11, + 156, + }, + dictWord{8, 11, 347}, + dictWord{137, 11, 265}, + dictWord{132, 10, 507}, + dictWord{10, 0, 989}, + dictWord{140, 0, 956}, + dictWord{133, 0, 990}, + dictWord{5, 0, 194}, + dictWord{6, 0, 927}, + dictWord{7, 0, 1662}, + dictWord{9, 0, 90}, + dictWord{140, 0, 564}, + dictWord{4, 10, 343}, + dictWord{133, 10, 511}, + dictWord{133, 0, 425}, + dictWord{7, 10, 455}, + dictWord{138, 10, 591}, + dictWord{4, 0, 774}, + dictWord{7, 11, 476}, + dictWord{7, 11, 1592}, + dictWord{138, 11, 87}, + dictWord{5, 0, 971}, + dictWord{135, 10, 1381}, + dictWord{5, 11, 318}, + dictWord{147, 11, 121}, + dictWord{5, 11, 291}, + dictWord{7, 11, 765}, + dictWord{9, 11, 389}, + dictWord{140, 11, 548}, + dictWord{134, 10, 575}, + dictWord{4, 0, 827}, + dictWord{12, 0, 646}, + dictWord{12, 0, 705}, + dictWord{12, 0, 712}, + dictWord{140, 0, 714}, + dictWord{139, 0, 752}, + dictWord{137, 0, 662}, + dictWord{5, 0, 72}, + dictWord{6, 0, 264}, + dictWord{7, 0, 21}, + dictWord{7, 0, 46}, + dictWord{7, 0, 2013}, + dictWord{ + 8, + 0, + 215, + }, + dictWord{8, 0, 513}, + dictWord{10, 0, 266}, + dictWord{139, 0, 22}, + dictWord{139, 11, 522}, + dictWord{6, 0, 239}, + dictWord{7, 0, 118}, + dictWord{10, 0, 95}, + dictWord{11, 0, 603}, + dictWord{13, 0, 443}, + dictWord{14, 0, 160}, + dictWord{143, 0, 4}, + dictWord{6, 0, 431}, + dictWord{134, 0, 669}, + dictWord{7, 10, 1127}, + dictWord{ + 7, + 10, + 1572, + }, + dictWord{10, 10, 297}, + dictWord{10, 10, 422}, + dictWord{11, 10, 764}, + dictWord{11, 10, 810}, + dictWord{12, 10, 264}, + dictWord{13, 10, 102}, + dictWord{13, 10, 300}, + dictWord{13, 10, 484}, + dictWord{14, 10, 147}, + dictWord{14, 10, 229}, + dictWord{17, 10, 71}, + dictWord{18, 10, 118}, + dictWord{ + 147, + 10, + 120, + }, + dictWord{5, 0, 874}, + dictWord{6, 0, 1677}, + dictWord{15, 0, 0}, + dictWord{10, 11, 525}, + dictWord{139, 11, 82}, + dictWord{6, 0, 65}, + dictWord{7, 0, 939}, + dictWord{ + 7, + 0, + 1172, + }, + dictWord{7, 0, 1671}, + dictWord{9, 0, 540}, + dictWord{10, 0, 696}, + dictWord{11, 0, 265}, + dictWord{11, 0, 732}, + dictWord{11, 0, 928}, + dictWord{ + 11, + 0, + 937, + }, + dictWord{141, 0, 438}, + dictWord{134, 0, 1350}, + dictWord{136, 11, 547}, + dictWord{132, 11, 422}, + dictWord{5, 11, 355}, + dictWord{145, 11, 0}, + dictWord{137, 11, 905}, + dictWord{5, 0, 682}, + dictWord{135, 0, 1887}, + dictWord{132, 0, 809}, + dictWord{4, 0, 696}, + dictWord{133, 11, 865}, + dictWord{6, 0, 1074}, + dictWord{6, 0, 1472}, + dictWord{14, 10, 35}, + dictWord{142, 10, 191}, + dictWord{5, 11, 914}, + dictWord{134, 11, 1625}, + dictWord{133, 11, 234}, + dictWord{ + 135, + 11, + 1383, + }, + dictWord{137, 11, 780}, + dictWord{132, 10, 125}, + dictWord{4, 0, 726}, + dictWord{133, 0, 630}, + dictWord{8, 0, 802}, + dictWord{136, 0, 838}, + dictWord{132, 10, 721}, + dictWord{6, 0, 1337}, + dictWord{7, 0, 776}, + dictWord{19, 0, 56}, + dictWord{136, 10, 145}, + dictWord{132, 0, 970}, + dictWord{7, 10, 792}, + dictWord{8, 10, 147}, + dictWord{10, 10, 821}, + dictWord{139, 10, 1021}, + dictWord{139, 10, 970}, + dictWord{8, 0, 940}, + dictWord{137, 0, 797}, + dictWord{ + 135, + 11, + 1312, + }, + dictWord{9, 0, 248}, + dictWord{10, 0, 400}, + dictWord{7, 11, 816}, + dictWord{7, 11, 1241}, + dictWord{7, 10, 1999}, + dictWord{9, 11, 283}, + dictWord{ + 9, + 11, + 520, + }, + dictWord{10, 11, 213}, + dictWord{10, 11, 307}, + dictWord{10, 11, 463}, + dictWord{10, 11, 671}, + dictWord{10, 11, 746}, + dictWord{11, 11, 401}, + dictWord{ + 11, + 11, + 794, + }, + dictWord{12, 11, 517}, + dictWord{18, 11, 107}, + dictWord{147, 11, 115}, + dictWord{6, 0, 1951}, + dictWord{134, 0, 2040}, + dictWord{ + 135, + 11, + 339, + }, + dictWord{13, 0, 41}, + dictWord{15, 0, 93}, + dictWord{5, 10, 168}, + dictWord{5, 10, 930}, + dictWord{8, 10, 74}, + dictWord{9, 10, 623}, + dictWord{12, 10, 500}, + dictWord{140, 10, 579}, + dictWord{6, 0, 118}, + dictWord{7, 0, 215}, + dictWord{7, 0, 1521}, + dictWord{140, 0, 11}, + dictWord{6, 10, 220}, + dictWord{7, 10, 1101}, + dictWord{141, 10, 105}, + dictWord{6, 11, 421}, + dictWord{7, 11, 61}, + dictWord{7, 11, 1540}, + dictWord{10, 11, 11}, + dictWord{138, 11, 501}, + dictWord{7, 0, 615}, + dictWord{138, 0, 251}, + dictWord{140, 11, 631}, + dictWord{135, 0, 1044}, + dictWord{6, 10, 19}, + dictWord{7, 10, 1413}, + dictWord{139, 10, 428}, + dictWord{ + 133, + 0, + 225, + }, + dictWord{7, 10, 96}, + dictWord{8, 10, 401}, + dictWord{8, 10, 703}, + dictWord{137, 10, 896}, + dictWord{145, 10, 116}, + dictWord{6, 11, 102}, + dictWord{ + 7, + 11, + 72, + }, + dictWord{15, 11, 142}, + dictWord{147, 11, 67}, + dictWord{7, 10, 1961}, + dictWord{7, 10, 1965}, + dictWord{8, 10, 702}, + dictWord{136, 10, 750}, + dictWord{ + 7, + 10, + 2030, + }, + dictWord{8, 10, 150}, + dictWord{8, 10, 737}, + dictWord{12, 10, 366}, + dictWord{151, 11, 30}, + dictWord{4, 0, 370}, + dictWord{5, 0, 756}, + dictWord{ + 7, + 0, + 1326, + }, + dictWord{135, 11, 823}, + dictWord{8, 10, 800}, + dictWord{9, 10, 148}, + dictWord{9, 10, 872}, + dictWord{9, 10, 890}, + dictWord{11, 10, 309}, + dictWord{ + 11, + 10, + 1001, + }, + dictWord{13, 10, 267}, + dictWord{141, 10, 323}, + dictWord{6, 0, 1662}, + dictWord{7, 0, 48}, + dictWord{8, 0, 771}, + dictWord{10, 0, 116}, + dictWord{ + 13, + 0, + 104, + }, + dictWord{14, 0, 105}, + dictWord{14, 0, 184}, + dictWord{15, 0, 168}, + dictWord{19, 0, 92}, + dictWord{148, 0, 68}, + dictWord{10, 0, 209}, + dictWord{ + 135, + 11, + 1870, + }, + dictWord{7, 11, 68}, + dictWord{8, 11, 48}, + dictWord{8, 11, 88}, + dictWord{8, 11, 582}, + dictWord{8, 11, 681}, + dictWord{9, 11, 373}, + dictWord{9, 11, 864}, + dictWord{11, 11, 157}, + dictWord{11, 11, 336}, + dictWord{11, 11, 843}, + dictWord{148, 11, 27}, + dictWord{134, 0, 930}, + dictWord{4, 11, 88}, + dictWord{5, 11, 137}, + dictWord{5, 11, 174}, + dictWord{5, 11, 777}, + dictWord{6, 11, 1664}, + dictWord{6, 11, 1725}, + dictWord{7, 11, 77}, + dictWord{7, 11, 426}, + dictWord{7, 11, 1317}, + dictWord{7, 11, 1355}, + dictWord{8, 11, 126}, + dictWord{8, 11, 563}, + dictWord{9, 11, 523}, + dictWord{9, 11, 750}, + dictWord{10, 11, 310}, + dictWord{10, 11, 836}, + dictWord{11, 11, 42}, + dictWord{11, 11, 318}, + dictWord{11, 11, 731}, + dictWord{12, 11, 68}, + dictWord{12, 11, 92}, + dictWord{12, 11, 507}, + dictWord{12, 11, 692}, + dictWord{13, 11, 81}, + dictWord{13, 11, 238}, + dictWord{13, 11, 374}, + dictWord{18, 11, 138}, + dictWord{19, 11, 78}, + dictWord{19, 11, 111}, + dictWord{20, 11, 55}, + dictWord{20, 11, 77}, + dictWord{148, 11, 92}, + dictWord{4, 11, 938}, + dictWord{135, 11, 1831}, + dictWord{5, 10, 547}, + dictWord{7, 10, 424}, + dictWord{ + 8, + 11, + 617, + }, + dictWord{138, 11, 351}, + dictWord{6, 0, 1286}, + dictWord{6, 11, 1668}, + dictWord{7, 11, 1499}, + dictWord{8, 11, 117}, + dictWord{9, 11, 314}, + dictWord{ + 138, + 11, + 174, + }, + dictWord{6, 0, 759}, + dictWord{6, 0, 894}, + dictWord{7, 11, 707}, + dictWord{139, 11, 563}, + dictWord{4, 0, 120}, + dictWord{135, 0, 1894}, + dictWord{ + 9, + 0, + 385, + }, + dictWord{149, 0, 17}, + dictWord{138, 0, 429}, + dictWord{133, 11, 403}, + dictWord{5, 0, 820}, + dictWord{135, 0, 931}, + dictWord{10, 0, 199}, + dictWord{ + 133, + 10, + 133, + }, + dictWord{6, 0, 151}, + dictWord{6, 0, 1675}, + dictWord{7, 0, 383}, + dictWord{151, 0, 10}, + dictWord{6, 0, 761}, + dictWord{136, 10, 187}, + dictWord{ + 8, + 0, + 365, + }, + dictWord{10, 10, 0}, + dictWord{10, 10, 818}, + dictWord{139, 10, 988}, + dictWord{4, 11, 44}, + dictWord{5, 11, 311}, + dictWord{6, 11, 156}, + dictWord{ + 7, + 11, + 639, + }, + dictWord{7, 11, 762}, + dictWord{7, 11, 1827}, + dictWord{9, 11, 8}, + dictWord{9, 11, 462}, + dictWord{148, 11, 83}, + dictWord{4, 11, 346}, + dictWord{7, 11, 115}, + dictWord{9, 11, 180}, + dictWord{9, 11, 456}, + dictWord{138, 11, 363}, + dictWord{136, 10, 685}, + dictWord{7, 0, 1086}, + dictWord{145, 0, 46}, + dictWord{ + 6, + 0, + 1624, + }, + dictWord{11, 0, 11}, + dictWord{12, 0, 422}, + dictWord{13, 0, 444}, + dictWord{142, 0, 360}, + dictWord{6, 0, 1020}, + dictWord{6, 0, 1260}, + dictWord{ + 134, + 0, + 1589, + }, + dictWord{4, 0, 43}, + dictWord{5, 0, 344}, + dictWord{5, 0, 357}, + dictWord{14, 0, 472}, + dictWord{150, 0, 58}, + dictWord{6, 0, 1864}, + dictWord{6, 0, 1866}, + dictWord{6, 0, 1868}, + dictWord{6, 0, 1869}, + dictWord{6, 0, 1874}, + dictWord{6, 0, 1877}, + dictWord{6, 0, 1903}, + dictWord{6, 0, 1911}, + dictWord{9, 0, 920}, + dictWord{ + 9, + 0, + 921, + }, + dictWord{9, 0, 924}, + dictWord{9, 0, 946}, + dictWord{9, 0, 959}, + dictWord{9, 0, 963}, + dictWord{9, 0, 970}, + dictWord{9, 0, 997}, + dictWord{9, 0, 1008}, + dictWord{ + 9, + 0, + 1017, + }, + dictWord{12, 0, 795}, + dictWord{12, 0, 797}, + dictWord{12, 0, 798}, + dictWord{12, 0, 800}, + dictWord{12, 0, 803}, + dictWord{12, 0, 811}, + dictWord{ + 12, + 0, + 820, + }, + dictWord{12, 0, 821}, + dictWord{12, 0, 839}, + dictWord{12, 0, 841}, + dictWord{12, 0, 848}, + dictWord{12, 0, 911}, + dictWord{12, 0, 921}, + dictWord{12, 0, 922}, + dictWord{12, 0, 925}, + dictWord{12, 0, 937}, + dictWord{12, 0, 944}, + dictWord{12, 0, 945}, + dictWord{12, 0, 953}, + dictWord{15, 0, 184}, + dictWord{15, 0, 191}, + dictWord{15, 0, 199}, + dictWord{15, 0, 237}, + dictWord{15, 0, 240}, + dictWord{15, 0, 243}, + dictWord{15, 0, 246}, + dictWord{18, 0, 203}, + dictWord{21, 0, 40}, + dictWord{ + 21, + 0, + 52, + }, + dictWord{21, 0, 57}, + dictWord{24, 0, 23}, + dictWord{24, 0, 28}, + dictWord{152, 0, 30}, + dictWord{134, 0, 725}, + dictWord{145, 11, 58}, + dictWord{133, 0, 888}, + dictWord{137, 10, 874}, + dictWord{4, 0, 711}, + dictWord{8, 10, 774}, + dictWord{10, 10, 670}, + dictWord{140, 10, 51}, + dictWord{144, 11, 40}, + dictWord{ + 6, + 11, + 185, + }, + dictWord{7, 11, 1899}, + dictWord{139, 11, 673}, + dictWord{137, 10, 701}, + dictWord{137, 0, 440}, + dictWord{4, 11, 327}, + dictWord{5, 11, 478}, + dictWord{ + 7, + 11, + 1332, + }, + dictWord{8, 11, 753}, + dictWord{140, 11, 227}, + dictWord{4, 10, 127}, + dictWord{5, 10, 350}, + dictWord{6, 10, 356}, + dictWord{8, 10, 426}, + dictWord{ + 9, + 10, + 572, + }, + dictWord{10, 10, 247}, + dictWord{139, 10, 312}, + dictWord{5, 11, 1020}, + dictWord{133, 11, 1022}, + dictWord{4, 11, 103}, + dictWord{ + 133, + 11, + 401, + }, + dictWord{6, 0, 1913}, + dictWord{6, 0, 1926}, + dictWord{6, 0, 1959}, + dictWord{9, 0, 914}, + dictWord{9, 0, 939}, + dictWord{9, 0, 952}, + dictWord{9, 0, 979}, + dictWord{ + 9, + 0, + 990, + }, + dictWord{9, 0, 998}, + dictWord{9, 0, 1003}, + dictWord{9, 0, 1023}, + dictWord{12, 0, 827}, + dictWord{12, 0, 834}, + dictWord{12, 0, 845}, + dictWord{ + 12, + 0, + 912, + }, + dictWord{12, 0, 935}, + dictWord{12, 0, 951}, + dictWord{15, 0, 172}, + dictWord{15, 0, 174}, + dictWord{18, 0, 198}, + dictWord{149, 0, 63}, + dictWord{5, 0, 958}, + dictWord{5, 0, 987}, + dictWord{4, 11, 499}, + dictWord{135, 11, 1421}, + dictWord{7, 0, 885}, + dictWord{6, 10, 59}, + dictWord{6, 10, 1762}, + dictWord{9, 10, 603}, + dictWord{141, 10, 397}, + dictWord{10, 11, 62}, + dictWord{141, 11, 164}, + dictWord{4, 0, 847}, + dictWord{135, 0, 326}, + dictWord{11, 0, 276}, + dictWord{142, 0, 293}, + dictWord{4, 0, 65}, + dictWord{5, 0, 479}, + dictWord{5, 0, 1004}, + dictWord{7, 0, 1913}, + dictWord{8, 0, 317}, + dictWord{9, 0, 302}, + dictWord{10, 0, 612}, + dictWord{ + 13, + 0, + 22, + }, + dictWord{132, 11, 96}, + dictWord{4, 0, 261}, + dictWord{135, 0, 510}, + dictWord{135, 0, 1514}, + dictWord{6, 10, 111}, + dictWord{7, 10, 4}, + dictWord{8, 10, 163}, + dictWord{8, 10, 776}, + dictWord{138, 10, 566}, + dictWord{4, 0, 291}, + dictWord{9, 0, 515}, + dictWord{12, 0, 152}, + dictWord{12, 0, 443}, + dictWord{13, 0, 392}, + dictWord{142, 0, 357}, + dictWord{7, 11, 399}, + dictWord{135, 11, 1492}, + dictWord{4, 0, 589}, + dictWord{139, 0, 282}, + dictWord{6, 11, 563}, + dictWord{ + 135, + 10, + 1994, + }, + dictWord{5, 10, 297}, + dictWord{135, 10, 1038}, + dictWord{4, 0, 130}, + dictWord{7, 0, 843}, + dictWord{135, 0, 1562}, + dictWord{5, 0, 42}, + dictWord{ + 5, + 0, + 879, + }, + dictWord{7, 0, 245}, + dictWord{7, 0, 324}, + dictWord{7, 0, 1532}, + dictWord{11, 0, 463}, + dictWord{11, 0, 472}, + dictWord{13, 0, 363}, + dictWord{144, 0, 52}, + dictWord{4, 0, 134}, + dictWord{133, 0, 372}, + dictWord{133, 0, 680}, + dictWord{136, 10, 363}, + dictWord{6, 0, 1997}, + dictWord{8, 0, 935}, + dictWord{136, 0, 977}, + dictWord{4, 0, 810}, + dictWord{135, 0, 1634}, + dictWord{135, 10, 1675}, + dictWord{7, 0, 1390}, + dictWord{4, 11, 910}, + dictWord{133, 11, 832}, + dictWord{ + 7, + 10, + 808, + }, + dictWord{8, 11, 266}, + dictWord{139, 11, 578}, + dictWord{132, 0, 644}, + dictWord{4, 0, 982}, + dictWord{138, 0, 867}, + dictWord{132, 10, 280}, + dictWord{ + 135, + 0, + 540, + }, + dictWord{140, 10, 54}, + dictWord{135, 0, 123}, + dictWord{134, 0, 1978}, + dictWord{4, 10, 421}, + dictWord{133, 10, 548}, + dictWord{6, 0, 623}, + dictWord{136, 0, 789}, + dictWord{4, 0, 908}, + dictWord{5, 0, 359}, + dictWord{5, 0, 508}, + dictWord{6, 0, 1723}, + dictWord{7, 0, 343}, + dictWord{7, 0, 1996}, + dictWord{ + 135, + 0, + 2026, + }, + dictWord{134, 0, 1220}, + dictWord{4, 0, 341}, + dictWord{135, 0, 480}, + dictWord{6, 10, 254}, + dictWord{9, 10, 109}, + dictWord{138, 10, 103}, + dictWord{ + 134, + 0, + 888, + }, + dictWord{8, 11, 528}, + dictWord{137, 11, 348}, + dictWord{7, 0, 1995}, + dictWord{8, 0, 299}, + dictWord{11, 0, 890}, + dictWord{12, 0, 674}, + dictWord{ + 4, + 11, + 20, + }, + dictWord{133, 11, 616}, + dictWord{135, 11, 1094}, + dictWord{134, 10, 1630}, + dictWord{4, 0, 238}, + dictWord{5, 0, 503}, + dictWord{6, 0, 179}, + dictWord{ + 7, + 0, + 2003, + }, + dictWord{8, 0, 381}, + dictWord{8, 0, 473}, + dictWord{9, 0, 149}, + dictWord{10, 0, 788}, + dictWord{15, 0, 45}, + dictWord{15, 0, 86}, + dictWord{20, 0, 110}, + dictWord{150, 0, 57}, + dictWord{133, 10, 671}, + dictWord{4, 11, 26}, + dictWord{5, 11, 429}, + dictWord{6, 11, 245}, + dictWord{7, 11, 704}, + dictWord{7, 11, 1379}, + dictWord{135, 11, 1474}, + dictWord{4, 0, 121}, + dictWord{5, 0, 156}, + dictWord{5, 0, 349}, + dictWord{9, 0, 431}, + dictWord{10, 0, 605}, + dictWord{142, 0, 342}, + dictWord{ + 7, + 11, + 943, + }, + dictWord{139, 11, 614}, + dictWord{132, 10, 889}, + dictWord{132, 11, 621}, + dictWord{7, 10, 1382}, + dictWord{7, 11, 1382}, + dictWord{ + 135, + 10, + 1910, + }, + dictWord{132, 10, 627}, + dictWord{133, 10, 775}, + dictWord{133, 11, 542}, + dictWord{133, 11, 868}, + dictWord{136, 11, 433}, + dictWord{6, 0, 1373}, + dictWord{7, 0, 1011}, + dictWord{11, 10, 362}, + dictWord{11, 10, 948}, + dictWord{140, 10, 388}, + dictWord{6, 0, 80}, + dictWord{7, 0, 173}, + dictWord{9, 0, 547}, + dictWord{10, 0, 730}, + dictWord{14, 0, 18}, + dictWord{22, 0, 39}, + dictWord{135, 11, 1495}, + dictWord{6, 0, 1694}, + dictWord{135, 0, 1974}, + dictWord{140, 0, 196}, + dictWord{4, 0, 923}, + dictWord{6, 0, 507}, + dictWord{6, 0, 1711}, + dictWord{7, 10, 451}, + dictWord{8, 10, 389}, + dictWord{12, 10, 490}, + dictWord{13, 10, 16}, + dictWord{ + 13, + 10, + 215, + }, + dictWord{13, 10, 351}, + dictWord{18, 10, 132}, + dictWord{147, 10, 125}, + dictWord{6, 0, 646}, + dictWord{134, 0, 1047}, + dictWord{135, 10, 841}, + dictWord{136, 10, 566}, + dictWord{6, 0, 1611}, + dictWord{135, 0, 1214}, + dictWord{139, 0, 926}, + dictWord{132, 11, 525}, + dictWord{132, 0, 595}, + dictWord{ + 5, + 0, + 240, + }, + dictWord{6, 0, 459}, + dictWord{7, 0, 12}, + dictWord{7, 0, 114}, + dictWord{7, 0, 949}, + dictWord{7, 0, 1753}, + dictWord{7, 0, 1805}, + dictWord{8, 0, 658}, + dictWord{ + 9, + 0, + 1, + }, + dictWord{11, 0, 959}, + dictWord{141, 0, 446}, + dictWord{5, 10, 912}, + dictWord{134, 10, 1695}, + dictWord{132, 0, 446}, + dictWord{7, 11, 62}, + dictWord{ + 12, + 11, + 45, + }, + dictWord{147, 11, 112}, + dictWord{5, 10, 236}, + dictWord{6, 10, 572}, + dictWord{8, 10, 492}, + dictWord{11, 10, 618}, + dictWord{144, 10, 56}, + dictWord{ + 5, + 10, + 190, + }, + dictWord{136, 10, 318}, + dictWord{135, 10, 1376}, + dictWord{4, 11, 223}, + dictWord{6, 11, 359}, + dictWord{11, 11, 3}, + dictWord{13, 11, 108}, + dictWord{ + 14, + 11, + 89, + }, + dictWord{144, 11, 22}, + dictWord{132, 11, 647}, + dictWord{134, 0, 490}, + dictWord{134, 0, 491}, + dictWord{134, 0, 1584}, + dictWord{ + 135, + 11, + 685, + }, + dictWord{138, 11, 220}, + dictWord{7, 0, 250}, + dictWord{136, 0, 507}, + dictWord{132, 0, 158}, + dictWord{4, 0, 140}, + dictWord{7, 0, 362}, + dictWord{8, 0, 209}, + dictWord{9, 0, 10}, + dictWord{9, 0, 160}, + dictWord{9, 0, 503}, + dictWord{9, 0, 614}, + dictWord{10, 0, 689}, + dictWord{11, 0, 327}, + dictWord{11, 0, 553}, + dictWord{ + 11, + 0, + 725, + }, + dictWord{11, 0, 767}, + dictWord{12, 0, 252}, + dictWord{12, 0, 583}, + dictWord{13, 0, 192}, + dictWord{14, 0, 269}, + dictWord{14, 0, 356}, + dictWord{148, 0, 50}, + dictWord{19, 0, 1}, + dictWord{19, 0, 26}, + dictWord{150, 0, 9}, + dictWord{132, 11, 109}, + dictWord{6, 0, 228}, + dictWord{7, 0, 1341}, + dictWord{9, 0, 408}, + dictWord{ + 138, + 0, + 343, + }, + dictWord{4, 0, 373}, + dictWord{5, 0, 283}, + dictWord{6, 0, 480}, + dictWord{7, 0, 609}, + dictWord{10, 0, 860}, + dictWord{138, 0, 878}, + dictWord{6, 0, 779}, + dictWord{134, 0, 1209}, + dictWord{4, 0, 557}, + dictWord{7, 11, 263}, + dictWord{7, 11, 628}, + dictWord{136, 11, 349}, + dictWord{132, 0, 548}, + dictWord{7, 0, 197}, + dictWord{8, 0, 142}, + dictWord{8, 0, 325}, + dictWord{9, 0, 150}, + dictWord{9, 0, 596}, + dictWord{10, 0, 350}, + dictWord{10, 0, 353}, + dictWord{11, 0, 74}, + dictWord{ + 11, + 0, + 315, + }, + dictWord{12, 0, 662}, + dictWord{12, 0, 681}, + dictWord{14, 0, 423}, + dictWord{143, 0, 141}, + dictWord{4, 11, 40}, + dictWord{10, 11, 67}, + dictWord{ + 11, + 11, + 117, + }, + dictWord{11, 11, 768}, + dictWord{139, 11, 935}, + dictWord{7, 11, 992}, + dictWord{8, 11, 301}, + dictWord{9, 11, 722}, + dictWord{12, 11, 63}, + dictWord{ + 13, + 11, + 29, + }, + dictWord{14, 11, 161}, + dictWord{143, 11, 18}, + dictWord{6, 0, 1490}, + dictWord{138, 11, 532}, + dictWord{5, 0, 580}, + dictWord{7, 0, 378}, + dictWord{ + 7, + 0, + 674, + }, + dictWord{7, 0, 1424}, + dictWord{15, 0, 83}, + dictWord{16, 0, 11}, + dictWord{15, 11, 83}, + dictWord{144, 11, 11}, + dictWord{6, 0, 1057}, + dictWord{6, 0, 1335}, + dictWord{10, 0, 316}, + dictWord{7, 10, 85}, + dictWord{7, 10, 247}, + dictWord{8, 10, 585}, + dictWord{138, 10, 163}, + dictWord{4, 0, 169}, + dictWord{5, 0, 83}, + dictWord{ + 6, + 0, + 399, + }, + dictWord{6, 0, 579}, + dictWord{6, 0, 1513}, + dictWord{7, 0, 692}, + dictWord{7, 0, 846}, + dictWord{7, 0, 1015}, + dictWord{7, 0, 1799}, + dictWord{8, 0, 403}, + dictWord{9, 0, 394}, + dictWord{10, 0, 133}, + dictWord{12, 0, 4}, + dictWord{12, 0, 297}, + dictWord{12, 0, 452}, + dictWord{16, 0, 81}, + dictWord{18, 0, 25}, + dictWord{21, 0, 14}, + dictWord{22, 0, 12}, + dictWord{151, 0, 18}, + dictWord{134, 0, 1106}, + dictWord{7, 0, 1546}, + dictWord{11, 0, 299}, + dictWord{142, 0, 407}, + dictWord{134, 0, 1192}, + dictWord{132, 0, 177}, + dictWord{5, 0, 411}, + dictWord{135, 0, 653}, + dictWord{7, 0, 439}, + dictWord{10, 0, 727}, + dictWord{11, 0, 260}, + dictWord{139, 0, 684}, + dictWord{138, 10, 145}, + dictWord{147, 10, 83}, + dictWord{5, 0, 208}, + dictWord{7, 0, 753}, + dictWord{135, 0, 1528}, + dictWord{137, 11, 617}, + dictWord{ + 135, + 10, + 1922, + }, + dictWord{135, 11, 825}, + dictWord{11, 0, 422}, + dictWord{13, 0, 389}, + dictWord{4, 10, 124}, + dictWord{10, 10, 457}, + dictWord{11, 10, 121}, + dictWord{ + 11, + 10, + 169, + }, + dictWord{11, 10, 870}, + dictWord{12, 10, 214}, + dictWord{14, 10, 187}, + dictWord{143, 10, 77}, + dictWord{11, 0, 615}, + dictWord{15, 0, 58}, + dictWord{ + 11, + 11, + 615, + }, + dictWord{143, 11, 58}, + dictWord{9, 0, 618}, + dictWord{138, 0, 482}, + dictWord{6, 0, 1952}, + dictWord{6, 0, 1970}, + dictWord{142, 0, 505}, + dictWord{ + 7, + 10, + 1193, + }, + dictWord{135, 11, 1838}, + dictWord{133, 0, 242}, + dictWord{135, 10, 1333}, + dictWord{6, 10, 107}, + dictWord{7, 10, 638}, + dictWord{ + 7, + 10, + 1632, + }, + dictWord{137, 10, 396}, + dictWord{133, 0, 953}, + dictWord{5, 10, 370}, + dictWord{134, 10, 1756}, + dictWord{5, 11, 28}, + dictWord{6, 11, 204}, + dictWord{ + 10, + 11, + 320, + }, + dictWord{10, 11, 583}, + dictWord{13, 11, 502}, + dictWord{14, 11, 72}, + dictWord{14, 11, 274}, + dictWord{14, 11, 312}, + dictWord{14, 11, 344}, + dictWord{15, 11, 159}, + dictWord{16, 11, 62}, + dictWord{16, 11, 69}, + dictWord{17, 11, 30}, + dictWord{18, 11, 42}, + dictWord{18, 11, 53}, + dictWord{18, 11, 84}, + dictWord{18, 11, 140}, + dictWord{19, 11, 68}, + dictWord{19, 11, 85}, + dictWord{20, 11, 5}, + dictWord{20, 11, 45}, + dictWord{20, 11, 101}, + dictWord{22, 11, 7}, + dictWord{ + 150, + 11, + 20, + }, + dictWord{4, 11, 558}, + dictWord{6, 11, 390}, + dictWord{7, 11, 162}, + dictWord{7, 11, 689}, + dictWord{9, 11, 360}, + dictWord{138, 11, 653}, + dictWord{ + 11, + 0, + 802, + }, + dictWord{141, 0, 67}, + dictWord{133, 10, 204}, + dictWord{133, 0, 290}, + dictWord{5, 10, 970}, + dictWord{134, 10, 1706}, + dictWord{132, 0, 380}, + dictWord{5, 0, 52}, + dictWord{7, 0, 277}, + dictWord{9, 0, 368}, + dictWord{139, 0, 791}, + dictWord{5, 11, 856}, + dictWord{6, 11, 1672}, + dictWord{6, 11, 1757}, + dictWord{ + 6, + 11, + 1781, + }, + dictWord{7, 11, 1150}, + dictWord{7, 11, 1425}, + dictWord{7, 11, 1453}, + dictWord{140, 11, 513}, + dictWord{5, 11, 92}, + dictWord{7, 10, 3}, + dictWord{ + 10, + 11, + 736, + }, + dictWord{140, 11, 102}, + dictWord{4, 0, 112}, + dictWord{5, 0, 653}, + dictWord{5, 10, 483}, + dictWord{5, 10, 685}, + dictWord{6, 10, 489}, + dictWord{ + 7, + 10, + 1204, + }, + dictWord{136, 10, 394}, + dictWord{132, 10, 921}, + dictWord{6, 0, 1028}, + dictWord{133, 10, 1007}, + dictWord{5, 11, 590}, + dictWord{9, 11, 213}, + dictWord{145, 11, 91}, + dictWord{135, 10, 1696}, + dictWord{10, 0, 138}, + dictWord{139, 0, 476}, + dictWord{5, 0, 725}, + dictWord{5, 0, 727}, + dictWord{135, 0, 1811}, + dictWord{4, 0, 979}, + dictWord{6, 0, 1821}, + dictWord{6, 0, 1838}, + dictWord{8, 0, 876}, + dictWord{8, 0, 883}, + dictWord{8, 0, 889}, + dictWord{8, 0, 893}, + dictWord{ + 8, + 0, + 895, + }, + dictWord{10, 0, 934}, + dictWord{12, 0, 720}, + dictWord{14, 0, 459}, + dictWord{148, 0, 123}, + dictWord{135, 11, 551}, + dictWord{4, 0, 38}, + dictWord{6, 0, 435}, + dictWord{7, 0, 307}, + dictWord{7, 0, 999}, + dictWord{7, 0, 1481}, + dictWord{7, 0, 1732}, + dictWord{7, 0, 1738}, + dictWord{8, 0, 371}, + dictWord{9, 0, 414}, + dictWord{ + 11, + 0, + 316, + }, + dictWord{12, 0, 52}, + dictWord{13, 0, 420}, + dictWord{147, 0, 100}, + dictWord{135, 0, 1296}, + dictWord{132, 10, 712}, + dictWord{134, 10, 1629}, + dictWord{133, 0, 723}, + dictWord{134, 0, 651}, + dictWord{136, 11, 191}, + dictWord{9, 11, 791}, + dictWord{10, 11, 93}, + dictWord{11, 11, 301}, + dictWord{16, 11, 13}, + dictWord{17, 11, 23}, + dictWord{18, 11, 135}, + dictWord{19, 11, 12}, + dictWord{20, 11, 1}, + dictWord{20, 11, 12}, + dictWord{148, 11, 14}, + dictWord{136, 11, 503}, + dictWord{6, 11, 466}, + dictWord{135, 11, 671}, + dictWord{6, 0, 1200}, + dictWord{134, 0, 1330}, + dictWord{135, 0, 1255}, + dictWord{134, 0, 986}, + dictWord{ + 5, + 0, + 109, + }, + dictWord{6, 0, 1784}, + dictWord{7, 0, 1895}, + dictWord{12, 0, 296}, + dictWord{140, 0, 302}, + dictWord{135, 11, 983}, + dictWord{133, 10, 485}, + dictWord{ + 134, + 0, + 660, + }, + dictWord{134, 0, 800}, + dictWord{5, 0, 216}, + dictWord{5, 0, 294}, + dictWord{6, 0, 591}, + dictWord{7, 0, 1879}, + dictWord{9, 0, 141}, + dictWord{9, 0, 270}, + dictWord{9, 0, 679}, + dictWord{10, 0, 159}, + dictWord{11, 0, 197}, + dictWord{11, 0, 438}, + dictWord{12, 0, 538}, + dictWord{12, 0, 559}, + dictWord{14, 0, 144}, + dictWord{ + 14, + 0, + 167, + }, + dictWord{15, 0, 67}, + dictWord{4, 10, 285}, + dictWord{5, 10, 317}, + dictWord{6, 10, 301}, + dictWord{7, 10, 7}, + dictWord{8, 10, 153}, + dictWord{ + 10, + 10, + 766, + }, + dictWord{11, 10, 468}, + dictWord{12, 10, 467}, + dictWord{141, 10, 143}, + dictWord{136, 0, 945}, + dictWord{134, 0, 1090}, + dictWord{137, 0, 81}, + dictWord{12, 11, 468}, + dictWord{19, 11, 96}, + dictWord{148, 11, 24}, + dictWord{134, 0, 391}, + dictWord{138, 11, 241}, + dictWord{7, 0, 322}, + dictWord{136, 0, 249}, + dictWord{134, 0, 1412}, + dictWord{135, 11, 795}, + dictWord{5, 0, 632}, + dictWord{138, 0, 526}, + dictWord{136, 10, 819}, + dictWord{6, 0, 144}, + dictWord{7, 0, 948}, + dictWord{7, 0, 1042}, + dictWord{8, 0, 235}, + dictWord{8, 0, 461}, + dictWord{9, 0, 453}, + dictWord{9, 0, 796}, + dictWord{10, 0, 354}, + dictWord{17, 0, 77}, + dictWord{ + 135, + 11, + 954, + }, + dictWord{139, 10, 917}, + dictWord{6, 0, 940}, + dictWord{134, 0, 1228}, + dictWord{4, 0, 362}, + dictWord{7, 0, 52}, + dictWord{135, 0, 303}, + dictWord{ + 6, + 11, + 549, + }, + dictWord{8, 11, 34}, + dictWord{8, 11, 283}, + dictWord{9, 11, 165}, + dictWord{138, 11, 475}, + dictWord{7, 11, 370}, + dictWord{7, 11, 1007}, + dictWord{ + 7, + 11, + 1177, + }, + dictWord{135, 11, 1565}, + dictWord{5, 11, 652}, + dictWord{5, 11, 701}, + dictWord{135, 11, 449}, + dictWord{5, 0, 196}, + dictWord{6, 0, 486}, + dictWord{ + 7, + 0, + 212, + }, + dictWord{8, 0, 309}, + dictWord{136, 0, 346}, + dictWord{6, 10, 1719}, + dictWord{6, 10, 1735}, + dictWord{7, 10, 2016}, + dictWord{7, 10, 2020}, + dictWord{ + 8, + 10, + 837, + }, + dictWord{137, 10, 852}, + dictWord{6, 11, 159}, + dictWord{6, 11, 364}, + dictWord{7, 11, 516}, + dictWord{7, 11, 1439}, + dictWord{137, 11, 518}, + dictWord{135, 0, 1912}, + dictWord{135, 0, 1290}, + dictWord{132, 0, 686}, + dictWord{141, 11, 151}, + dictWord{138, 0, 625}, + dictWord{136, 0, 706}, + dictWord{ + 138, + 10, + 568, + }, + dictWord{139, 0, 412}, + dictWord{4, 0, 30}, + dictWord{133, 0, 43}, + dictWord{8, 10, 67}, + dictWord{138, 10, 419}, + dictWord{7, 0, 967}, + dictWord{ + 141, + 0, + 11, + }, + dictWord{12, 0, 758}, + dictWord{14, 0, 441}, + dictWord{142, 0, 462}, + dictWord{10, 10, 657}, + dictWord{14, 10, 297}, + dictWord{142, 10, 361}, + dictWord{ + 139, + 10, + 729, + }, + dictWord{4, 0, 220}, + dictWord{135, 0, 1535}, + dictWord{7, 11, 501}, + dictWord{9, 11, 111}, + dictWord{10, 11, 141}, + dictWord{11, 11, 332}, + dictWord{ + 13, + 11, + 43, + }, + dictWord{13, 11, 429}, + dictWord{14, 11, 130}, + dictWord{14, 11, 415}, + dictWord{145, 11, 102}, + dictWord{4, 0, 950}, + dictWord{6, 0, 1859}, + dictWord{ + 7, + 0, + 11, + }, + dictWord{8, 0, 873}, + dictWord{12, 0, 710}, + dictWord{12, 0, 718}, + dictWord{12, 0, 748}, + dictWord{12, 0, 765}, + dictWord{148, 0, 124}, + dictWord{ + 5, + 11, + 149, + }, + dictWord{5, 11, 935}, + dictWord{136, 11, 233}, + dictWord{142, 11, 291}, + dictWord{134, 0, 1579}, + dictWord{7, 0, 890}, + dictWord{8, 10, 51}, + dictWord{ + 9, + 10, + 868, + }, + dictWord{10, 10, 833}, + dictWord{12, 10, 481}, + dictWord{12, 10, 570}, + dictWord{148, 10, 106}, + dictWord{141, 0, 2}, + dictWord{132, 10, 445}, + dictWord{136, 11, 801}, + dictWord{135, 0, 1774}, + dictWord{7, 0, 1725}, + dictWord{138, 0, 393}, + dictWord{5, 0, 263}, + dictWord{134, 0, 414}, + dictWord{ + 132, + 11, + 322, + }, + dictWord{133, 10, 239}, + dictWord{7, 0, 456}, + dictWord{7, 10, 1990}, + dictWord{8, 10, 130}, + dictWord{139, 10, 720}, + dictWord{137, 0, 818}, + dictWord{ + 5, + 10, + 123, + }, + dictWord{6, 10, 530}, + dictWord{7, 10, 348}, + dictWord{135, 10, 1419}, + dictWord{135, 10, 2024}, + dictWord{6, 0, 178}, + dictWord{6, 0, 1750}, + dictWord{8, 0, 251}, + dictWord{9, 0, 690}, + dictWord{10, 0, 155}, + dictWord{10, 0, 196}, + dictWord{10, 0, 373}, + dictWord{11, 0, 698}, + dictWord{13, 0, 155}, + dictWord{ + 148, + 0, + 93, + }, + dictWord{5, 0, 97}, + dictWord{137, 0, 393}, + dictWord{134, 0, 674}, + dictWord{11, 0, 223}, + dictWord{140, 0, 168}, + dictWord{132, 10, 210}, + dictWord{ + 139, + 11, + 464, + }, + dictWord{6, 0, 1639}, + dictWord{146, 0, 159}, + dictWord{139, 11, 2}, + dictWord{7, 0, 934}, + dictWord{8, 0, 647}, + dictWord{17, 0, 97}, + dictWord{19, 0, 59}, + dictWord{150, 0, 2}, + dictWord{132, 0, 191}, + dictWord{5, 0, 165}, + dictWord{9, 0, 346}, + dictWord{10, 0, 655}, + dictWord{11, 0, 885}, + dictWord{4, 10, 430}, + dictWord{135, 11, 357}, + dictWord{133, 0, 877}, + dictWord{5, 10, 213}, + dictWord{133, 11, 406}, + dictWord{8, 0, 128}, + dictWord{139, 0, 179}, + dictWord{6, 11, 69}, + dictWord{135, 11, 117}, + dictWord{135, 0, 1297}, + dictWord{11, 11, 43}, + dictWord{13, 11, 72}, + dictWord{141, 11, 142}, + dictWord{135, 11, 1830}, + dictWord{ + 142, + 0, + 164, + }, + dictWord{5, 0, 57}, + dictWord{6, 0, 101}, + dictWord{6, 0, 586}, + dictWord{6, 0, 1663}, + dictWord{7, 0, 132}, + dictWord{7, 0, 1154}, + dictWord{7, 0, 1415}, + dictWord{7, 0, 1507}, + dictWord{12, 0, 493}, + dictWord{15, 0, 105}, + dictWord{151, 0, 15}, + dictWord{5, 0, 459}, + dictWord{7, 0, 1073}, + dictWord{8, 0, 241}, + dictWord{ + 136, + 0, + 334, + }, + dictWord{133, 11, 826}, + dictWord{133, 10, 108}, + dictWord{5, 10, 219}, + dictWord{10, 11, 132}, + dictWord{11, 11, 191}, + dictWord{11, 11, 358}, + dictWord{139, 11, 460}, + dictWord{6, 0, 324}, + dictWord{6, 0, 520}, + dictWord{7, 0, 338}, + dictWord{7, 0, 1729}, + dictWord{8, 0, 228}, + dictWord{139, 0, 750}, + dictWord{ + 21, + 0, + 30, + }, + dictWord{22, 0, 53}, + dictWord{4, 10, 193}, + dictWord{5, 10, 916}, + dictWord{7, 10, 364}, + dictWord{10, 10, 398}, + dictWord{10, 10, 726}, + dictWord{ + 11, + 10, + 317, + }, + dictWord{11, 10, 626}, + dictWord{12, 10, 142}, + dictWord{12, 10, 288}, + dictWord{12, 10, 678}, + dictWord{13, 10, 313}, + dictWord{15, 10, 113}, + dictWord{146, 10, 114}, + dictWord{6, 11, 110}, + dictWord{135, 11, 1681}, + dictWord{135, 0, 910}, + dictWord{6, 10, 241}, + dictWord{7, 10, 907}, + dictWord{8, 10, 832}, + dictWord{9, 10, 342}, + dictWord{10, 10, 729}, + dictWord{11, 10, 284}, + dictWord{11, 10, 445}, + dictWord{11, 10, 651}, + dictWord{11, 10, 863}, + dictWord{ + 13, + 10, + 398, + }, + dictWord{146, 10, 99}, + dictWord{7, 0, 705}, + dictWord{9, 0, 734}, + dictWord{5, 11, 1000}, + dictWord{7, 11, 733}, + dictWord{137, 11, 583}, + dictWord{4, 0, 73}, + dictWord{6, 0, 612}, + dictWord{7, 0, 927}, + dictWord{7, 0, 1822}, + dictWord{8, 0, 217}, + dictWord{9, 0, 765}, + dictWord{9, 0, 766}, + dictWord{10, 0, 408}, + dictWord{ + 11, + 0, + 51, + }, + dictWord{11, 0, 793}, + dictWord{12, 0, 266}, + dictWord{15, 0, 158}, + dictWord{20, 0, 89}, + dictWord{150, 0, 32}, + dictWord{7, 0, 1330}, + dictWord{4, 11, 297}, + dictWord{6, 11, 529}, + dictWord{7, 11, 152}, + dictWord{7, 11, 713}, + dictWord{7, 11, 1845}, + dictWord{8, 11, 710}, + dictWord{8, 11, 717}, + dictWord{140, 11, 639}, + dictWord{5, 0, 389}, + dictWord{136, 0, 636}, + dictWord{134, 0, 1409}, + dictWord{4, 10, 562}, + dictWord{9, 10, 254}, + dictWord{139, 10, 879}, + dictWord{134, 0, 893}, + dictWord{132, 10, 786}, + dictWord{4, 11, 520}, + dictWord{135, 11, 575}, + dictWord{136, 0, 21}, + dictWord{140, 0, 721}, + dictWord{136, 0, 959}, + dictWord{ + 7, + 11, + 1428, + }, + dictWord{7, 11, 1640}, + dictWord{9, 11, 169}, + dictWord{9, 11, 182}, + dictWord{9, 11, 367}, + dictWord{9, 11, 478}, + dictWord{9, 11, 506}, + dictWord{ + 9, + 11, + 551, + }, + dictWord{9, 11, 648}, + dictWord{9, 11, 651}, + dictWord{9, 11, 697}, + dictWord{9, 11, 705}, + dictWord{9, 11, 725}, + dictWord{9, 11, 787}, + dictWord{9, 11, 794}, + dictWord{10, 11, 198}, + dictWord{10, 11, 214}, + dictWord{10, 11, 267}, + dictWord{10, 11, 275}, + dictWord{10, 11, 456}, + dictWord{10, 11, 551}, + dictWord{ + 10, + 11, + 561, + }, + dictWord{10, 11, 613}, + dictWord{10, 11, 627}, + dictWord{10, 11, 668}, + dictWord{10, 11, 675}, + dictWord{10, 11, 691}, + dictWord{10, 11, 695}, + dictWord{10, 11, 707}, + dictWord{10, 11, 715}, + dictWord{11, 11, 183}, + dictWord{11, 11, 201}, + dictWord{11, 11, 244}, + dictWord{11, 11, 262}, + dictWord{ + 11, + 11, + 352, + }, + dictWord{11, 11, 439}, + dictWord{11, 11, 493}, + dictWord{11, 11, 572}, + dictWord{11, 11, 591}, + dictWord{11, 11, 608}, + dictWord{11, 11, 611}, + dictWord{ + 11, + 11, + 646, + }, + dictWord{11, 11, 674}, + dictWord{11, 11, 711}, + dictWord{11, 11, 751}, + dictWord{11, 11, 761}, + dictWord{11, 11, 776}, + dictWord{11, 11, 785}, + dictWord{11, 11, 850}, + dictWord{11, 11, 853}, + dictWord{11, 11, 862}, + dictWord{11, 11, 865}, + dictWord{11, 11, 868}, + dictWord{11, 11, 898}, + dictWord{ + 11, + 11, + 902, + }, + dictWord{11, 11, 903}, + dictWord{11, 11, 910}, + dictWord{11, 11, 932}, + dictWord{11, 11, 942}, + dictWord{11, 11, 957}, + dictWord{11, 11, 967}, + dictWord{ + 11, + 11, + 972, + }, + dictWord{12, 11, 148}, + dictWord{12, 11, 195}, + dictWord{12, 11, 220}, + dictWord{12, 11, 237}, + dictWord{12, 11, 318}, + dictWord{12, 11, 339}, + dictWord{12, 11, 393}, + dictWord{12, 11, 445}, + dictWord{12, 11, 450}, + dictWord{12, 11, 474}, + dictWord{12, 11, 509}, + dictWord{12, 11, 533}, + dictWord{ + 12, + 11, + 591, + }, + dictWord{12, 11, 594}, + dictWord{12, 11, 597}, + dictWord{12, 11, 621}, + dictWord{12, 11, 633}, + dictWord{12, 11, 642}, + dictWord{13, 11, 59}, + dictWord{ + 13, + 11, + 60, + }, + dictWord{13, 11, 145}, + dictWord{13, 11, 239}, + dictWord{13, 11, 250}, + dictWord{13, 11, 273}, + dictWord{13, 11, 329}, + dictWord{13, 11, 344}, + dictWord{13, 11, 365}, + dictWord{13, 11, 372}, + dictWord{13, 11, 387}, + dictWord{13, 11, 403}, + dictWord{13, 11, 414}, + dictWord{13, 11, 456}, + dictWord{ + 13, + 11, + 478, + }, + dictWord{13, 11, 483}, + dictWord{13, 11, 489}, + dictWord{14, 11, 55}, + dictWord{14, 11, 57}, + dictWord{14, 11, 81}, + dictWord{14, 11, 90}, + dictWord{ + 14, + 11, + 148, + }, + dictWord{14, 11, 239}, + dictWord{14, 11, 266}, + dictWord{14, 11, 321}, + dictWord{14, 11, 326}, + dictWord{14, 11, 327}, + dictWord{14, 11, 330}, + dictWord{ + 14, + 11, + 347, + }, + dictWord{14, 11, 355}, + dictWord{14, 11, 401}, + dictWord{14, 11, 411}, + dictWord{14, 11, 414}, + dictWord{14, 11, 416}, + dictWord{14, 11, 420}, + dictWord{15, 11, 61}, + dictWord{15, 11, 74}, + dictWord{15, 11, 87}, + dictWord{15, 11, 88}, + dictWord{15, 11, 94}, + dictWord{15, 11, 96}, + dictWord{15, 11, 116}, + dictWord{15, 11, 149}, + dictWord{15, 11, 154}, + dictWord{16, 11, 50}, + dictWord{16, 11, 63}, + dictWord{16, 11, 73}, + dictWord{17, 11, 2}, + dictWord{17, 11, 66}, + dictWord{ + 17, + 11, + 92, + }, + dictWord{17, 11, 103}, + dictWord{17, 11, 112}, + dictWord{18, 11, 50}, + dictWord{18, 11, 54}, + dictWord{18, 11, 82}, + dictWord{18, 11, 86}, + dictWord{ + 18, + 11, + 90, + }, + dictWord{18, 11, 111}, + dictWord{18, 11, 115}, + dictWord{18, 11, 156}, + dictWord{19, 11, 40}, + dictWord{19, 11, 79}, + dictWord{20, 11, 78}, + dictWord{ + 149, + 11, + 22, + }, + dictWord{137, 11, 170}, + dictWord{134, 0, 1433}, + dictWord{135, 11, 1307}, + dictWord{139, 11, 411}, + dictWord{5, 0, 189}, + dictWord{7, 0, 442}, + dictWord{7, 0, 443}, + dictWord{8, 0, 281}, + dictWord{12, 0, 174}, + dictWord{141, 0, 261}, + dictWord{6, 10, 216}, + dictWord{7, 10, 901}, + dictWord{7, 10, 1343}, + dictWord{136, 10, 493}, + dictWord{5, 11, 397}, + dictWord{6, 11, 154}, + dictWord{7, 10, 341}, + dictWord{7, 11, 676}, + dictWord{8, 11, 443}, + dictWord{8, 11, 609}, + dictWord{ + 9, + 11, + 24, + }, + dictWord{9, 11, 325}, + dictWord{10, 11, 35}, + dictWord{11, 10, 219}, + dictWord{11, 11, 535}, + dictWord{11, 11, 672}, + dictWord{11, 11, 1018}, + dictWord{12, 11, 637}, + dictWord{144, 11, 30}, + dictWord{6, 0, 2}, + dictWord{7, 0, 191}, + dictWord{7, 0, 446}, + dictWord{7, 0, 1262}, + dictWord{7, 0, 1737}, + dictWord{8, 0, 22}, + dictWord{8, 0, 270}, + dictWord{8, 0, 612}, + dictWord{9, 0, 4}, + dictWord{9, 0, 312}, + dictWord{9, 0, 436}, + dictWord{9, 0, 626}, + dictWord{10, 0, 216}, + dictWord{10, 0, 311}, + dictWord{10, 0, 521}, + dictWord{10, 0, 623}, + dictWord{11, 0, 72}, + dictWord{11, 0, 330}, + dictWord{11, 0, 455}, + dictWord{12, 0, 321}, + dictWord{12, 0, 504}, + dictWord{12, 0, 530}, + dictWord{12, 0, 543}, + dictWord{13, 0, 17}, + dictWord{13, 0, 156}, + dictWord{13, 0, 334}, + dictWord{14, 0, 131}, + dictWord{17, 0, 60}, + dictWord{ + 148, + 0, + 64, + }, + dictWord{7, 0, 354}, + dictWord{10, 0, 410}, + dictWord{139, 0, 815}, + dictWord{139, 10, 130}, + dictWord{7, 10, 1734}, + dictWord{137, 11, 631}, + dictWord{ + 12, + 0, + 425, + }, + dictWord{15, 0, 112}, + dictWord{10, 10, 115}, + dictWord{11, 10, 420}, + dictWord{13, 10, 404}, + dictWord{14, 10, 346}, + dictWord{143, 10, 54}, + dictWord{ + 6, + 0, + 60, + }, + dictWord{6, 0, 166}, + dictWord{7, 0, 374}, + dictWord{7, 0, 670}, + dictWord{7, 0, 1327}, + dictWord{8, 0, 411}, + dictWord{8, 0, 435}, + dictWord{9, 0, 653}, + dictWord{ + 9, + 0, + 740, + }, + dictWord{10, 0, 385}, + dictWord{11, 0, 222}, + dictWord{11, 0, 324}, + dictWord{11, 0, 829}, + dictWord{140, 0, 611}, + dictWord{7, 0, 1611}, + dictWord{ + 13, + 0, + 14, + }, + dictWord{15, 0, 44}, + dictWord{19, 0, 13}, + dictWord{148, 0, 76}, + dictWord{133, 11, 981}, + dictWord{4, 11, 56}, + dictWord{7, 11, 1791}, + dictWord{8, 11, 607}, + dictWord{8, 11, 651}, + dictWord{11, 11, 465}, + dictWord{11, 11, 835}, + dictWord{12, 11, 337}, + dictWord{141, 11, 480}, + dictWord{6, 0, 1478}, + dictWord{ + 5, + 10, + 1011, + }, + dictWord{136, 10, 701}, + dictWord{139, 0, 596}, + dictWord{5, 0, 206}, + dictWord{134, 0, 398}, + dictWord{4, 10, 54}, + dictWord{5, 10, 666}, + dictWord{ + 7, + 10, + 1039, + }, + dictWord{7, 10, 1130}, + dictWord{9, 10, 195}, + dictWord{138, 10, 302}, + dictWord{7, 0, 50}, + dictWord{9, 11, 158}, + dictWord{138, 11, 411}, + dictWord{ + 135, + 11, + 1120, + }, + dictWord{6, 0, 517}, + dictWord{7, 0, 1159}, + dictWord{10, 0, 621}, + dictWord{11, 0, 192}, + dictWord{134, 10, 1669}, + dictWord{4, 0, 592}, + dictWord{ + 6, + 0, + 600, + }, + dictWord{135, 0, 1653}, + dictWord{10, 0, 223}, + dictWord{139, 0, 645}, + dictWord{136, 11, 139}, + dictWord{7, 0, 64}, + dictWord{136, 0, 245}, + dictWord{ + 142, + 0, + 278, + }, + dictWord{6, 11, 622}, + dictWord{135, 11, 1030}, + dictWord{136, 0, 604}, + dictWord{134, 0, 1502}, + dictWord{138, 0, 265}, + dictWord{ + 141, + 11, + 168, + }, + dictWord{7, 0, 1763}, + dictWord{140, 0, 310}, + dictWord{7, 10, 798}, + dictWord{139, 11, 719}, + dictWord{7, 11, 160}, + dictWord{10, 11, 624}, + dictWord{ + 142, + 11, + 279, + }, + dictWord{132, 11, 363}, + dictWord{7, 10, 122}, + dictWord{9, 10, 259}, + dictWord{10, 10, 84}, + dictWord{11, 10, 470}, + dictWord{12, 10, 541}, + dictWord{141, 10, 379}, + dictWord{5, 0, 129}, + dictWord{6, 0, 61}, + dictWord{135, 0, 947}, + dictWord{134, 0, 1356}, + dictWord{135, 11, 1191}, + dictWord{13, 0, 505}, + dictWord{141, 0, 506}, + dictWord{11, 0, 1000}, + dictWord{5, 10, 82}, + dictWord{5, 10, 131}, + dictWord{7, 10, 1755}, + dictWord{8, 10, 31}, + dictWord{9, 10, 168}, + dictWord{9, 10, 764}, + dictWord{139, 10, 869}, + dictWord{134, 0, 966}, + dictWord{134, 10, 605}, + dictWord{134, 11, 292}, + dictWord{5, 11, 177}, + dictWord{ + 6, + 11, + 616, + }, + dictWord{7, 11, 827}, + dictWord{9, 11, 525}, + dictWord{138, 11, 656}, + dictWord{135, 11, 1486}, + dictWord{138, 11, 31}, + dictWord{5, 10, 278}, + dictWord{137, 10, 68}, + dictWord{4, 10, 163}, + dictWord{5, 10, 201}, + dictWord{5, 10, 307}, + dictWord{5, 10, 310}, + dictWord{6, 10, 335}, + dictWord{7, 10, 284}, + dictWord{136, 10, 165}, + dictWord{6, 0, 839}, + dictWord{135, 10, 1660}, + dictWord{136, 10, 781}, + dictWord{6, 10, 33}, + dictWord{135, 10, 1244}, + dictWord{ + 133, + 0, + 637, + }, + dictWord{4, 11, 161}, + dictWord{133, 11, 631}, + dictWord{137, 0, 590}, + dictWord{7, 10, 1953}, + dictWord{136, 10, 720}, + dictWord{5, 0, 280}, + dictWord{ + 7, + 0, + 1226, + }, + dictWord{138, 10, 203}, + dictWord{134, 0, 1386}, + dictWord{5, 0, 281}, + dictWord{6, 0, 1026}, + dictWord{6, 10, 326}, + dictWord{7, 10, 677}, + dictWord{ + 137, + 10, + 425, + }, + dictWord{7, 11, 1557}, + dictWord{135, 11, 1684}, + dictWord{135, 0, 1064}, + dictWord{9, 11, 469}, + dictWord{9, 11, 709}, + dictWord{12, 11, 512}, + dictWord{14, 11, 65}, + dictWord{145, 11, 12}, + dictWord{134, 0, 917}, + dictWord{10, 11, 229}, + dictWord{11, 11, 73}, + dictWord{11, 11, 376}, + dictWord{ + 139, + 11, + 433, + }, + dictWord{7, 0, 555}, + dictWord{9, 0, 192}, + dictWord{13, 0, 30}, + dictWord{13, 0, 49}, + dictWord{15, 0, 150}, + dictWord{16, 0, 76}, + dictWord{20, 0, 52}, + dictWord{ + 7, + 10, + 1316, + }, + dictWord{7, 10, 1412}, + dictWord{7, 10, 1839}, + dictWord{9, 10, 589}, + dictWord{11, 10, 241}, + dictWord{11, 10, 676}, + dictWord{11, 10, 811}, + dictWord{11, 10, 891}, + dictWord{12, 10, 140}, + dictWord{12, 10, 346}, + dictWord{12, 10, 479}, + dictWord{13, 10, 381}, + dictWord{14, 10, 188}, + dictWord{ + 146, + 10, + 30, + }, + dictWord{149, 0, 15}, + dictWord{6, 0, 1882}, + dictWord{6, 0, 1883}, + dictWord{6, 0, 1897}, + dictWord{9, 0, 945}, + dictWord{9, 0, 1014}, + dictWord{9, 0, 1020}, + dictWord{12, 0, 823}, + dictWord{12, 0, 842}, + dictWord{12, 0, 866}, + dictWord{12, 0, 934}, + dictWord{15, 0, 242}, + dictWord{146, 0, 208}, + dictWord{6, 0, 965}, + dictWord{134, 0, 1499}, + dictWord{7, 0, 33}, + dictWord{7, 0, 120}, + dictWord{8, 0, 489}, + dictWord{9, 0, 319}, + dictWord{10, 0, 820}, + dictWord{11, 0, 1004}, + dictWord{ + 12, + 0, + 379, + }, + dictWord{12, 0, 679}, + dictWord{13, 0, 117}, + dictWord{13, 0, 412}, + dictWord{14, 0, 25}, + dictWord{15, 0, 52}, + dictWord{15, 0, 161}, + dictWord{16, 0, 47}, + dictWord{149, 0, 2}, + dictWord{6, 11, 558}, + dictWord{7, 11, 651}, + dictWord{8, 11, 421}, + dictWord{9, 11, 0}, + dictWord{138, 11, 34}, + dictWord{4, 0, 937}, + dictWord{ + 5, + 0, + 801, + }, + dictWord{7, 0, 473}, + dictWord{5, 10, 358}, + dictWord{7, 10, 1184}, + dictWord{10, 10, 662}, + dictWord{13, 10, 212}, + dictWord{13, 10, 304}, + dictWord{ + 13, + 10, + 333, + }, + dictWord{145, 10, 98}, + dictWord{132, 0, 877}, + dictWord{6, 0, 693}, + dictWord{134, 0, 824}, + dictWord{132, 0, 365}, + dictWord{7, 11, 1832}, + dictWord{ + 138, + 11, + 374, + }, + dictWord{5, 0, 7}, + dictWord{139, 0, 774}, + dictWord{4, 0, 734}, + dictWord{5, 0, 662}, + dictWord{134, 0, 430}, + dictWord{4, 0, 746}, + dictWord{ + 135, + 0, + 1090, + }, + dictWord{5, 0, 360}, + dictWord{8, 0, 237}, + dictWord{10, 0, 231}, + dictWord{147, 0, 124}, + dictWord{138, 11, 348}, + dictWord{6, 11, 6}, + dictWord{7, 11, 81}, + dictWord{7, 11, 771}, + dictWord{7, 11, 1731}, + dictWord{9, 11, 405}, + dictWord{138, 11, 421}, + dictWord{6, 0, 740}, + dictWord{137, 0, 822}, + dictWord{ + 133, + 10, + 946, + }, + dictWord{7, 0, 1485}, + dictWord{136, 0, 929}, + dictWord{7, 10, 411}, + dictWord{8, 10, 631}, + dictWord{9, 10, 323}, + dictWord{10, 10, 355}, + dictWord{ + 11, + 10, + 491, + }, + dictWord{12, 10, 143}, + dictWord{12, 10, 402}, + dictWord{13, 10, 73}, + dictWord{14, 10, 408}, + dictWord{15, 10, 107}, + dictWord{146, 10, 71}, + dictWord{ + 135, + 10, + 590, + }, + dictWord{5, 11, 881}, + dictWord{133, 11, 885}, + dictWord{150, 11, 25}, + dictWord{4, 0, 852}, + dictWord{5, 11, 142}, + dictWord{134, 11, 546}, + dictWord{7, 10, 1467}, + dictWord{8, 10, 328}, + dictWord{10, 10, 544}, + dictWord{11, 10, 955}, + dictWord{13, 10, 320}, + dictWord{145, 10, 83}, + dictWord{9, 0, 17}, + dictWord{10, 0, 291}, + dictWord{11, 10, 511}, + dictWord{13, 10, 394}, + dictWord{14, 10, 298}, + dictWord{14, 10, 318}, + dictWord{146, 10, 103}, + dictWord{5, 11, 466}, + dictWord{11, 11, 571}, + dictWord{12, 11, 198}, + dictWord{13, 11, 283}, + dictWord{14, 11, 186}, + dictWord{15, 11, 21}, + dictWord{143, 11, 103}, + dictWord{ + 134, + 0, + 1001, + }, + dictWord{4, 11, 185}, + dictWord{5, 11, 257}, + dictWord{5, 11, 839}, + dictWord{5, 11, 936}, + dictWord{7, 11, 171}, + dictWord{9, 11, 399}, + dictWord{ + 10, + 11, + 258, + }, + dictWord{10, 11, 395}, + dictWord{10, 11, 734}, + dictWord{11, 11, 1014}, + dictWord{12, 11, 23}, + dictWord{13, 11, 350}, + dictWord{14, 11, 150}, + dictWord{147, 11, 6}, + dictWord{143, 0, 35}, + dictWord{132, 0, 831}, + dictWord{5, 10, 835}, + dictWord{134, 10, 483}, + dictWord{4, 0, 277}, + dictWord{5, 0, 608}, + dictWord{ + 6, + 0, + 493, + }, + dictWord{7, 0, 457}, + dictWord{12, 0, 384}, + dictWord{7, 11, 404}, + dictWord{7, 11, 1377}, + dictWord{7, 11, 1430}, + dictWord{7, 11, 2017}, + dictWord{ + 8, + 11, + 149, + }, + dictWord{8, 11, 239}, + dictWord{8, 11, 512}, + dictWord{8, 11, 793}, + dictWord{8, 11, 818}, + dictWord{9, 11, 474}, + dictWord{9, 11, 595}, + dictWord{ + 10, + 11, + 122, + }, + dictWord{10, 11, 565}, + dictWord{10, 11, 649}, + dictWord{10, 11, 783}, + dictWord{11, 11, 239}, + dictWord{11, 11, 295}, + dictWord{11, 11, 447}, + dictWord{ + 11, + 11, + 528, + }, + dictWord{11, 11, 639}, + dictWord{11, 11, 800}, + dictWord{11, 11, 936}, + dictWord{12, 11, 25}, + dictWord{12, 11, 73}, + dictWord{12, 11, 77}, + dictWord{12, 11, 157}, + dictWord{12, 11, 316}, + dictWord{12, 11, 390}, + dictWord{12, 11, 391}, + dictWord{12, 11, 394}, + dictWord{12, 11, 395}, + dictWord{ + 12, + 11, + 478, + }, + dictWord{12, 11, 503}, + dictWord{12, 11, 592}, + dictWord{12, 11, 680}, + dictWord{13, 11, 50}, + dictWord{13, 11, 53}, + dictWord{13, 11, 132}, + dictWord{ + 13, + 11, + 198, + }, + dictWord{13, 11, 275}, + dictWord{13, 11, 322}, + dictWord{13, 11, 415}, + dictWord{14, 11, 71}, + dictWord{14, 11, 257}, + dictWord{14, 11, 395}, + dictWord{15, 11, 71}, + dictWord{15, 11, 136}, + dictWord{17, 11, 123}, + dictWord{18, 11, 93}, + dictWord{147, 11, 58}, + dictWord{134, 0, 1351}, + dictWord{7, 0, 27}, + dictWord{135, 0, 316}, + dictWord{136, 11, 712}, + dictWord{136, 0, 984}, + dictWord{133, 0, 552}, + dictWord{137, 0, 264}, + dictWord{132, 0, 401}, + dictWord{6, 0, 710}, + dictWord{6, 0, 1111}, + dictWord{134, 0, 1343}, + dictWord{134, 0, 1211}, + dictWord{9, 0, 543}, + dictWord{10, 0, 524}, + dictWord{11, 0, 108}, + dictWord{11, 0, 653}, + dictWord{12, 0, 524}, + dictWord{13, 0, 123}, + dictWord{14, 0, 252}, + dictWord{16, 0, 18}, + dictWord{19, 0, 38}, + dictWord{20, 0, 26}, + dictWord{20, 0, 65}, + dictWord{ + 21, + 0, + 3, + }, + dictWord{151, 0, 11}, + dictWord{4, 0, 205}, + dictWord{5, 0, 623}, + dictWord{7, 0, 104}, + dictWord{8, 0, 519}, + dictWord{137, 0, 716}, + dictWord{132, 10, 677}, + dictWord{4, 11, 377}, + dictWord{152, 11, 13}, + dictWord{135, 11, 1673}, + dictWord{7, 0, 579}, + dictWord{9, 0, 41}, + dictWord{9, 0, 244}, + dictWord{9, 0, 669}, + dictWord{ + 10, + 0, + 5, + }, + dictWord{11, 0, 861}, + dictWord{11, 0, 951}, + dictWord{139, 0, 980}, + dictWord{132, 0, 717}, + dictWord{136, 0, 1011}, + dictWord{132, 0, 805}, + dictWord{ + 4, + 11, + 180, + }, + dictWord{135, 11, 1906}, + dictWord{132, 10, 777}, + dictWord{132, 10, 331}, + dictWord{132, 0, 489}, + dictWord{6, 0, 1024}, + dictWord{4, 11, 491}, + dictWord{133, 10, 747}, + dictWord{135, 11, 1182}, + dictWord{4, 11, 171}, + dictWord{138, 11, 234}, + dictWord{4, 11, 586}, + dictWord{7, 11, 1186}, + dictWord{ + 138, + 11, + 631, + }, + dictWord{135, 0, 892}, + dictWord{135, 11, 336}, + dictWord{9, 11, 931}, + dictWord{10, 11, 334}, + dictWord{148, 11, 71}, + dictWord{137, 0, 473}, + dictWord{6, 0, 864}, + dictWord{12, 0, 659}, + dictWord{139, 11, 926}, + dictWord{7, 0, 819}, + dictWord{9, 0, 26}, + dictWord{9, 0, 392}, + dictWord{10, 0, 152}, + dictWord{ + 10, + 0, + 226, + }, + dictWord{11, 0, 19}, + dictWord{12, 0, 276}, + dictWord{12, 0, 426}, + dictWord{12, 0, 589}, + dictWord{13, 0, 460}, + dictWord{15, 0, 97}, + dictWord{19, 0, 48}, + dictWord{148, 0, 104}, + dictWord{135, 0, 51}, + dictWord{133, 10, 326}, + dictWord{4, 10, 691}, + dictWord{146, 10, 16}, + dictWord{9, 0, 130}, + dictWord{11, 0, 765}, + dictWord{10, 10, 680}, + dictWord{10, 10, 793}, + dictWord{141, 10, 357}, + dictWord{133, 11, 765}, + dictWord{8, 0, 229}, + dictWord{6, 10, 32}, + dictWord{7, 10, 385}, + dictWord{7, 10, 757}, + dictWord{7, 10, 1916}, + dictWord{8, 10, 94}, + dictWord{8, 10, 711}, + dictWord{9, 10, 541}, + dictWord{10, 10, 162}, + dictWord{10, 10, 795}, + dictWord{11, 10, 989}, + dictWord{11, 10, 1010}, + dictWord{12, 10, 14}, + dictWord{142, 10, 308}, + dictWord{7, 11, 474}, + dictWord{137, 11, 578}, + dictWord{ + 132, + 0, + 674, + }, + dictWord{132, 0, 770}, + dictWord{5, 0, 79}, + dictWord{7, 0, 1027}, + dictWord{7, 0, 1477}, + dictWord{139, 0, 52}, + dictWord{133, 11, 424}, + dictWord{ + 134, + 0, + 1666, + }, + dictWord{6, 0, 409}, + dictWord{6, 10, 349}, + dictWord{6, 10, 1682}, + dictWord{7, 10, 1252}, + dictWord{8, 10, 112}, + dictWord{8, 11, 714}, + dictWord{ + 9, + 10, + 435, + }, + dictWord{9, 10, 668}, + dictWord{10, 10, 290}, + dictWord{10, 10, 319}, + dictWord{10, 10, 815}, + dictWord{11, 10, 180}, + dictWord{11, 10, 837}, + dictWord{ + 12, + 10, + 240, + }, + dictWord{13, 10, 152}, + dictWord{13, 10, 219}, + dictWord{142, 10, 158}, + dictWord{5, 0, 789}, + dictWord{134, 0, 195}, + dictWord{4, 0, 251}, + dictWord{ + 4, + 0, + 688, + }, + dictWord{7, 0, 513}, + dictWord{135, 0, 1284}, + dictWord{132, 10, 581}, + dictWord{9, 11, 420}, + dictWord{10, 11, 269}, + dictWord{10, 11, 285}, + dictWord{10, 11, 576}, + dictWord{11, 11, 397}, + dictWord{13, 11, 175}, + dictWord{145, 11, 90}, + dictWord{6, 10, 126}, + dictWord{7, 10, 573}, + dictWord{8, 10, 397}, + dictWord{142, 10, 44}, + dictWord{132, 11, 429}, + dictWord{133, 0, 889}, + dictWord{4, 0, 160}, + dictWord{5, 0, 330}, + dictWord{7, 0, 1434}, + dictWord{136, 0, 174}, + dictWord{7, 11, 18}, + dictWord{7, 11, 699}, + dictWord{7, 11, 1966}, + dictWord{8, 11, 752}, + dictWord{9, 11, 273}, + dictWord{9, 11, 412}, + dictWord{9, 11, 703}, + dictWord{ + 10, + 11, + 71, + }, + dictWord{10, 11, 427}, + dictWord{10, 11, 508}, + dictWord{146, 11, 97}, + dictWord{6, 0, 872}, + dictWord{134, 0, 899}, + dictWord{133, 10, 926}, + dictWord{134, 0, 1126}, + dictWord{134, 0, 918}, + dictWord{4, 11, 53}, + dictWord{5, 11, 186}, + dictWord{135, 11, 752}, + dictWord{7, 0, 268}, + dictWord{136, 0, 569}, + dictWord{134, 0, 1224}, + dictWord{6, 0, 1361}, + dictWord{7, 10, 1232}, + dictWord{137, 10, 531}, + dictWord{8, 11, 575}, + dictWord{10, 11, 289}, + dictWord{ + 139, + 11, + 319, + }, + dictWord{133, 10, 670}, + dictWord{132, 11, 675}, + dictWord{133, 0, 374}, + dictWord{135, 10, 1957}, + dictWord{133, 0, 731}, + dictWord{11, 0, 190}, + dictWord{15, 0, 49}, + dictWord{11, 11, 190}, + dictWord{143, 11, 49}, + dictWord{4, 0, 626}, + dictWord{5, 0, 506}, + dictWord{5, 0, 642}, + dictWord{6, 0, 425}, + dictWord{ + 10, + 0, + 202, + }, + dictWord{139, 0, 141}, + dictWord{137, 0, 444}, + dictWord{7, 10, 242}, + dictWord{135, 10, 1942}, + dictWord{6, 11, 209}, + dictWord{8, 11, 468}, + dictWord{ + 9, + 11, + 210, + }, + dictWord{11, 11, 36}, + dictWord{12, 11, 28}, + dictWord{12, 11, 630}, + dictWord{13, 11, 21}, + dictWord{13, 11, 349}, + dictWord{14, 11, 7}, + dictWord{ + 145, + 11, + 13, + }, + dictWord{4, 11, 342}, + dictWord{135, 11, 1179}, + dictWord{5, 10, 834}, + dictWord{7, 10, 1202}, + dictWord{8, 10, 14}, + dictWord{9, 10, 481}, + dictWord{ + 137, + 10, + 880, + }, + dictWord{4, 11, 928}, + dictWord{133, 11, 910}, + dictWord{4, 11, 318}, + dictWord{4, 11, 496}, + dictWord{7, 11, 856}, + dictWord{139, 11, 654}, + dictWord{136, 0, 835}, + dictWord{7, 0, 1526}, + dictWord{138, 10, 465}, + dictWord{151, 0, 17}, + dictWord{135, 0, 477}, + dictWord{4, 10, 357}, + dictWord{6, 10, 172}, + dictWord{7, 10, 143}, + dictWord{137, 10, 413}, + dictWord{6, 0, 1374}, + dictWord{138, 0, 994}, + dictWord{18, 0, 76}, + dictWord{132, 10, 590}, + dictWord{7, 0, 287}, + dictWord{8, 0, 355}, + dictWord{9, 0, 293}, + dictWord{137, 0, 743}, + dictWord{134, 0, 1389}, + dictWord{7, 11, 915}, + dictWord{8, 11, 247}, + dictWord{147, 11, 0}, + dictWord{ + 4, + 11, + 202, + }, + dictWord{5, 11, 382}, + dictWord{6, 11, 454}, + dictWord{7, 11, 936}, + dictWord{7, 11, 1803}, + dictWord{8, 11, 758}, + dictWord{9, 11, 375}, + dictWord{ + 9, + 11, + 895, + }, + dictWord{10, 11, 743}, + dictWord{10, 11, 792}, + dictWord{11, 11, 978}, + dictWord{11, 11, 1012}, + dictWord{142, 11, 109}, + dictWord{5, 0, 384}, + dictWord{8, 0, 455}, + dictWord{140, 0, 48}, + dictWord{132, 11, 390}, + dictWord{5, 10, 169}, + dictWord{7, 10, 333}, + dictWord{136, 10, 45}, + dictWord{5, 0, 264}, + dictWord{134, 0, 184}, + dictWord{138, 11, 791}, + dictWord{133, 11, 717}, + dictWord{132, 10, 198}, + dictWord{6, 11, 445}, + dictWord{7, 11, 332}, + dictWord{ + 137, + 11, + 909, + }, + dictWord{136, 0, 1001}, + dictWord{4, 10, 24}, + dictWord{5, 10, 140}, + dictWord{5, 10, 185}, + dictWord{7, 10, 1500}, + dictWord{11, 10, 565}, + dictWord{ + 139, + 10, + 838, + }, + dictWord{134, 11, 578}, + dictWord{5, 0, 633}, + dictWord{6, 0, 28}, + dictWord{135, 0, 1323}, + dictWord{132, 0, 851}, + dictWord{136, 11, 267}, + dictWord{ + 7, + 0, + 359, + }, + dictWord{8, 0, 243}, + dictWord{140, 0, 175}, + dictWord{4, 10, 334}, + dictWord{133, 10, 593}, + dictWord{141, 11, 87}, + dictWord{136, 11, 766}, + dictWord{10, 0, 287}, + dictWord{12, 0, 138}, + dictWord{10, 11, 287}, + dictWord{140, 11, 138}, + dictWord{4, 0, 105}, + dictWord{132, 0, 740}, + dictWord{140, 10, 116}, + dictWord{134, 0, 857}, + dictWord{135, 11, 1841}, + dictWord{6, 0, 1402}, + dictWord{137, 0, 819}, + dictWord{132, 11, 584}, + dictWord{132, 10, 709}, + dictWord{ + 133, + 10, + 897, + }, + dictWord{5, 0, 224}, + dictWord{13, 0, 174}, + dictWord{146, 0, 52}, + dictWord{135, 10, 1840}, + dictWord{4, 10, 608}, + dictWord{133, 10, 497}, + dictWord{139, 11, 60}, + dictWord{4, 0, 758}, + dictWord{135, 0, 1649}, + dictWord{4, 11, 226}, + dictWord{4, 11, 326}, + dictWord{135, 11, 1770}, + dictWord{5, 11, 426}, + dictWord{8, 11, 30}, + dictWord{9, 11, 2}, + dictWord{11, 11, 549}, + dictWord{147, 11, 122}, + dictWord{135, 10, 2039}, + dictWord{6, 10, 540}, + dictWord{ + 136, + 10, + 136, + }, + dictWord{4, 0, 573}, + dictWord{8, 0, 655}, + dictWord{4, 10, 897}, + dictWord{133, 10, 786}, + dictWord{7, 0, 351}, + dictWord{139, 0, 128}, + dictWord{ + 133, + 10, + 999, + }, + dictWord{4, 10, 299}, + dictWord{135, 10, 1004}, + dictWord{133, 0, 918}, + dictWord{132, 11, 345}, + dictWord{4, 11, 385}, + dictWord{7, 11, 265}, + dictWord{135, 11, 587}, + dictWord{133, 10, 456}, + dictWord{136, 10, 180}, + dictWord{6, 0, 687}, + dictWord{134, 0, 1537}, + dictWord{4, 11, 347}, + dictWord{ + 5, + 11, + 423, + }, + dictWord{5, 11, 996}, + dictWord{135, 11, 1329}, + dictWord{132, 10, 755}, + dictWord{7, 11, 1259}, + dictWord{9, 11, 125}, + dictWord{11, 11, 65}, + dictWord{140, 11, 285}, + dictWord{5, 11, 136}, + dictWord{6, 11, 136}, + dictWord{136, 11, 644}, + dictWord{134, 0, 1525}, + dictWord{4, 0, 1009}, + dictWord{ + 135, + 0, + 1139, + }, + dictWord{139, 10, 338}, + dictWord{132, 0, 340}, + dictWord{135, 10, 1464}, + dictWord{8, 0, 847}, + dictWord{10, 0, 861}, + dictWord{10, 0, 876}, + dictWord{ + 10, + 0, + 889, + }, + dictWord{10, 0, 922}, + dictWord{10, 0, 929}, + dictWord{10, 0, 933}, + dictWord{12, 0, 784}, + dictWord{140, 0, 791}, + dictWord{139, 0, 176}, + dictWord{ + 9, + 11, + 134, + }, + dictWord{10, 11, 2}, + dictWord{10, 11, 27}, + dictWord{10, 11, 333}, + dictWord{11, 11, 722}, + dictWord{143, 11, 1}, + dictWord{4, 11, 433}, + dictWord{ + 133, + 11, + 719, + }, + dictWord{5, 0, 985}, + dictWord{7, 0, 509}, + dictWord{7, 0, 529}, + dictWord{145, 0, 96}, + dictWord{132, 0, 615}, + dictWord{4, 10, 890}, + dictWord{ + 5, + 10, + 805, + }, + dictWord{5, 10, 819}, + dictWord{5, 10, 961}, + dictWord{6, 10, 396}, + dictWord{6, 10, 1631}, + dictWord{6, 10, 1678}, + dictWord{7, 10, 1967}, + dictWord{ + 7, + 10, + 2041, + }, + dictWord{9, 10, 630}, + dictWord{11, 10, 8}, + dictWord{11, 10, 1019}, + dictWord{12, 10, 176}, + dictWord{13, 10, 225}, + dictWord{14, 10, 292}, + dictWord{ + 149, + 10, + 24, + }, + dictWord{135, 0, 1919}, + dictWord{134, 0, 1131}, + dictWord{144, 11, 21}, + dictWord{144, 11, 51}, + dictWord{135, 10, 1815}, + dictWord{4, 0, 247}, + dictWord{7, 10, 1505}, + dictWord{10, 10, 190}, + dictWord{10, 10, 634}, + dictWord{11, 10, 792}, + dictWord{12, 10, 358}, + dictWord{140, 10, 447}, + dictWord{ + 5, + 10, + 0, + }, + dictWord{6, 10, 536}, + dictWord{7, 10, 604}, + dictWord{13, 10, 445}, + dictWord{145, 10, 126}, + dictWord{4, 0, 184}, + dictWord{5, 0, 390}, + dictWord{6, 0, 337}, + dictWord{7, 0, 23}, + dictWord{7, 0, 494}, + dictWord{7, 0, 618}, + dictWord{7, 0, 1456}, + dictWord{8, 0, 27}, + dictWord{8, 0, 599}, + dictWord{10, 0, 153}, + dictWord{ + 139, + 0, + 710, + }, + dictWord{6, 10, 232}, + dictWord{6, 10, 412}, + dictWord{7, 10, 1074}, + dictWord{8, 10, 9}, + dictWord{8, 10, 157}, + dictWord{8, 10, 786}, + dictWord{9, 10, 196}, + dictWord{9, 10, 352}, + dictWord{9, 10, 457}, + dictWord{10, 10, 337}, + dictWord{11, 10, 232}, + dictWord{11, 10, 877}, + dictWord{12, 10, 480}, + dictWord{ + 140, + 10, + 546, + }, + dictWord{13, 0, 38}, + dictWord{135, 10, 958}, + dictWord{4, 10, 382}, + dictWord{136, 10, 579}, + dictWord{4, 10, 212}, + dictWord{135, 10, 1206}, + dictWord{ + 4, + 11, + 555, + }, + dictWord{8, 11, 536}, + dictWord{138, 11, 288}, + dictWord{11, 11, 139}, + dictWord{139, 11, 171}, + dictWord{9, 11, 370}, + dictWord{138, 11, 90}, + dictWord{132, 0, 1015}, + dictWord{134, 0, 1088}, + dictWord{5, 10, 655}, + dictWord{135, 11, 977}, + dictWord{134, 0, 1585}, + dictWord{17, 10, 67}, + dictWord{ + 147, + 10, + 74, + }, + dictWord{10, 0, 227}, + dictWord{11, 0, 497}, + dictWord{11, 0, 709}, + dictWord{140, 0, 415}, + dictWord{6, 0, 360}, + dictWord{7, 0, 1664}, + dictWord{ + 136, + 0, + 478, + }, + dictWord{7, 0, 95}, + dictWord{6, 10, 231}, + dictWord{136, 10, 423}, + dictWord{140, 11, 65}, + dictWord{4, 11, 257}, + dictWord{135, 11, 2031}, + dictWord{ + 135, + 11, + 1768, + }, + dictWord{133, 10, 300}, + dictWord{139, 11, 211}, + dictWord{136, 0, 699}, + dictWord{6, 10, 237}, + dictWord{7, 10, 611}, + dictWord{8, 10, 100}, + dictWord{9, 10, 416}, + dictWord{11, 10, 335}, + dictWord{12, 10, 173}, + dictWord{146, 10, 101}, + dictWord{14, 0, 26}, + dictWord{146, 0, 150}, + dictWord{6, 0, 581}, + dictWord{135, 0, 1119}, + dictWord{135, 10, 1208}, + dictWord{132, 0, 739}, + dictWord{6, 11, 83}, + dictWord{6, 11, 1733}, + dictWord{135, 11, 1389}, + dictWord{ + 137, + 0, + 869, + }, + dictWord{4, 0, 67}, + dictWord{5, 0, 422}, + dictWord{7, 0, 1037}, + dictWord{7, 0, 1289}, + dictWord{7, 0, 1555}, + dictWord{9, 0, 741}, + dictWord{145, 0, 108}, + dictWord{133, 10, 199}, + dictWord{12, 10, 427}, + dictWord{146, 10, 38}, + dictWord{136, 0, 464}, + dictWord{142, 0, 42}, + dictWord{10, 0, 96}, + dictWord{8, 11, 501}, + dictWord{137, 11, 696}, + dictWord{134, 11, 592}, + dictWord{4, 0, 512}, + dictWord{4, 0, 966}, + dictWord{5, 0, 342}, + dictWord{6, 0, 1855}, + dictWord{8, 0, 869}, + dictWord{8, 0, 875}, + dictWord{8, 0, 901}, + dictWord{144, 0, 26}, + dictWord{8, 0, 203}, + dictWord{11, 0, 823}, + dictWord{11, 0, 846}, + dictWord{12, 0, 482}, + dictWord{ + 13, + 0, + 277, + }, + dictWord{13, 0, 302}, + dictWord{13, 0, 464}, + dictWord{14, 0, 205}, + dictWord{142, 0, 221}, + dictWord{4, 0, 449}, + dictWord{133, 0, 718}, + dictWord{ + 7, + 11, + 1718, + }, + dictWord{9, 11, 95}, + dictWord{9, 11, 274}, + dictWord{10, 11, 279}, + dictWord{10, 11, 317}, + dictWord{10, 11, 420}, + dictWord{11, 11, 303}, + dictWord{ + 11, + 11, + 808, + }, + dictWord{12, 11, 134}, + dictWord{12, 11, 367}, + dictWord{13, 11, 149}, + dictWord{13, 11, 347}, + dictWord{14, 11, 349}, + dictWord{14, 11, 406}, + dictWord{18, 11, 22}, + dictWord{18, 11, 89}, + dictWord{18, 11, 122}, + dictWord{147, 11, 47}, + dictWord{133, 11, 26}, + dictWord{4, 0, 355}, + dictWord{6, 0, 311}, + dictWord{ + 9, + 0, + 256, + }, + dictWord{138, 0, 404}, + dictWord{132, 11, 550}, + dictWord{10, 0, 758}, + dictWord{6, 10, 312}, + dictWord{6, 10, 1715}, + dictWord{10, 10, 584}, + dictWord{11, 10, 546}, + dictWord{11, 10, 692}, + dictWord{12, 10, 259}, + dictWord{12, 10, 295}, + dictWord{13, 10, 46}, + dictWord{141, 10, 154}, + dictWord{ + 136, + 11, + 822, + }, + dictWord{5, 0, 827}, + dictWord{4, 11, 902}, + dictWord{5, 11, 809}, + dictWord{6, 11, 122}, + dictWord{135, 11, 896}, + dictWord{5, 0, 64}, + dictWord{140, 0, 581}, + dictWord{4, 0, 442}, + dictWord{6, 0, 739}, + dictWord{7, 0, 1047}, + dictWord{7, 0, 1352}, + dictWord{7, 0, 1643}, + dictWord{7, 11, 1911}, + dictWord{9, 11, 449}, + dictWord{10, 11, 192}, + dictWord{138, 11, 740}, + dictWord{135, 11, 262}, + dictWord{132, 10, 588}, + dictWord{133, 11, 620}, + dictWord{5, 0, 977}, + dictWord{ + 6, + 0, + 288, + }, + dictWord{7, 0, 528}, + dictWord{4, 11, 34}, + dictWord{5, 11, 574}, + dictWord{7, 11, 279}, + dictWord{7, 11, 1624}, + dictWord{136, 11, 601}, + dictWord{ + 6, + 0, + 1375, + }, + dictWord{4, 10, 231}, + dictWord{5, 10, 61}, + dictWord{6, 10, 104}, + dictWord{7, 10, 729}, + dictWord{7, 10, 964}, + dictWord{7, 10, 1658}, + dictWord{ + 140, + 10, + 414, + }, + dictWord{6, 10, 263}, + dictWord{138, 10, 757}, + dictWord{132, 10, 320}, + dictWord{4, 0, 254}, + dictWord{7, 0, 1309}, + dictWord{5, 11, 332}, + dictWord{ + 135, + 11, + 1309, + }, + dictWord{6, 11, 261}, + dictWord{8, 11, 182}, + dictWord{139, 11, 943}, + dictWord{132, 10, 225}, + dictWord{6, 0, 12}, + dictWord{135, 0, 1219}, + dictWord{4, 0, 275}, + dictWord{12, 0, 376}, + dictWord{6, 11, 1721}, + dictWord{141, 11, 490}, + dictWord{4, 11, 933}, + dictWord{133, 11, 880}, + dictWord{6, 0, 951}, + dictWord{6, 0, 1109}, + dictWord{6, 0, 1181}, + dictWord{7, 0, 154}, + dictWord{4, 10, 405}, + dictWord{7, 10, 817}, + dictWord{14, 10, 58}, + dictWord{17, 10, 37}, + dictWord{ + 146, + 10, + 124, + }, + dictWord{6, 0, 1520}, + dictWord{133, 10, 974}, + dictWord{134, 0, 1753}, + dictWord{6, 0, 369}, + dictWord{6, 0, 502}, + dictWord{7, 0, 1036}, + dictWord{ + 8, + 0, + 348, + }, + dictWord{9, 0, 452}, + dictWord{10, 0, 26}, + dictWord{11, 0, 224}, + dictWord{11, 0, 387}, + dictWord{11, 0, 772}, + dictWord{12, 0, 95}, + dictWord{12, 0, 629}, + dictWord{13, 0, 195}, + dictWord{13, 0, 207}, + dictWord{13, 0, 241}, + dictWord{14, 0, 260}, + dictWord{14, 0, 270}, + dictWord{143, 0, 140}, + dictWord{132, 0, 269}, + dictWord{5, 0, 480}, + dictWord{7, 0, 532}, + dictWord{7, 0, 1197}, + dictWord{7, 0, 1358}, + dictWord{8, 0, 291}, + dictWord{11, 0, 349}, + dictWord{142, 0, 396}, + dictWord{ + 5, + 10, + 235, + }, + dictWord{7, 10, 1239}, + dictWord{11, 10, 131}, + dictWord{140, 10, 370}, + dictWord{7, 10, 956}, + dictWord{7, 10, 1157}, + dictWord{7, 10, 1506}, + dictWord{ + 7, + 10, + 1606, + }, + dictWord{7, 10, 1615}, + dictWord{7, 10, 1619}, + dictWord{7, 10, 1736}, + dictWord{7, 10, 1775}, + dictWord{8, 10, 590}, + dictWord{9, 10, 324}, + dictWord{9, 10, 736}, + dictWord{9, 10, 774}, + dictWord{9, 10, 776}, + dictWord{9, 10, 784}, + dictWord{10, 10, 567}, + dictWord{10, 10, 708}, + dictWord{11, 10, 518}, + dictWord{11, 10, 613}, + dictWord{11, 10, 695}, + dictWord{11, 10, 716}, + dictWord{11, 10, 739}, + dictWord{11, 10, 770}, + dictWord{11, 10, 771}, + dictWord{ + 11, + 10, + 848, + }, + dictWord{11, 10, 857}, + dictWord{11, 10, 931}, + dictWord{11, 10, 947}, + dictWord{12, 10, 326}, + dictWord{12, 10, 387}, + dictWord{12, 10, 484}, + dictWord{ + 12, + 10, + 528, + }, + dictWord{12, 10, 552}, + dictWord{12, 10, 613}, + dictWord{13, 10, 189}, + dictWord{13, 10, 256}, + dictWord{13, 10, 340}, + dictWord{13, 10, 432}, + dictWord{13, 10, 436}, + dictWord{13, 10, 440}, + dictWord{13, 10, 454}, + dictWord{14, 10, 174}, + dictWord{14, 10, 220}, + dictWord{14, 10, 284}, + dictWord{ + 14, + 10, + 390, + }, + dictWord{145, 10, 121}, + dictWord{8, 11, 598}, + dictWord{9, 11, 664}, + dictWord{138, 11, 441}, + dictWord{9, 10, 137}, + dictWord{138, 10, 221}, + dictWord{133, 11, 812}, + dictWord{148, 0, 15}, + dictWord{134, 0, 1341}, + dictWord{6, 0, 1017}, + dictWord{4, 11, 137}, + dictWord{7, 11, 1178}, + dictWord{ + 135, + 11, + 1520, + }, + dictWord{7, 10, 390}, + dictWord{138, 10, 140}, + dictWord{7, 11, 1260}, + dictWord{135, 11, 1790}, + dictWord{137, 11, 191}, + dictWord{ + 135, + 10, + 1144, + }, + dictWord{6, 0, 1810}, + dictWord{7, 0, 657}, + dictWord{8, 0, 886}, + dictWord{10, 0, 857}, + dictWord{14, 0, 440}, + dictWord{144, 0, 96}, + dictWord{8, 0, 533}, + dictWord{6, 11, 1661}, + dictWord{7, 11, 1975}, + dictWord{7, 11, 2009}, + dictWord{135, 11, 2011}, + dictWord{6, 0, 1453}, + dictWord{134, 10, 464}, + dictWord{ + 132, + 11, + 715, + }, + dictWord{5, 10, 407}, + dictWord{11, 10, 204}, + dictWord{11, 10, 243}, + dictWord{11, 10, 489}, + dictWord{12, 10, 293}, + dictWord{19, 10, 37}, + dictWord{20, 10, 73}, + dictWord{150, 10, 38}, + dictWord{133, 11, 703}, + dictWord{4, 0, 211}, + dictWord{7, 0, 1483}, + dictWord{5, 10, 325}, + dictWord{8, 10, 5}, + dictWord{ + 8, + 10, + 227, + }, + dictWord{9, 10, 105}, + dictWord{10, 10, 585}, + dictWord{140, 10, 614}, + dictWord{4, 0, 332}, + dictWord{5, 0, 335}, + dictWord{6, 0, 238}, + dictWord{ + 7, + 0, + 269, + }, + dictWord{7, 0, 811}, + dictWord{7, 0, 1797}, + dictWord{8, 0, 836}, + dictWord{9, 0, 507}, + dictWord{141, 0, 242}, + dictWord{5, 11, 89}, + dictWord{7, 11, 1915}, + dictWord{9, 11, 185}, + dictWord{9, 11, 235}, + dictWord{9, 11, 496}, + dictWord{10, 11, 64}, + dictWord{10, 11, 270}, + dictWord{10, 11, 403}, + dictWord{10, 11, 469}, + dictWord{10, 11, 529}, + dictWord{10, 11, 590}, + dictWord{11, 11, 140}, + dictWord{11, 11, 860}, + dictWord{13, 11, 1}, + dictWord{13, 11, 422}, + dictWord{14, 11, 341}, + dictWord{14, 11, 364}, + dictWord{17, 11, 93}, + dictWord{18, 11, 113}, + dictWord{19, 11, 97}, + dictWord{147, 11, 113}, + dictWord{133, 11, 695}, + dictWord{ + 16, + 0, + 19, + }, + dictWord{5, 11, 6}, + dictWord{6, 11, 183}, + dictWord{6, 10, 621}, + dictWord{7, 11, 680}, + dictWord{7, 11, 978}, + dictWord{7, 11, 1013}, + dictWord{7, 11, 1055}, + dictWord{12, 11, 230}, + dictWord{13, 11, 172}, + dictWord{13, 10, 504}, + dictWord{146, 11, 29}, + dictWord{136, 0, 156}, + dictWord{133, 0, 1009}, + dictWord{ + 6, + 11, + 29, + }, + dictWord{139, 11, 63}, + dictWord{134, 0, 820}, + dictWord{134, 10, 218}, + dictWord{7, 10, 454}, + dictWord{7, 10, 782}, + dictWord{8, 10, 768}, + dictWord{ + 140, + 10, + 686, + }, + dictWord{5, 0, 228}, + dictWord{6, 0, 203}, + dictWord{7, 0, 156}, + dictWord{8, 0, 347}, + dictWord{9, 0, 265}, + dictWord{18, 0, 39}, + dictWord{20, 0, 54}, + dictWord{21, 0, 31}, + dictWord{22, 0, 3}, + dictWord{23, 0, 0}, + dictWord{15, 11, 8}, + dictWord{18, 11, 39}, + dictWord{20, 11, 54}, + dictWord{21, 11, 31}, + dictWord{22, 11, 3}, + dictWord{151, 11, 0}, + dictWord{7, 0, 1131}, + dictWord{135, 0, 1468}, + dictWord{144, 10, 0}, + dictWord{134, 0, 1276}, + dictWord{10, 10, 676}, + dictWord{ + 140, + 10, + 462, + }, + dictWord{132, 11, 311}, + dictWord{134, 11, 1740}, + dictWord{7, 11, 170}, + dictWord{8, 11, 90}, + dictWord{8, 11, 177}, + dictWord{8, 11, 415}, + dictWord{ + 11, + 11, + 714, + }, + dictWord{142, 11, 281}, + dictWord{134, 10, 164}, + dictWord{6, 0, 1792}, + dictWord{138, 0, 849}, + dictWord{150, 10, 50}, + dictWord{5, 0, 291}, + dictWord{5, 0, 318}, + dictWord{7, 0, 765}, + dictWord{9, 0, 389}, + dictWord{12, 0, 548}, + dictWord{8, 11, 522}, + dictWord{142, 11, 328}, + dictWord{11, 11, 91}, + dictWord{ + 13, + 11, + 129, + }, + dictWord{15, 11, 101}, + dictWord{145, 11, 125}, + dictWord{4, 11, 494}, + dictWord{6, 11, 74}, + dictWord{7, 11, 44}, + dictWord{7, 11, 407}, + dictWord{ + 8, + 11, + 551, + }, + dictWord{12, 11, 17}, + dictWord{15, 11, 5}, + dictWord{148, 11, 11}, + dictWord{4, 11, 276}, + dictWord{133, 11, 296}, + dictWord{6, 10, 343}, + dictWord{ + 7, + 10, + 195, + }, + dictWord{7, 11, 1777}, + dictWord{9, 10, 226}, + dictWord{10, 10, 197}, + dictWord{10, 10, 575}, + dictWord{11, 10, 502}, + dictWord{139, 10, 899}, + dictWord{ + 10, + 0, + 525, + }, + dictWord{139, 0, 82}, + dictWord{14, 0, 453}, + dictWord{4, 11, 7}, + dictWord{5, 11, 90}, + dictWord{5, 11, 158}, + dictWord{6, 11, 542}, + dictWord{7, 11, 221}, + dictWord{7, 11, 1574}, + dictWord{9, 11, 490}, + dictWord{10, 11, 540}, + dictWord{11, 11, 443}, + dictWord{139, 11, 757}, + dictWord{135, 0, 666}, + dictWord{ + 22, + 10, + 29, + }, + dictWord{150, 11, 29}, + dictWord{4, 0, 422}, + dictWord{147, 10, 8}, + dictWord{5, 0, 355}, + dictWord{145, 0, 0}, + dictWord{6, 0, 1873}, + dictWord{9, 0, 918}, + dictWord{7, 11, 588}, + dictWord{9, 11, 175}, + dictWord{138, 11, 530}, + dictWord{143, 11, 31}, + dictWord{11, 0, 165}, + dictWord{7, 10, 1125}, + dictWord{9, 10, 143}, + dictWord{14, 10, 405}, + dictWord{150, 10, 21}, + dictWord{9, 0, 260}, + dictWord{137, 0, 905}, + dictWord{5, 11, 872}, + dictWord{6, 11, 57}, + dictWord{6, 11, 479}, + dictWord{ + 6, + 11, + 562, + }, + dictWord{7, 11, 471}, + dictWord{7, 11, 1060}, + dictWord{9, 11, 447}, + dictWord{9, 11, 454}, + dictWord{141, 11, 6}, + dictWord{138, 11, 704}, + dictWord{133, 0, 865}, + dictWord{5, 0, 914}, + dictWord{134, 0, 1625}, + dictWord{133, 0, 234}, + dictWord{7, 0, 1383}, + dictWord{5, 11, 31}, + dictWord{6, 11, 614}, + dictWord{145, 11, 61}, + dictWord{7, 11, 1200}, + dictWord{138, 11, 460}, + dictWord{6, 11, 424}, + dictWord{135, 11, 1866}, + dictWord{136, 0, 306}, + dictWord{ + 5, + 10, + 959, + }, + dictWord{12, 11, 30}, + dictWord{13, 11, 148}, + dictWord{14, 11, 87}, + dictWord{14, 11, 182}, + dictWord{16, 11, 42}, + dictWord{18, 11, 92}, + dictWord{ + 148, + 11, + 70, + }, + dictWord{6, 0, 1919}, + dictWord{6, 0, 1921}, + dictWord{9, 0, 923}, + dictWord{9, 0, 930}, + dictWord{9, 0, 941}, + dictWord{9, 0, 949}, + dictWord{9, 0, 987}, + dictWord{ + 9, + 0, + 988, + }, + dictWord{9, 0, 992}, + dictWord{12, 0, 802}, + dictWord{12, 0, 815}, + dictWord{12, 0, 856}, + dictWord{12, 0, 885}, + dictWord{12, 0, 893}, + dictWord{ + 12, + 0, + 898, + }, + dictWord{12, 0, 919}, + dictWord{12, 0, 920}, + dictWord{12, 0, 941}, + dictWord{12, 0, 947}, + dictWord{15, 0, 183}, + dictWord{15, 0, 185}, + dictWord{15, 0, 189}, + dictWord{15, 0, 197}, + dictWord{15, 0, 202}, + dictWord{15, 0, 233}, + dictWord{18, 0, 218}, + dictWord{18, 0, 219}, + dictWord{18, 0, 233}, + dictWord{143, 11, 156}, + dictWord{135, 10, 1759}, + dictWord{136, 10, 173}, + dictWord{13, 0, 163}, + dictWord{13, 0, 180}, + dictWord{18, 0, 78}, + dictWord{20, 0, 35}, + dictWord{5, 11, 13}, + dictWord{134, 11, 142}, + dictWord{134, 10, 266}, + dictWord{6, 11, 97}, + dictWord{7, 11, 116}, + dictWord{8, 11, 322}, + dictWord{8, 11, 755}, + dictWord{9, 11, 548}, + dictWord{10, 11, 714}, + dictWord{11, 11, 884}, + dictWord{141, 11, 324}, + dictWord{135, 0, 1312}, + dictWord{9, 0, 814}, + dictWord{137, 11, 676}, + dictWord{ + 133, + 0, + 707, + }, + dictWord{135, 0, 1493}, + dictWord{6, 0, 421}, + dictWord{7, 0, 61}, + dictWord{7, 0, 1540}, + dictWord{10, 0, 11}, + dictWord{138, 0, 501}, + dictWord{12, 0, 733}, + dictWord{12, 0, 766}, + dictWord{7, 11, 866}, + dictWord{135, 11, 1163}, + dictWord{137, 0, 341}, + dictWord{142, 0, 98}, + dictWord{145, 11, 115}, + dictWord{ + 135, + 11, + 1111, + }, + dictWord{136, 10, 300}, + dictWord{136, 0, 1014}, + dictWord{8, 11, 1}, + dictWord{9, 11, 112}, + dictWord{138, 11, 326}, + dictWord{132, 11, 730}, + dictWord{5, 11, 488}, + dictWord{6, 11, 527}, + dictWord{7, 11, 489}, + dictWord{7, 11, 1636}, + dictWord{8, 11, 121}, + dictWord{8, 11, 144}, + dictWord{8, 11, 359}, + dictWord{ + 9, + 11, + 193, + }, + dictWord{9, 11, 241}, + dictWord{9, 11, 336}, + dictWord{9, 11, 882}, + dictWord{11, 11, 266}, + dictWord{11, 11, 372}, + dictWord{11, 11, 944}, + dictWord{ + 12, + 11, + 401, + }, + dictWord{140, 11, 641}, + dictWord{6, 0, 971}, + dictWord{134, 0, 1121}, + dictWord{6, 0, 102}, + dictWord{7, 0, 72}, + dictWord{15, 0, 142}, + dictWord{ + 147, + 0, + 67, + }, + dictWord{151, 0, 30}, + dictWord{135, 0, 823}, + dictWord{134, 0, 1045}, + dictWord{5, 10, 427}, + dictWord{5, 10, 734}, + dictWord{7, 10, 478}, + dictWord{ + 136, + 10, + 52, + }, + dictWord{7, 0, 1930}, + dictWord{11, 10, 217}, + dictWord{142, 10, 165}, + dictWord{6, 0, 1512}, + dictWord{135, 0, 1870}, + dictWord{9, 11, 31}, + dictWord{ + 10, + 11, + 244, + }, + dictWord{10, 11, 699}, + dictWord{12, 11, 149}, + dictWord{141, 11, 497}, + dictWord{133, 11, 377}, + dictWord{145, 11, 101}, + dictWord{ + 10, + 11, + 158, + }, + dictWord{13, 11, 13}, + dictWord{13, 11, 137}, + dictWord{13, 11, 258}, + dictWord{14, 11, 111}, + dictWord{14, 11, 225}, + dictWord{14, 11, 253}, + dictWord{ + 14, + 11, + 304, + }, + dictWord{14, 11, 339}, + dictWord{14, 11, 417}, + dictWord{146, 11, 33}, + dictWord{6, 0, 87}, + dictWord{6, 10, 1734}, + dictWord{7, 10, 20}, + dictWord{ + 7, + 10, + 1056, + }, + dictWord{8, 10, 732}, + dictWord{9, 10, 406}, + dictWord{9, 10, 911}, + dictWord{138, 10, 694}, + dictWord{134, 0, 1243}, + dictWord{137, 0, 245}, + dictWord{ + 7, + 0, + 68, + }, + dictWord{8, 0, 48}, + dictWord{8, 0, 88}, + dictWord{8, 0, 582}, + dictWord{8, 0, 681}, + dictWord{9, 0, 373}, + dictWord{9, 0, 864}, + dictWord{11, 0, 157}, + dictWord{ + 11, + 0, + 336, + }, + dictWord{11, 0, 843}, + dictWord{148, 0, 27}, + dictWord{8, 11, 663}, + dictWord{144, 11, 8}, + dictWord{133, 10, 613}, + dictWord{4, 0, 88}, + dictWord{ + 5, + 0, + 137, + }, + dictWord{5, 0, 174}, + dictWord{5, 0, 777}, + dictWord{6, 0, 1664}, + dictWord{6, 0, 1725}, + dictWord{7, 0, 77}, + dictWord{7, 0, 426}, + dictWord{7, 0, 1317}, + dictWord{ + 7, + 0, + 1355, + }, + dictWord{8, 0, 126}, + dictWord{8, 0, 563}, + dictWord{9, 0, 523}, + dictWord{9, 0, 750}, + dictWord{10, 0, 310}, + dictWord{10, 0, 836}, + dictWord{11, 0, 42}, + dictWord{11, 0, 318}, + dictWord{11, 0, 731}, + dictWord{12, 0, 68}, + dictWord{12, 0, 92}, + dictWord{12, 0, 507}, + dictWord{12, 0, 692}, + dictWord{13, 0, 81}, + dictWord{ + 13, + 0, + 238, + }, + dictWord{13, 0, 374}, + dictWord{14, 0, 436}, + dictWord{18, 0, 138}, + dictWord{19, 0, 78}, + dictWord{19, 0, 111}, + dictWord{20, 0, 55}, + dictWord{20, 0, 77}, + dictWord{148, 0, 92}, + dictWord{141, 0, 418}, + dictWord{4, 0, 938}, + dictWord{137, 0, 625}, + dictWord{138, 0, 351}, + dictWord{5, 11, 843}, + dictWord{7, 10, 32}, + dictWord{ + 7, + 10, + 984, + }, + dictWord{8, 10, 85}, + dictWord{8, 10, 709}, + dictWord{9, 10, 579}, + dictWord{9, 10, 847}, + dictWord{9, 10, 856}, + dictWord{10, 10, 799}, + dictWord{ + 11, + 10, + 258, + }, + dictWord{11, 10, 1007}, + dictWord{12, 10, 331}, + dictWord{12, 10, 615}, + dictWord{13, 10, 188}, + dictWord{13, 10, 435}, + dictWord{14, 10, 8}, + dictWord{ + 15, + 10, + 165, + }, + dictWord{16, 10, 27}, + dictWord{148, 10, 40}, + dictWord{6, 0, 1668}, + dictWord{7, 0, 1499}, + dictWord{8, 0, 117}, + dictWord{9, 0, 314}, + dictWord{ + 138, + 0, + 174, + }, + dictWord{135, 0, 707}, + dictWord{132, 11, 554}, + dictWord{133, 11, 536}, + dictWord{5, 0, 403}, + dictWord{5, 11, 207}, + dictWord{9, 11, 79}, + dictWord{ + 11, + 11, + 625, + }, + dictWord{145, 11, 7}, + dictWord{132, 11, 424}, + dictWord{136, 11, 785}, + dictWord{4, 10, 167}, + dictWord{135, 10, 82}, + dictWord{9, 0, 7}, + dictWord{ + 23, + 0, + 6, + }, + dictWord{9, 11, 7}, + dictWord{151, 11, 6}, + dictWord{6, 0, 282}, + dictWord{5, 10, 62}, + dictWord{6, 10, 534}, + dictWord{7, 10, 74}, + dictWord{7, 10, 678}, + dictWord{ + 7, + 10, + 684, + }, + dictWord{7, 10, 1043}, + dictWord{7, 10, 1072}, + dictWord{8, 10, 280}, + dictWord{8, 10, 541}, + dictWord{8, 10, 686}, + dictWord{9, 10, 258}, + dictWord{ + 10, + 10, + 519, + }, + dictWord{11, 10, 252}, + dictWord{140, 10, 282}, + dictWord{138, 10, 33}, + dictWord{132, 10, 359}, + dictWord{4, 0, 44}, + dictWord{5, 0, 311}, + dictWord{ + 6, + 0, + 156, + }, + dictWord{7, 0, 639}, + dictWord{7, 0, 762}, + dictWord{7, 0, 1827}, + dictWord{9, 0, 8}, + dictWord{9, 0, 462}, + dictWord{148, 0, 83}, + dictWord{7, 11, 769}, + dictWord{ + 9, + 11, + 18, + }, + dictWord{138, 11, 358}, + dictWord{4, 0, 346}, + dictWord{7, 0, 115}, + dictWord{9, 0, 180}, + dictWord{9, 0, 456}, + dictWord{10, 0, 363}, + dictWord{ + 4, + 11, + 896, + }, + dictWord{134, 11, 1777}, + dictWord{133, 10, 211}, + dictWord{7, 0, 761}, + dictWord{7, 0, 1051}, + dictWord{137, 0, 545}, + dictWord{6, 10, 145}, + dictWord{ + 141, + 10, + 336, + }, + dictWord{7, 11, 750}, + dictWord{9, 11, 223}, + dictWord{11, 11, 27}, + dictWord{11, 11, 466}, + dictWord{12, 11, 624}, + dictWord{14, 11, 265}, + dictWord{146, 11, 61}, + dictWord{6, 0, 752}, + dictWord{6, 0, 768}, + dictWord{6, 0, 1195}, + dictWord{6, 0, 1254}, + dictWord{6, 0, 1619}, + dictWord{137, 0, 835}, + dictWord{ + 6, + 0, + 1936, + }, + dictWord{8, 0, 930}, + dictWord{136, 0, 960}, + dictWord{132, 10, 263}, + dictWord{132, 11, 249}, + dictWord{12, 0, 653}, + dictWord{132, 10, 916}, + dictWord{4, 11, 603}, + dictWord{133, 11, 661}, + dictWord{8, 0, 344}, + dictWord{4, 11, 11}, + dictWord{6, 11, 128}, + dictWord{7, 11, 231}, + dictWord{7, 11, 1533}, + dictWord{138, 11, 725}, + dictWord{134, 0, 1483}, + dictWord{134, 0, 875}, + dictWord{6, 0, 185}, + dictWord{7, 0, 1899}, + dictWord{9, 0, 875}, + dictWord{139, 0, 673}, + dictWord{15, 10, 155}, + dictWord{144, 10, 79}, + dictWord{7, 0, 93}, + dictWord{7, 0, 210}, + dictWord{7, 0, 1223}, + dictWord{8, 0, 451}, + dictWord{8, 0, 460}, + dictWord{ + 11, + 0, + 353, + }, + dictWord{11, 0, 475}, + dictWord{4, 10, 599}, + dictWord{6, 10, 1634}, + dictWord{7, 10, 67}, + dictWord{7, 10, 691}, + dictWord{7, 10, 979}, + dictWord{ + 7, + 10, + 1697, + }, + dictWord{8, 10, 207}, + dictWord{8, 10, 214}, + dictWord{8, 10, 231}, + dictWord{8, 10, 294}, + dictWord{8, 10, 336}, + dictWord{8, 10, 428}, + dictWord{ + 8, + 10, + 471, + }, + dictWord{8, 10, 622}, + dictWord{8, 10, 626}, + dictWord{8, 10, 679}, + dictWord{8, 10, 759}, + dictWord{8, 10, 829}, + dictWord{9, 10, 11}, + dictWord{9, 10, 246}, + dictWord{9, 10, 484}, + dictWord{9, 10, 573}, + dictWord{9, 10, 706}, + dictWord{9, 10, 762}, + dictWord{9, 10, 798}, + dictWord{9, 10, 855}, + dictWord{9, 10, 870}, + dictWord{ + 9, + 10, + 912, + }, + dictWord{10, 10, 303}, + dictWord{10, 10, 335}, + dictWord{10, 10, 424}, + dictWord{10, 10, 461}, + dictWord{10, 10, 543}, + dictWord{10, 10, 759}, + dictWord{10, 10, 814}, + dictWord{11, 10, 59}, + dictWord{11, 10, 235}, + dictWord{11, 10, 590}, + dictWord{11, 10, 929}, + dictWord{11, 10, 963}, + dictWord{ + 11, + 10, + 987, + }, + dictWord{12, 10, 114}, + dictWord{12, 10, 182}, + dictWord{12, 10, 226}, + dictWord{12, 10, 332}, + dictWord{12, 10, 439}, + dictWord{12, 10, 575}, + dictWord{ + 12, + 10, + 598, + }, + dictWord{12, 10, 675}, + dictWord{13, 10, 8}, + dictWord{13, 10, 125}, + dictWord{13, 10, 194}, + dictWord{13, 10, 287}, + dictWord{14, 10, 197}, + dictWord{14, 10, 383}, + dictWord{15, 10, 53}, + dictWord{17, 10, 63}, + dictWord{19, 10, 46}, + dictWord{19, 10, 98}, + dictWord{19, 10, 106}, + dictWord{148, 10, 85}, + dictWord{132, 11, 476}, + dictWord{4, 0, 327}, + dictWord{5, 0, 478}, + dictWord{7, 0, 1332}, + dictWord{136, 0, 753}, + dictWord{5, 0, 1020}, + dictWord{133, 0, 1022}, + dictWord{135, 11, 1807}, + dictWord{4, 0, 103}, + dictWord{133, 0, 401}, + dictWord{4, 0, 499}, + dictWord{135, 0, 1421}, + dictWord{10, 0, 207}, + dictWord{13, 0, 164}, + dictWord{147, 10, 126}, + dictWord{9, 11, 20}, + dictWord{10, 11, 324}, + dictWord{139, 11, 488}, + dictWord{132, 0, 96}, + dictWord{9, 11, 280}, + dictWord{ + 138, + 11, + 134, + }, + dictWord{135, 0, 968}, + dictWord{133, 10, 187}, + dictWord{135, 10, 1286}, + dictWord{5, 11, 112}, + dictWord{6, 11, 103}, + dictWord{134, 11, 150}, + dictWord{8, 0, 914}, + dictWord{10, 0, 3}, + dictWord{4, 10, 215}, + dictWord{9, 10, 38}, + dictWord{11, 10, 23}, + dictWord{11, 10, 127}, + dictWord{139, 10, 796}, + dictWord{ + 135, + 0, + 399, + }, + dictWord{6, 0, 563}, + dictWord{137, 0, 224}, + dictWord{6, 0, 704}, + dictWord{134, 0, 1214}, + dictWord{4, 11, 708}, + dictWord{8, 11, 15}, + dictWord{ + 9, + 11, + 50, + }, + dictWord{9, 11, 386}, + dictWord{11, 11, 18}, + dictWord{11, 11, 529}, + dictWord{140, 11, 228}, + dictWord{4, 11, 563}, + dictWord{7, 11, 109}, + dictWord{ + 7, + 11, + 592, + }, + dictWord{7, 11, 637}, + dictWord{7, 11, 770}, + dictWord{7, 11, 1701}, + dictWord{8, 11, 436}, + dictWord{8, 11, 463}, + dictWord{9, 11, 60}, + dictWord{9, 11, 335}, + dictWord{9, 11, 904}, + dictWord{10, 11, 73}, + dictWord{11, 11, 434}, + dictWord{12, 11, 585}, + dictWord{13, 11, 331}, + dictWord{18, 11, 110}, + dictWord{ + 148, + 11, + 60, + }, + dictWord{134, 0, 1559}, + dictWord{132, 11, 502}, + dictWord{6, 11, 347}, + dictWord{138, 11, 161}, + dictWord{4, 11, 33}, + dictWord{5, 11, 102}, + dictWord{ + 5, + 11, + 500, + }, + dictWord{6, 11, 284}, + dictWord{7, 11, 1079}, + dictWord{7, 11, 1423}, + dictWord{7, 11, 1702}, + dictWord{8, 11, 470}, + dictWord{9, 11, 554}, + dictWord{ + 9, + 11, + 723, + }, + dictWord{139, 11, 333}, + dictWord{7, 11, 246}, + dictWord{135, 11, 840}, + dictWord{6, 11, 10}, + dictWord{8, 11, 571}, + dictWord{9, 11, 739}, + dictWord{ + 143, + 11, + 91, + }, + dictWord{8, 0, 861}, + dictWord{10, 0, 905}, + dictWord{12, 0, 730}, + dictWord{12, 0, 789}, + dictWord{133, 11, 626}, + dictWord{134, 0, 946}, + dictWord{ + 5, + 0, + 746, + }, + dictWord{12, 0, 333}, + dictWord{14, 0, 332}, + dictWord{12, 11, 333}, + dictWord{142, 11, 332}, + dictWord{5, 11, 18}, + dictWord{6, 11, 526}, + dictWord{ + 13, + 11, + 24, + }, + dictWord{13, 11, 110}, + dictWord{19, 11, 5}, + dictWord{147, 11, 44}, + dictWord{4, 0, 910}, + dictWord{5, 0, 832}, + dictWord{135, 10, 2002}, + dictWord{ + 10, + 11, + 768, + }, + dictWord{139, 11, 787}, + dictWord{4, 11, 309}, + dictWord{5, 11, 462}, + dictWord{7, 11, 970}, + dictWord{135, 11, 1097}, + dictWord{4, 10, 28}, + dictWord{ + 5, + 10, + 440, + }, + dictWord{7, 10, 248}, + dictWord{11, 10, 833}, + dictWord{140, 10, 344}, + dictWord{134, 10, 1654}, + dictWord{6, 0, 632}, + dictWord{6, 0, 652}, + dictWord{ + 6, + 0, + 1272, + }, + dictWord{6, 0, 1384}, + dictWord{134, 0, 1560}, + dictWord{134, 11, 1704}, + dictWord{6, 0, 1393}, + dictWord{133, 10, 853}, + dictWord{6, 10, 249}, + dictWord{7, 10, 1234}, + dictWord{139, 10, 573}, + dictWord{5, 11, 86}, + dictWord{7, 11, 743}, + dictWord{9, 11, 85}, + dictWord{10, 11, 281}, + dictWord{10, 11, 432}, + dictWord{11, 11, 490}, + dictWord{12, 11, 251}, + dictWord{13, 11, 118}, + dictWord{14, 11, 378}, + dictWord{146, 11, 143}, + dictWord{5, 11, 524}, + dictWord{ + 133, + 11, + 744, + }, + dictWord{134, 0, 1514}, + dictWord{10, 0, 201}, + dictWord{142, 0, 319}, + dictWord{7, 0, 717}, + dictWord{10, 0, 510}, + dictWord{7, 10, 392}, + dictWord{ + 8, + 10, + 20, + }, + dictWord{8, 10, 172}, + dictWord{8, 10, 690}, + dictWord{9, 10, 383}, + dictWord{9, 10, 845}, + dictWord{11, 10, 293}, + dictWord{11, 10, 832}, + dictWord{ + 11, + 10, + 920, + }, + dictWord{11, 10, 984}, + dictWord{141, 10, 221}, + dictWord{134, 0, 1381}, + dictWord{5, 10, 858}, + dictWord{133, 10, 992}, + dictWord{8, 0, 528}, + dictWord{137, 0, 348}, + dictWord{10, 11, 107}, + dictWord{140, 11, 436}, + dictWord{4, 0, 20}, + dictWord{133, 0, 616}, + dictWord{134, 0, 1251}, + dictWord{ + 132, + 11, + 927, + }, + dictWord{10, 11, 123}, + dictWord{12, 11, 670}, + dictWord{13, 11, 371}, + dictWord{14, 11, 142}, + dictWord{146, 11, 94}, + dictWord{134, 0, 1163}, + dictWord{ + 7, + 11, + 1149, + }, + dictWord{137, 11, 156}, + dictWord{134, 0, 307}, + dictWord{133, 11, 778}, + dictWord{7, 0, 1091}, + dictWord{135, 0, 1765}, + dictWord{ + 5, + 11, + 502, + }, + dictWord{6, 10, 268}, + dictWord{137, 10, 62}, + dictWord{8, 11, 196}, + dictWord{10, 11, 283}, + dictWord{139, 11, 406}, + dictWord{4, 0, 26}, + dictWord{ + 5, + 0, + 429, + }, + dictWord{6, 0, 245}, + dictWord{7, 0, 704}, + dictWord{7, 0, 1379}, + dictWord{135, 0, 1474}, + dictWord{133, 11, 855}, + dictWord{132, 0, 881}, + dictWord{ + 4, + 0, + 621, + }, + dictWord{135, 11, 1596}, + dictWord{7, 11, 1400}, + dictWord{9, 11, 446}, + dictWord{138, 11, 45}, + dictWord{6, 0, 736}, + dictWord{138, 10, 106}, + dictWord{133, 0, 542}, + dictWord{134, 0, 348}, + dictWord{133, 0, 868}, + dictWord{136, 0, 433}, + dictWord{135, 0, 1495}, + dictWord{138, 0, 771}, + dictWord{ + 6, + 10, + 613, + }, + dictWord{136, 10, 223}, + dictWord{138, 0, 215}, + dictWord{141, 0, 124}, + dictWord{136, 11, 391}, + dictWord{135, 11, 172}, + dictWord{132, 10, 670}, + dictWord{140, 0, 55}, + dictWord{9, 10, 40}, + dictWord{139, 10, 136}, + dictWord{7, 0, 62}, + dictWord{147, 0, 112}, + dictWord{132, 0, 856}, + dictWord{132, 11, 568}, + dictWord{12, 0, 270}, + dictWord{139, 10, 259}, + dictWord{8, 0, 572}, + dictWord{137, 0, 698}, + dictWord{4, 11, 732}, + dictWord{9, 10, 310}, + dictWord{137, 10, 682}, + dictWord{142, 10, 296}, + dictWord{134, 0, 939}, + dictWord{136, 11, 733}, + dictWord{135, 11, 1435}, + dictWord{7, 10, 1401}, + dictWord{135, 10, 1476}, + dictWord{6, 0, 352}, + dictWord{4, 10, 296}, + dictWord{7, 10, 401}, + dictWord{7, 10, 1410}, + dictWord{7, 10, 1594}, + dictWord{7, 10, 1674}, + dictWord{8, 10, 63}, + dictWord{ + 8, + 10, + 660, + }, + dictWord{137, 10, 74}, + dictWord{4, 11, 428}, + dictWord{133, 11, 668}, + dictWord{4, 10, 139}, + dictWord{4, 10, 388}, + dictWord{140, 10, 188}, + dictWord{7, 11, 2015}, + dictWord{140, 11, 665}, + dictWord{132, 0, 647}, + dictWord{146, 0, 10}, + dictWord{138, 0, 220}, + dictWord{142, 0, 464}, + dictWord{ + 132, + 0, + 109, + }, + dictWord{134, 0, 1746}, + dictWord{6, 0, 515}, + dictWord{4, 10, 747}, + dictWord{6, 11, 1623}, + dictWord{6, 11, 1681}, + dictWord{7, 10, 649}, + dictWord{ + 7, + 10, + 1479, + }, + dictWord{135, 10, 1583}, + dictWord{133, 10, 232}, + dictWord{135, 0, 566}, + dictWord{137, 10, 887}, + dictWord{4, 0, 40}, + dictWord{10, 0, 67}, + dictWord{ + 11, + 0, + 117, + }, + dictWord{11, 0, 768}, + dictWord{139, 0, 935}, + dictWord{132, 0, 801}, + dictWord{7, 0, 992}, + dictWord{8, 0, 301}, + dictWord{9, 0, 722}, + dictWord{ + 12, + 0, + 63, + }, + dictWord{13, 0, 29}, + dictWord{14, 0, 161}, + dictWord{143, 0, 18}, + dictWord{139, 0, 923}, + dictWord{6, 11, 1748}, + dictWord{8, 11, 715}, + dictWord{9, 11, 802}, + dictWord{10, 11, 46}, + dictWord{10, 11, 819}, + dictWord{13, 11, 308}, + dictWord{14, 11, 351}, + dictWord{14, 11, 363}, + dictWord{146, 11, 67}, + dictWord{ + 137, + 11, + 745, + }, + dictWord{7, 0, 1145}, + dictWord{4, 10, 14}, + dictWord{7, 10, 1801}, + dictWord{10, 10, 748}, + dictWord{141, 10, 458}, + dictWord{4, 11, 63}, + dictWord{ + 5, + 11, + 347, + }, + dictWord{134, 11, 474}, + dictWord{135, 0, 568}, + dictWord{4, 10, 425}, + dictWord{7, 11, 577}, + dictWord{7, 11, 1432}, + dictWord{9, 11, 475}, + dictWord{ + 9, + 11, + 505, + }, + dictWord{9, 11, 526}, + dictWord{9, 11, 609}, + dictWord{9, 11, 689}, + dictWord{9, 11, 726}, + dictWord{9, 11, 735}, + dictWord{9, 11, 738}, + dictWord{ + 10, + 11, + 556, + }, + dictWord{10, 11, 674}, + dictWord{10, 11, 684}, + dictWord{11, 11, 89}, + dictWord{11, 11, 202}, + dictWord{11, 11, 272}, + dictWord{11, 11, 380}, + dictWord{ + 11, + 11, + 415, + }, + dictWord{11, 11, 505}, + dictWord{11, 11, 537}, + dictWord{11, 11, 550}, + dictWord{11, 11, 562}, + dictWord{11, 11, 640}, + dictWord{11, 11, 667}, + dictWord{11, 11, 688}, + dictWord{11, 11, 847}, + dictWord{11, 11, 927}, + dictWord{11, 11, 930}, + dictWord{11, 11, 940}, + dictWord{12, 11, 144}, + dictWord{ + 12, + 11, + 325, + }, + dictWord{12, 11, 329}, + dictWord{12, 11, 389}, + dictWord{12, 11, 403}, + dictWord{12, 11, 451}, + dictWord{12, 11, 515}, + dictWord{12, 11, 604}, + dictWord{ + 12, + 11, + 616, + }, + dictWord{12, 11, 626}, + dictWord{13, 11, 66}, + dictWord{13, 11, 131}, + dictWord{13, 11, 167}, + dictWord{13, 11, 236}, + dictWord{13, 11, 368}, + dictWord{13, 11, 411}, + dictWord{13, 11, 434}, + dictWord{13, 11, 453}, + dictWord{13, 11, 461}, + dictWord{13, 11, 474}, + dictWord{14, 11, 59}, + dictWord{14, 11, 60}, + dictWord{14, 11, 139}, + dictWord{14, 11, 152}, + dictWord{14, 11, 276}, + dictWord{14, 11, 353}, + dictWord{14, 11, 402}, + dictWord{15, 11, 28}, + dictWord{ + 15, + 11, + 81, + }, + dictWord{15, 11, 123}, + dictWord{15, 11, 152}, + dictWord{18, 11, 136}, + dictWord{148, 11, 88}, + dictWord{137, 0, 247}, + dictWord{135, 11, 1622}, + dictWord{ + 9, + 11, + 544, + }, + dictWord{11, 11, 413}, + dictWord{144, 11, 25}, + dictWord{4, 0, 645}, + dictWord{7, 0, 825}, + dictWord{6, 10, 1768}, + dictWord{135, 11, 89}, + dictWord{140, 0, 328}, + dictWord{5, 10, 943}, + dictWord{134, 10, 1779}, + dictWord{134, 0, 1363}, + dictWord{5, 10, 245}, + dictWord{6, 10, 576}, + dictWord{7, 10, 582}, + dictWord{136, 10, 225}, + dictWord{134, 0, 1280}, + dictWord{5, 11, 824}, + dictWord{133, 11, 941}, + dictWord{7, 11, 440}, + dictWord{8, 11, 230}, + dictWord{ + 139, + 11, + 106, + }, + dictWord{5, 0, 28}, + dictWord{6, 0, 204}, + dictWord{10, 0, 320}, + dictWord{10, 0, 583}, + dictWord{13, 0, 502}, + dictWord{14, 0, 72}, + dictWord{14, 0, 274}, + dictWord{14, 0, 312}, + dictWord{14, 0, 344}, + dictWord{15, 0, 159}, + dictWord{16, 0, 62}, + dictWord{16, 0, 69}, + dictWord{17, 0, 30}, + dictWord{18, 0, 42}, + dictWord{ + 18, + 0, + 53, + }, + dictWord{18, 0, 84}, + dictWord{18, 0, 140}, + dictWord{19, 0, 68}, + dictWord{19, 0, 85}, + dictWord{20, 0, 5}, + dictWord{20, 0, 45}, + dictWord{20, 0, 101}, + dictWord{ + 22, + 0, + 7, + }, + dictWord{150, 0, 20}, + dictWord{4, 0, 558}, + dictWord{6, 0, 390}, + dictWord{7, 0, 162}, + dictWord{7, 0, 689}, + dictWord{9, 0, 360}, + dictWord{138, 0, 653}, + dictWord{134, 0, 764}, + dictWord{6, 0, 862}, + dictWord{137, 0, 833}, + dictWord{5, 0, 856}, + dictWord{6, 0, 1672}, + dictWord{6, 0, 1757}, + dictWord{134, 0, 1781}, + dictWord{ + 5, + 0, + 92, + }, + dictWord{10, 0, 736}, + dictWord{140, 0, 102}, + dictWord{6, 0, 1927}, + dictWord{6, 0, 1944}, + dictWord{8, 0, 924}, + dictWord{8, 0, 948}, + dictWord{ + 10, + 0, + 967, + }, + dictWord{138, 0, 978}, + dictWord{134, 0, 1479}, + dictWord{5, 0, 590}, + dictWord{8, 0, 360}, + dictWord{9, 0, 213}, + dictWord{138, 0, 63}, + dictWord{ + 134, + 0, + 1521, + }, + dictWord{6, 0, 709}, + dictWord{134, 0, 891}, + dictWord{132, 10, 443}, + dictWord{13, 0, 477}, + dictWord{14, 0, 120}, + dictWord{148, 0, 61}, + dictWord{ + 4, + 11, + 914, + }, + dictWord{5, 11, 800}, + dictWord{133, 11, 852}, + dictWord{10, 11, 54}, + dictWord{141, 11, 115}, + dictWord{4, 11, 918}, + dictWord{133, 11, 876}, + dictWord{139, 11, 152}, + dictWord{4, 11, 92}, + dictWord{133, 11, 274}, + dictWord{135, 11, 1901}, + dictWord{9, 11, 800}, + dictWord{10, 11, 693}, + dictWord{ + 11, + 11, + 482, + }, + dictWord{11, 11, 734}, + dictWord{139, 11, 789}, + dictWord{9, 0, 483}, + dictWord{132, 10, 298}, + dictWord{6, 0, 1213}, + dictWord{141, 11, 498}, + dictWord{135, 11, 1451}, + dictWord{133, 11, 743}, + dictWord{4, 0, 1022}, + dictWord{10, 0, 1000}, + dictWord{12, 0, 957}, + dictWord{12, 0, 980}, + dictWord{ + 12, + 0, + 1013, + }, + dictWord{14, 0, 481}, + dictWord{144, 0, 116}, + dictWord{8, 0, 503}, + dictWord{17, 0, 29}, + dictWord{4, 11, 49}, + dictWord{7, 11, 280}, + dictWord{ + 135, + 11, + 1633, + }, + dictWord{135, 0, 1712}, + dictWord{134, 0, 466}, + dictWord{136, 11, 47}, + dictWord{5, 10, 164}, + dictWord{7, 10, 121}, + dictWord{142, 10, 189}, + dictWord{ + 7, + 10, + 812, + }, + dictWord{7, 10, 1261}, + dictWord{7, 10, 1360}, + dictWord{9, 10, 632}, + dictWord{140, 10, 352}, + dictWord{139, 10, 556}, + dictWord{132, 0, 731}, + dictWord{5, 11, 272}, + dictWord{5, 11, 908}, + dictWord{5, 11, 942}, + dictWord{7, 11, 1008}, + dictWord{7, 11, 1560}, + dictWord{8, 11, 197}, + dictWord{9, 11, 47}, + dictWord{11, 11, 538}, + dictWord{139, 11, 742}, + dictWord{4, 10, 172}, + dictWord{9, 10, 611}, + dictWord{10, 10, 436}, + dictWord{12, 10, 673}, + dictWord{ + 141, + 10, + 255, + }, + dictWord{133, 10, 844}, + dictWord{10, 0, 484}, + dictWord{11, 0, 754}, + dictWord{12, 0, 457}, + dictWord{14, 0, 171}, + dictWord{14, 0, 389}, + dictWord{ + 146, + 0, + 153, + }, + dictWord{9, 10, 263}, + dictWord{10, 10, 147}, + dictWord{138, 10, 492}, + dictWord{137, 11, 891}, + dictWord{138, 0, 241}, + dictWord{133, 10, 537}, + dictWord{6, 0, 2005}, + dictWord{136, 0, 964}, + dictWord{137, 10, 842}, + dictWord{151, 11, 8}, + dictWord{4, 11, 407}, + dictWord{132, 11, 560}, + dictWord{ + 135, + 11, + 1884, + }, + dictWord{6, 0, 1100}, + dictWord{134, 0, 1242}, + dictWord{135, 0, 954}, + dictWord{5, 10, 230}, + dictWord{5, 10, 392}, + dictWord{6, 10, 420}, + dictWord{ + 9, + 10, + 568, + }, + dictWord{140, 10, 612}, + dictWord{4, 11, 475}, + dictWord{11, 11, 35}, + dictWord{11, 11, 90}, + dictWord{13, 11, 7}, + dictWord{13, 11, 71}, + dictWord{ + 13, + 11, + 177, + }, + dictWord{142, 11, 422}, + dictWord{136, 11, 332}, + dictWord{135, 0, 1958}, + dictWord{6, 0, 549}, + dictWord{8, 0, 34}, + dictWord{8, 0, 283}, + dictWord{ + 9, + 0, + 165, + }, + dictWord{138, 0, 475}, + dictWord{10, 0, 952}, + dictWord{12, 0, 966}, + dictWord{140, 0, 994}, + dictWord{5, 0, 652}, + dictWord{5, 0, 701}, + dictWord{ + 135, + 0, + 449, + }, + dictWord{4, 0, 655}, + dictWord{7, 0, 850}, + dictWord{17, 0, 75}, + dictWord{146, 0, 137}, + dictWord{4, 0, 146}, + dictWord{7, 0, 1618}, + dictWord{8, 0, 670}, + dictWord{ + 5, + 10, + 41, + }, + dictWord{7, 10, 1459}, + dictWord{7, 10, 1469}, + dictWord{7, 10, 1859}, + dictWord{9, 10, 549}, + dictWord{139, 10, 905}, + dictWord{133, 10, 696}, + dictWord{6, 0, 159}, + dictWord{6, 0, 364}, + dictWord{7, 0, 516}, + dictWord{137, 0, 518}, + dictWord{135, 0, 1439}, + dictWord{6, 11, 222}, + dictWord{7, 11, 636}, + dictWord{ + 7, + 11, + 1620, + }, + dictWord{8, 11, 409}, + dictWord{9, 11, 693}, + dictWord{139, 11, 77}, + dictWord{13, 0, 151}, + dictWord{141, 11, 45}, + dictWord{6, 0, 1027}, + dictWord{ + 4, + 11, + 336, + }, + dictWord{132, 10, 771}, + dictWord{139, 11, 392}, + dictWord{10, 11, 121}, + dictWord{11, 11, 175}, + dictWord{149, 11, 16}, + dictWord{8, 0, 950}, + dictWord{138, 0, 983}, + dictWord{133, 10, 921}, + dictWord{135, 0, 993}, + dictWord{6, 10, 180}, + dictWord{7, 10, 1137}, + dictWord{8, 10, 751}, + dictWord{ + 139, + 10, + 805, + }, + dictWord{7, 0, 501}, + dictWord{9, 0, 111}, + dictWord{10, 0, 141}, + dictWord{11, 0, 332}, + dictWord{13, 0, 43}, + dictWord{13, 0, 429}, + dictWord{14, 0, 130}, + dictWord{14, 0, 415}, + dictWord{145, 0, 102}, + dictWord{4, 10, 183}, + dictWord{5, 11, 882}, + dictWord{7, 10, 271}, + dictWord{11, 10, 824}, + dictWord{11, 10, 952}, + dictWord{13, 10, 278}, + dictWord{13, 10, 339}, + dictWord{13, 10, 482}, + dictWord{14, 10, 424}, + dictWord{148, 10, 99}, + dictWord{4, 10, 19}, + dictWord{5, 10, 477}, + dictWord{5, 10, 596}, + dictWord{6, 10, 505}, + dictWord{7, 10, 1221}, + dictWord{11, 10, 907}, + dictWord{12, 10, 209}, + dictWord{141, 10, 214}, + dictWord{ + 135, + 10, + 1215, + }, + dictWord{133, 0, 452}, + dictWord{132, 11, 426}, + dictWord{5, 0, 149}, + dictWord{136, 0, 233}, + dictWord{133, 0, 935}, + dictWord{6, 11, 58}, + dictWord{ + 7, + 11, + 654, + }, + dictWord{7, 11, 745}, + dictWord{7, 11, 1969}, + dictWord{8, 11, 240}, + dictWord{8, 11, 675}, + dictWord{9, 11, 479}, + dictWord{9, 11, 731}, + dictWord{ + 10, + 11, + 330, + }, + dictWord{10, 11, 593}, + dictWord{10, 11, 817}, + dictWord{11, 11, 32}, + dictWord{11, 11, 133}, + dictWord{11, 11, 221}, + dictWord{145, 11, 68}, + dictWord{ + 12, + 0, + 582, + }, + dictWord{18, 0, 131}, + dictWord{7, 11, 102}, + dictWord{137, 11, 538}, + dictWord{136, 0, 801}, + dictWord{134, 10, 1645}, + dictWord{132, 0, 70}, + dictWord{6, 10, 92}, + dictWord{6, 10, 188}, + dictWord{7, 10, 1269}, + dictWord{7, 10, 1524}, + dictWord{7, 10, 1876}, + dictWord{10, 10, 228}, + dictWord{139, 10, 1020}, + dictWord{4, 10, 459}, + dictWord{133, 10, 966}, + dictWord{138, 0, 369}, + dictWord{16, 0, 36}, + dictWord{140, 10, 330}, + dictWord{141, 11, 366}, + dictWord{ + 7, + 0, + 721, + }, + dictWord{10, 0, 236}, + dictWord{12, 0, 204}, + dictWord{6, 10, 18}, + dictWord{7, 10, 932}, + dictWord{8, 10, 757}, + dictWord{9, 10, 54}, + dictWord{9, 10, 65}, + dictWord{9, 10, 844}, + dictWord{10, 10, 113}, + dictWord{10, 10, 315}, + dictWord{10, 10, 798}, + dictWord{11, 10, 153}, + dictWord{12, 10, 151}, + dictWord{12, 10, 392}, + dictWord{12, 10, 666}, + dictWord{142, 10, 248}, + dictWord{7, 0, 241}, + dictWord{10, 0, 430}, + dictWord{8, 10, 548}, + dictWord{9, 10, 532}, + dictWord{10, 10, 117}, + dictWord{11, 10, 351}, + dictWord{11, 10, 375}, + dictWord{143, 10, 23}, + dictWord{134, 10, 1742}, + dictWord{133, 10, 965}, + dictWord{133, 11, 566}, + dictWord{ + 6, + 11, + 48, + }, + dictWord{135, 11, 63}, + dictWord{134, 10, 182}, + dictWord{10, 10, 65}, + dictWord{10, 10, 488}, + dictWord{138, 10, 497}, + dictWord{6, 11, 114}, + dictWord{7, 11, 1224}, + dictWord{7, 11, 1556}, + dictWord{136, 11, 3}, + dictWord{134, 0, 1817}, + dictWord{8, 11, 576}, + dictWord{137, 11, 267}, + dictWord{ + 6, + 0, + 1078, + }, + dictWord{144, 0, 16}, + dictWord{9, 10, 588}, + dictWord{138, 10, 260}, + dictWord{138, 0, 1021}, + dictWord{5, 0, 406}, + dictWord{134, 0, 2022}, + dictWord{133, 11, 933}, + dictWord{6, 0, 69}, + dictWord{135, 0, 117}, + dictWord{7, 0, 1830}, + dictWord{136, 11, 427}, + dictWord{4, 0, 432}, + dictWord{135, 0, 824}, + dictWord{134, 10, 1786}, + dictWord{133, 0, 826}, + dictWord{139, 11, 67}, + dictWord{133, 11, 759}, + dictWord{135, 10, 308}, + dictWord{137, 0, 816}, + dictWord{ + 133, + 0, + 1000, + }, + dictWord{4, 0, 297}, + dictWord{6, 0, 529}, + dictWord{7, 0, 152}, + dictWord{7, 0, 713}, + dictWord{7, 0, 1845}, + dictWord{8, 0, 710}, + dictWord{8, 0, 717}, + dictWord{12, 0, 639}, + dictWord{140, 0, 685}, + dictWord{7, 0, 423}, + dictWord{136, 10, 588}, + dictWord{136, 10, 287}, + dictWord{136, 0, 510}, + dictWord{ + 134, + 0, + 1048, + }, + dictWord{6, 0, 618}, + dictWord{7, 11, 56}, + dictWord{7, 11, 1989}, + dictWord{8, 11, 337}, + dictWord{8, 11, 738}, + dictWord{9, 11, 600}, + dictWord{ + 10, + 11, + 483, + }, + dictWord{12, 11, 37}, + dictWord{13, 11, 447}, + dictWord{142, 11, 92}, + dictWord{4, 0, 520}, + dictWord{135, 0, 575}, + dictWord{8, 0, 990}, + dictWord{ + 138, + 0, + 977, + }, + dictWord{135, 11, 774}, + dictWord{9, 11, 347}, + dictWord{11, 11, 24}, + dictWord{140, 11, 170}, + dictWord{136, 11, 379}, + dictWord{140, 10, 290}, + dictWord{132, 11, 328}, + dictWord{4, 0, 321}, + dictWord{134, 0, 569}, + dictWord{4, 11, 101}, + dictWord{135, 11, 1171}, + dictWord{7, 0, 723}, + dictWord{7, 0, 1135}, + dictWord{5, 11, 833}, + dictWord{136, 11, 744}, + dictWord{7, 10, 719}, + dictWord{8, 10, 809}, + dictWord{136, 10, 834}, + dictWord{8, 0, 921}, + dictWord{136, 10, 796}, + dictWord{5, 10, 210}, + dictWord{6, 10, 213}, + dictWord{7, 10, 60}, + dictWord{10, 10, 364}, + dictWord{139, 10, 135}, + dictWord{5, 0, 397}, + dictWord{6, 0, 154}, + dictWord{7, 0, 676}, + dictWord{8, 0, 443}, + dictWord{8, 0, 609}, + dictWord{9, 0, 24}, + dictWord{9, 0, 325}, + dictWord{10, 0, 35}, + dictWord{11, 0, 535}, + dictWord{11, 0, 672}, + dictWord{11, 0, 1018}, + dictWord{12, 0, 637}, + dictWord{16, 0, 30}, + dictWord{5, 10, 607}, + dictWord{8, 10, 326}, + dictWord{136, 10, 490}, + dictWord{4, 10, 701}, + dictWord{5, 10, 472}, + dictWord{6, 11, 9}, + dictWord{6, 11, 397}, + dictWord{7, 11, 53}, + dictWord{7, 11, 1742}, + dictWord{9, 10, 758}, + dictWord{10, 11, 632}, + dictWord{ + 11, + 11, + 828, + }, + dictWord{140, 11, 146}, + dictWord{135, 10, 380}, + dictWord{135, 10, 1947}, + dictWord{148, 11, 109}, + dictWord{10, 10, 278}, + dictWord{ + 138, + 11, + 278, + }, + dictWord{134, 0, 856}, + dictWord{7, 0, 139}, + dictWord{4, 10, 386}, + dictWord{8, 10, 405}, + dictWord{8, 10, 728}, + dictWord{9, 10, 497}, + dictWord{ + 11, + 10, + 110, + }, + dictWord{11, 10, 360}, + dictWord{15, 10, 37}, + dictWord{144, 10, 84}, + dictWord{141, 0, 282}, + dictWord{133, 0, 981}, + dictWord{5, 0, 288}, + dictWord{ + 7, + 10, + 1452, + }, + dictWord{7, 10, 1480}, + dictWord{8, 10, 634}, + dictWord{140, 10, 472}, + dictWord{7, 0, 1890}, + dictWord{8, 11, 367}, + dictWord{10, 11, 760}, + dictWord{ + 14, + 11, + 79, + }, + dictWord{20, 11, 17}, + dictWord{152, 11, 0}, + dictWord{4, 10, 524}, + dictWord{136, 10, 810}, + dictWord{4, 0, 56}, + dictWord{7, 0, 1791}, + dictWord{ + 8, + 0, + 607, + }, + dictWord{8, 0, 651}, + dictWord{11, 0, 465}, + dictWord{11, 0, 835}, + dictWord{12, 0, 337}, + dictWord{141, 0, 480}, + dictWord{10, 10, 238}, + dictWord{ + 141, + 10, + 33, + }, + dictWord{11, 11, 417}, + dictWord{12, 11, 223}, + dictWord{140, 11, 265}, + dictWord{9, 0, 158}, + dictWord{10, 0, 411}, + dictWord{140, 0, 261}, + dictWord{ + 133, + 10, + 532, + }, + dictWord{133, 10, 997}, + dictWord{12, 11, 186}, + dictWord{12, 11, 292}, + dictWord{14, 11, 100}, + dictWord{146, 11, 70}, + dictWord{6, 0, 1403}, + dictWord{136, 0, 617}, + dictWord{134, 0, 1205}, + dictWord{139, 0, 563}, + dictWord{4, 0, 242}, + dictWord{134, 0, 333}, + dictWord{4, 11, 186}, + dictWord{5, 11, 157}, + dictWord{8, 11, 168}, + dictWord{138, 11, 6}, + dictWord{132, 0, 369}, + dictWord{133, 11, 875}, + dictWord{5, 10, 782}, + dictWord{5, 10, 829}, + dictWord{ + 134, + 10, + 1738, + }, + dictWord{134, 0, 622}, + dictWord{135, 11, 1272}, + dictWord{6, 0, 1407}, + dictWord{7, 11, 111}, + dictWord{136, 11, 581}, + dictWord{7, 10, 1823}, + dictWord{139, 10, 693}, + dictWord{7, 0, 160}, + dictWord{10, 0, 624}, + dictWord{142, 0, 279}, + dictWord{132, 0, 363}, + dictWord{10, 11, 589}, + dictWord{12, 11, 111}, + dictWord{13, 11, 260}, + dictWord{14, 11, 82}, + dictWord{18, 11, 63}, + dictWord{147, 11, 45}, + dictWord{7, 11, 1364}, + dictWord{7, 11, 1907}, + dictWord{ + 141, + 11, + 158, + }, + dictWord{4, 11, 404}, + dictWord{4, 11, 659}, + dictWord{135, 11, 675}, + dictWord{13, 11, 211}, + dictWord{14, 11, 133}, + dictWord{14, 11, 204}, + dictWord{ + 15, + 11, + 64, + }, + dictWord{15, 11, 69}, + dictWord{15, 11, 114}, + dictWord{16, 11, 10}, + dictWord{19, 11, 23}, + dictWord{19, 11, 35}, + dictWord{19, 11, 39}, + dictWord{ + 19, + 11, + 51, + }, + dictWord{19, 11, 71}, + dictWord{19, 11, 75}, + dictWord{152, 11, 15}, + dictWord{4, 10, 78}, + dictWord{5, 10, 96}, + dictWord{5, 10, 182}, + dictWord{7, 10, 1724}, + dictWord{7, 10, 1825}, + dictWord{10, 10, 394}, + dictWord{10, 10, 471}, + dictWord{11, 10, 532}, + dictWord{14, 10, 340}, + dictWord{145, 10, 88}, + dictWord{ + 135, + 10, + 1964, + }, + dictWord{133, 11, 391}, + dictWord{11, 11, 887}, + dictWord{14, 11, 365}, + dictWord{142, 11, 375}, + dictWord{5, 11, 540}, + dictWord{6, 11, 1697}, + dictWord{7, 11, 222}, + dictWord{136, 11, 341}, + dictWord{134, 11, 78}, + dictWord{9, 0, 601}, + dictWord{9, 0, 619}, + dictWord{10, 0, 505}, + dictWord{10, 0, 732}, + dictWord{11, 0, 355}, + dictWord{140, 0, 139}, + dictWord{134, 0, 292}, + dictWord{139, 0, 174}, + dictWord{5, 0, 177}, + dictWord{6, 0, 616}, + dictWord{7, 0, 827}, + dictWord{ + 9, + 0, + 525, + }, + dictWord{138, 0, 656}, + dictWord{10, 0, 31}, + dictWord{6, 10, 215}, + dictWord{7, 10, 1028}, + dictWord{7, 10, 1473}, + dictWord{7, 10, 1721}, + dictWord{ + 9, + 10, + 424, + }, + dictWord{138, 10, 779}, + dictWord{135, 10, 584}, + dictWord{136, 11, 293}, + dictWord{134, 0, 685}, + dictWord{135, 11, 1868}, + dictWord{ + 133, + 11, + 460, + }, + dictWord{7, 0, 647}, + dictWord{6, 10, 67}, + dictWord{7, 10, 1630}, + dictWord{9, 10, 354}, + dictWord{9, 10, 675}, + dictWord{10, 10, 830}, + dictWord{ + 14, + 10, + 80, + }, + dictWord{145, 10, 80}, + dictWord{4, 0, 161}, + dictWord{133, 0, 631}, + dictWord{6, 10, 141}, + dictWord{7, 10, 225}, + dictWord{9, 10, 59}, + dictWord{9, 10, 607}, + dictWord{10, 10, 312}, + dictWord{11, 10, 687}, + dictWord{12, 10, 555}, + dictWord{13, 10, 373}, + dictWord{13, 10, 494}, + dictWord{148, 10, 58}, + dictWord{ + 7, + 11, + 965, + }, + dictWord{7, 11, 1460}, + dictWord{135, 11, 1604}, + dictWord{136, 10, 783}, + dictWord{134, 11, 388}, + dictWord{6, 0, 722}, + dictWord{6, 0, 1267}, + dictWord{ + 4, + 11, + 511, + }, + dictWord{9, 11, 333}, + dictWord{9, 11, 379}, + dictWord{10, 11, 602}, + dictWord{11, 11, 441}, + dictWord{11, 11, 723}, + dictWord{11, 11, 976}, + dictWord{140, 11, 357}, + dictWord{134, 0, 1797}, + dictWord{135, 0, 1684}, + dictWord{9, 0, 469}, + dictWord{9, 0, 709}, + dictWord{12, 0, 512}, + dictWord{14, 0, 65}, + dictWord{17, 0, 12}, + dictWord{5, 11, 938}, + dictWord{136, 11, 707}, + dictWord{7, 0, 1230}, + dictWord{136, 0, 531}, + dictWord{10, 0, 229}, + dictWord{11, 0, 73}, + dictWord{ + 11, + 0, + 376, + }, + dictWord{139, 0, 433}, + dictWord{12, 0, 268}, + dictWord{12, 0, 640}, + dictWord{142, 0, 119}, + dictWord{7, 10, 430}, + dictWord{139, 10, 46}, + dictWord{ + 6, + 0, + 558, + }, + dictWord{7, 0, 651}, + dictWord{8, 0, 421}, + dictWord{9, 0, 0}, + dictWord{10, 0, 34}, + dictWord{139, 0, 1008}, + dictWord{6, 0, 106}, + dictWord{7, 0, 1786}, + dictWord{7, 0, 1821}, + dictWord{9, 0, 102}, + dictWord{9, 0, 763}, + dictWord{5, 10, 602}, + dictWord{7, 10, 2018}, + dictWord{137, 10, 418}, + dictWord{5, 0, 65}, + dictWord{ + 6, + 0, + 416, + }, + dictWord{7, 0, 1720}, + dictWord{7, 0, 1924}, + dictWord{10, 0, 109}, + dictWord{11, 0, 14}, + dictWord{11, 0, 70}, + dictWord{11, 0, 569}, + dictWord{11, 0, 735}, + dictWord{15, 0, 153}, + dictWord{20, 0, 80}, + dictWord{136, 10, 677}, + dictWord{135, 11, 1625}, + dictWord{137, 11, 772}, + dictWord{136, 0, 595}, + dictWord{ + 6, + 11, + 469, + }, + dictWord{7, 11, 1709}, + dictWord{138, 11, 515}, + dictWord{7, 0, 1832}, + dictWord{138, 0, 374}, + dictWord{9, 0, 106}, + dictWord{9, 0, 163}, + dictWord{ + 9, + 0, + 296, + }, + dictWord{10, 0, 167}, + dictWord{10, 0, 172}, + dictWord{10, 0, 777}, + dictWord{139, 0, 16}, + dictWord{6, 0, 6}, + dictWord{7, 0, 81}, + dictWord{7, 0, 771}, + dictWord{ + 7, + 0, + 1731, + }, + dictWord{9, 0, 405}, + dictWord{138, 0, 421}, + dictWord{4, 11, 500}, + dictWord{135, 11, 938}, + dictWord{5, 11, 68}, + dictWord{134, 11, 383}, + dictWord{ + 5, + 0, + 881, + }, + dictWord{133, 0, 885}, + dictWord{6, 0, 854}, + dictWord{6, 0, 1132}, + dictWord{6, 0, 1495}, + dictWord{6, 0, 1526}, + dictWord{6, 0, 1533}, + dictWord{ + 134, + 0, + 1577, + }, + dictWord{4, 11, 337}, + dictWord{6, 11, 353}, + dictWord{7, 11, 1934}, + dictWord{8, 11, 488}, + dictWord{137, 11, 429}, + dictWord{7, 11, 236}, + dictWord{ + 7, + 11, + 1795, + }, + dictWord{8, 11, 259}, + dictWord{9, 11, 135}, + dictWord{9, 11, 177}, + dictWord{10, 11, 825}, + dictWord{11, 11, 115}, + dictWord{11, 11, 370}, + dictWord{ + 11, + 11, + 405, + }, + dictWord{11, 11, 604}, + dictWord{12, 11, 10}, + dictWord{12, 11, 667}, + dictWord{12, 11, 669}, + dictWord{13, 11, 76}, + dictWord{14, 11, 310}, + dictWord{15, 11, 76}, + dictWord{15, 11, 147}, + dictWord{148, 11, 23}, + dictWord{5, 0, 142}, + dictWord{134, 0, 546}, + dictWord{4, 11, 15}, + dictWord{5, 11, 22}, + dictWord{ + 6, + 11, + 244, + }, + dictWord{7, 11, 40}, + dictWord{7, 11, 200}, + dictWord{7, 11, 906}, + dictWord{7, 11, 1199}, + dictWord{9, 11, 616}, + dictWord{10, 11, 716}, + dictWord{ + 11, + 11, + 635, + }, + dictWord{11, 11, 801}, + dictWord{140, 11, 458}, + dictWord{5, 0, 466}, + dictWord{11, 0, 571}, + dictWord{12, 0, 198}, + dictWord{13, 0, 283}, + dictWord{ + 14, + 0, + 186, + }, + dictWord{15, 0, 21}, + dictWord{15, 0, 103}, + dictWord{135, 10, 329}, + dictWord{4, 0, 185}, + dictWord{5, 0, 257}, + dictWord{5, 0, 839}, + dictWord{5, 0, 936}, + dictWord{9, 0, 399}, + dictWord{10, 0, 258}, + dictWord{10, 0, 395}, + dictWord{10, 0, 734}, + dictWord{11, 0, 1014}, + dictWord{12, 0, 23}, + dictWord{13, 0, 350}, + dictWord{ + 14, + 0, + 150, + }, + dictWord{19, 0, 6}, + dictWord{135, 11, 1735}, + dictWord{12, 11, 36}, + dictWord{141, 11, 337}, + dictWord{5, 11, 598}, + dictWord{7, 11, 791}, + dictWord{ + 8, + 11, + 108, + }, + dictWord{137, 11, 123}, + dictWord{132, 10, 469}, + dictWord{7, 0, 404}, + dictWord{7, 0, 1377}, + dictWord{7, 0, 1430}, + dictWord{7, 0, 2017}, + dictWord{ + 8, + 0, + 149, + }, + dictWord{8, 0, 239}, + dictWord{8, 0, 512}, + dictWord{8, 0, 793}, + dictWord{8, 0, 818}, + dictWord{9, 0, 474}, + dictWord{9, 0, 595}, + dictWord{10, 0, 122}, + dictWord{10, 0, 565}, + dictWord{10, 0, 649}, + dictWord{10, 0, 783}, + dictWord{11, 0, 239}, + dictWord{11, 0, 295}, + dictWord{11, 0, 447}, + dictWord{11, 0, 528}, + dictWord{ + 11, + 0, + 639, + }, + dictWord{11, 0, 800}, + dictWord{12, 0, 25}, + dictWord{12, 0, 77}, + dictWord{12, 0, 157}, + dictWord{12, 0, 256}, + dictWord{12, 0, 316}, + dictWord{12, 0, 390}, + dictWord{12, 0, 391}, + dictWord{12, 0, 395}, + dictWord{12, 0, 478}, + dictWord{12, 0, 503}, + dictWord{12, 0, 592}, + dictWord{12, 0, 680}, + dictWord{13, 0, 50}, + dictWord{13, 0, 53}, + dictWord{13, 0, 132}, + dictWord{13, 0, 198}, + dictWord{13, 0, 322}, + dictWord{13, 0, 415}, + dictWord{13, 0, 511}, + dictWord{14, 0, 71}, + dictWord{ + 14, + 0, + 395, + }, + dictWord{15, 0, 71}, + dictWord{15, 0, 136}, + dictWord{17, 0, 123}, + dictWord{18, 0, 93}, + dictWord{147, 0, 58}, + dictWord{136, 0, 712}, + dictWord{ + 134, + 10, + 1743, + }, + dictWord{5, 10, 929}, + dictWord{6, 10, 340}, + dictWord{8, 10, 376}, + dictWord{136, 10, 807}, + dictWord{6, 0, 1848}, + dictWord{8, 0, 860}, + dictWord{ + 10, + 0, + 856, + }, + dictWord{10, 0, 859}, + dictWord{10, 0, 925}, + dictWord{10, 0, 941}, + dictWord{140, 0, 762}, + dictWord{6, 0, 629}, + dictWord{6, 0, 906}, + dictWord{9, 0, 810}, + dictWord{140, 0, 652}, + dictWord{5, 10, 218}, + dictWord{7, 10, 1610}, + dictWord{138, 10, 83}, + dictWord{7, 10, 1512}, + dictWord{135, 10, 1794}, + dictWord{ + 4, + 0, + 377, + }, + dictWord{24, 0, 13}, + dictWord{4, 11, 155}, + dictWord{7, 11, 1689}, + dictWord{11, 10, 0}, + dictWord{144, 10, 78}, + dictWord{4, 11, 164}, + dictWord{5, 11, 151}, + dictWord{5, 11, 730}, + dictWord{5, 11, 741}, + dictWord{7, 11, 498}, + dictWord{7, 11, 870}, + dictWord{7, 11, 1542}, + dictWord{12, 11, 213}, + dictWord{14, 11, 36}, + dictWord{14, 11, 391}, + dictWord{17, 11, 111}, + dictWord{18, 11, 6}, + dictWord{18, 11, 46}, + dictWord{18, 11, 151}, + dictWord{19, 11, 36}, + dictWord{20, 11, 32}, + dictWord{20, 11, 56}, + dictWord{20, 11, 69}, + dictWord{20, 11, 102}, + dictWord{21, 11, 4}, + dictWord{22, 11, 8}, + dictWord{22, 11, 10}, + dictWord{22, 11, 14}, + dictWord{ + 150, + 11, + 31, + }, + dictWord{7, 0, 1842}, + dictWord{133, 10, 571}, + dictWord{4, 10, 455}, + dictWord{4, 11, 624}, + dictWord{135, 11, 1752}, + dictWord{134, 0, 1501}, + dictWord{4, 11, 492}, + dictWord{5, 11, 451}, + dictWord{6, 10, 161}, + dictWord{7, 10, 372}, + dictWord{137, 10, 597}, + dictWord{132, 10, 349}, + dictWord{4, 0, 180}, + dictWord{135, 0, 1906}, + dictWord{135, 11, 835}, + dictWord{141, 11, 70}, + dictWord{132, 0, 491}, + dictWord{137, 10, 751}, + dictWord{6, 10, 432}, + dictWord{ + 139, + 10, + 322, + }, + dictWord{4, 0, 171}, + dictWord{138, 0, 234}, + dictWord{6, 11, 113}, + dictWord{135, 11, 436}, + dictWord{4, 0, 586}, + dictWord{7, 0, 1186}, + dictWord{ + 138, + 0, + 631, + }, + dictWord{5, 10, 468}, + dictWord{10, 10, 325}, + dictWord{11, 10, 856}, + dictWord{12, 10, 345}, + dictWord{143, 10, 104}, + dictWord{5, 10, 223}, + dictWord{10, 11, 592}, + dictWord{10, 11, 753}, + dictWord{12, 11, 317}, + dictWord{12, 11, 355}, + dictWord{12, 11, 465}, + dictWord{12, 11, 469}, + dictWord{ + 12, + 11, + 560, + }, + dictWord{12, 11, 578}, + dictWord{141, 11, 243}, + dictWord{132, 10, 566}, + dictWord{135, 11, 520}, + dictWord{4, 10, 59}, + dictWord{135, 10, 1394}, + dictWord{6, 10, 436}, + dictWord{139, 10, 481}, + dictWord{9, 0, 931}, + dictWord{10, 0, 334}, + dictWord{20, 0, 71}, + dictWord{4, 10, 48}, + dictWord{5, 10, 271}, + dictWord{ + 7, + 10, + 953, + }, + dictWord{135, 11, 1878}, + dictWord{11, 0, 170}, + dictWord{5, 10, 610}, + dictWord{136, 10, 457}, + dictWord{133, 10, 755}, + dictWord{6, 0, 1587}, + dictWord{135, 10, 1217}, + dictWord{4, 10, 197}, + dictWord{149, 11, 26}, + dictWord{133, 11, 585}, + dictWord{137, 11, 521}, + dictWord{133, 0, 765}, + dictWord{ + 133, + 10, + 217, + }, + dictWord{139, 11, 586}, + dictWord{133, 0, 424}, + dictWord{9, 11, 752}, + dictWord{12, 11, 610}, + dictWord{13, 11, 431}, + dictWord{16, 11, 59}, + dictWord{146, 11, 109}, + dictWord{136, 0, 714}, + dictWord{7, 0, 685}, + dictWord{132, 11, 307}, + dictWord{9, 0, 420}, + dictWord{10, 0, 269}, + dictWord{10, 0, 285}, + dictWord{10, 0, 576}, + dictWord{11, 0, 397}, + dictWord{13, 0, 175}, + dictWord{145, 0, 90}, + dictWord{132, 0, 429}, + dictWord{133, 11, 964}, + dictWord{9, 11, 463}, + dictWord{138, 11, 595}, + dictWord{7, 0, 18}, + dictWord{7, 0, 699}, + dictWord{7, 0, 1966}, + dictWord{8, 0, 752}, + dictWord{9, 0, 273}, + dictWord{9, 0, 412}, + dictWord{ + 9, + 0, + 703, + }, + dictWord{10, 0, 71}, + dictWord{10, 0, 427}, + dictWord{138, 0, 508}, + dictWord{4, 10, 165}, + dictWord{7, 10, 1398}, + dictWord{135, 10, 1829}, + dictWord{ + 4, + 0, + 53, + }, + dictWord{5, 0, 186}, + dictWord{7, 0, 752}, + dictWord{7, 0, 828}, + dictWord{142, 0, 116}, + dictWord{8, 0, 575}, + dictWord{10, 0, 289}, + dictWord{139, 0, 319}, + dictWord{132, 0, 675}, + dictWord{134, 0, 1424}, + dictWord{4, 11, 75}, + dictWord{5, 11, 180}, + dictWord{6, 11, 500}, + dictWord{7, 11, 58}, + dictWord{7, 11, 710}, + dictWord{138, 11, 645}, + dictWord{133, 11, 649}, + dictWord{6, 11, 276}, + dictWord{7, 11, 282}, + dictWord{7, 11, 879}, + dictWord{7, 11, 924}, + dictWord{8, 11, 459}, + dictWord{9, 11, 599}, + dictWord{9, 11, 754}, + dictWord{11, 11, 574}, + dictWord{12, 11, 128}, + dictWord{12, 11, 494}, + dictWord{13, 11, 52}, + dictWord{13, 11, 301}, + dictWord{15, 11, 30}, + dictWord{143, 11, 132}, + dictWord{6, 0, 647}, + dictWord{134, 0, 1095}, + dictWord{5, 10, 9}, + dictWord{7, 10, 297}, + dictWord{7, 10, 966}, + dictWord{140, 10, 306}, + dictWord{132, 11, 200}, + dictWord{134, 0, 1334}, + dictWord{5, 10, 146}, + dictWord{6, 10, 411}, + dictWord{138, 10, 721}, + dictWord{ + 6, + 0, + 209, + }, + dictWord{6, 0, 1141}, + dictWord{6, 0, 1288}, + dictWord{8, 0, 468}, + dictWord{9, 0, 210}, + dictWord{11, 0, 36}, + dictWord{12, 0, 28}, + dictWord{12, 0, 630}, + dictWord{13, 0, 21}, + dictWord{13, 0, 349}, + dictWord{14, 0, 7}, + dictWord{145, 0, 13}, + dictWord{6, 10, 177}, + dictWord{135, 10, 467}, + dictWord{4, 0, 342}, + dictWord{ + 135, + 0, + 1179, + }, + dictWord{10, 11, 454}, + dictWord{140, 11, 324}, + dictWord{4, 0, 928}, + dictWord{133, 0, 910}, + dictWord{7, 0, 1838}, + dictWord{6, 11, 225}, + dictWord{ + 137, + 11, + 211, + }, + dictWord{16, 0, 101}, + dictWord{20, 0, 115}, + dictWord{20, 0, 118}, + dictWord{148, 0, 122}, + dictWord{4, 0, 496}, + dictWord{135, 0, 856}, + dictWord{ + 4, + 0, + 318, + }, + dictWord{11, 0, 654}, + dictWord{7, 11, 718}, + dictWord{139, 11, 102}, + dictWord{8, 11, 58}, + dictWord{9, 11, 724}, + dictWord{11, 11, 809}, + dictWord{ + 13, + 11, + 113, + }, + dictWord{145, 11, 72}, + dictWord{5, 10, 200}, + dictWord{6, 11, 345}, + dictWord{135, 11, 1247}, + dictWord{8, 11, 767}, + dictWord{8, 11, 803}, + dictWord{ + 9, + 11, + 301, + }, + dictWord{137, 11, 903}, + dictWord{7, 0, 915}, + dictWord{8, 0, 247}, + dictWord{19, 0, 0}, + dictWord{7, 11, 1949}, + dictWord{136, 11, 674}, + dictWord{ + 4, + 0, + 202, + }, + dictWord{5, 0, 382}, + dictWord{6, 0, 454}, + dictWord{7, 0, 936}, + dictWord{7, 0, 1803}, + dictWord{8, 0, 758}, + dictWord{9, 0, 375}, + dictWord{9, 0, 895}, + dictWord{ + 10, + 0, + 743, + }, + dictWord{10, 0, 792}, + dictWord{11, 0, 978}, + dictWord{11, 0, 1012}, + dictWord{142, 0, 109}, + dictWord{7, 0, 1150}, + dictWord{7, 0, 1425}, + dictWord{ + 7, + 0, + 1453, + }, + dictWord{140, 0, 513}, + dictWord{134, 11, 259}, + dictWord{138, 0, 791}, + dictWord{11, 0, 821}, + dictWord{12, 0, 110}, + dictWord{12, 0, 153}, + dictWord{ + 18, + 0, + 41, + }, + dictWord{150, 0, 19}, + dictWord{134, 10, 481}, + dictWord{132, 0, 796}, + dictWord{6, 0, 445}, + dictWord{9, 0, 909}, + dictWord{136, 11, 254}, + dictWord{ + 10, + 0, + 776, + }, + dictWord{13, 0, 345}, + dictWord{142, 0, 425}, + dictWord{4, 10, 84}, + dictWord{7, 10, 1482}, + dictWord{10, 10, 76}, + dictWord{138, 10, 142}, + dictWord{ + 135, + 11, + 742, + }, + dictWord{6, 0, 578}, + dictWord{133, 10, 1015}, + dictWord{6, 0, 1387}, + dictWord{4, 10, 315}, + dictWord{5, 10, 507}, + dictWord{135, 10, 1370}, + dictWord{4, 0, 438}, + dictWord{133, 0, 555}, + dictWord{136, 0, 766}, + dictWord{133, 11, 248}, + dictWord{134, 10, 1722}, + dictWord{4, 11, 116}, + dictWord{5, 11, 95}, + dictWord{5, 11, 445}, + dictWord{7, 11, 1688}, + dictWord{8, 11, 29}, + dictWord{9, 11, 272}, + dictWord{11, 11, 509}, + dictWord{139, 11, 915}, + dictWord{135, 0, 541}, + dictWord{133, 11, 543}, + dictWord{8, 10, 222}, + dictWord{8, 10, 476}, + dictWord{9, 10, 238}, + dictWord{11, 10, 516}, + dictWord{11, 10, 575}, + dictWord{ + 15, + 10, + 109, + }, + dictWord{146, 10, 100}, + dictWord{6, 0, 880}, + dictWord{134, 0, 1191}, + dictWord{5, 11, 181}, + dictWord{136, 11, 41}, + dictWord{134, 0, 1506}, + dictWord{132, 11, 681}, + dictWord{7, 11, 25}, + dictWord{8, 11, 202}, + dictWord{138, 11, 536}, + dictWord{139, 0, 983}, + dictWord{137, 0, 768}, + dictWord{132, 0, 584}, + dictWord{9, 11, 423}, + dictWord{140, 11, 89}, + dictWord{8, 11, 113}, + dictWord{9, 11, 877}, + dictWord{10, 11, 554}, + dictWord{11, 11, 83}, + dictWord{12, 11, 136}, + dictWord{147, 11, 109}, + dictWord{7, 10, 706}, + dictWord{7, 10, 1058}, + dictWord{138, 10, 538}, + dictWord{133, 11, 976}, + dictWord{4, 11, 206}, + dictWord{ + 135, + 11, + 746, + }, + dictWord{136, 11, 526}, + dictWord{140, 0, 737}, + dictWord{11, 10, 92}, + dictWord{11, 10, 196}, + dictWord{11, 10, 409}, + dictWord{11, 10, 450}, + dictWord{11, 10, 666}, + dictWord{11, 10, 777}, + dictWord{12, 10, 262}, + dictWord{13, 10, 385}, + dictWord{13, 10, 393}, + dictWord{15, 10, 115}, + dictWord{ + 16, + 10, + 45, + }, + dictWord{145, 10, 82}, + dictWord{4, 0, 226}, + dictWord{4, 0, 326}, + dictWord{7, 0, 1770}, + dictWord{4, 11, 319}, + dictWord{5, 11, 699}, + dictWord{138, 11, 673}, + dictWord{6, 10, 40}, + dictWord{135, 10, 1781}, + dictWord{5, 0, 426}, + dictWord{8, 0, 30}, + dictWord{9, 0, 2}, + dictWord{11, 0, 549}, + dictWord{147, 0, 122}, + dictWord{ + 6, + 0, + 1161, + }, + dictWord{134, 0, 1329}, + dictWord{138, 10, 97}, + dictWord{6, 10, 423}, + dictWord{7, 10, 665}, + dictWord{135, 10, 1210}, + dictWord{7, 11, 13}, + dictWord{ + 8, + 11, + 226, + }, + dictWord{10, 11, 537}, + dictWord{11, 11, 570}, + dictWord{11, 11, 605}, + dictWord{11, 11, 799}, + dictWord{11, 11, 804}, + dictWord{12, 11, 85}, + dictWord{12, 11, 516}, + dictWord{12, 11, 623}, + dictWord{13, 11, 112}, + dictWord{13, 11, 361}, + dictWord{14, 11, 77}, + dictWord{14, 11, 78}, + dictWord{17, 11, 28}, + dictWord{147, 11, 110}, + dictWord{132, 11, 769}, + dictWord{132, 11, 551}, + dictWord{132, 11, 728}, + dictWord{147, 0, 117}, + dictWord{9, 11, 57}, + dictWord{ + 9, + 11, + 459, + }, + dictWord{10, 11, 425}, + dictWord{11, 11, 119}, + dictWord{12, 11, 184}, + dictWord{12, 11, 371}, + dictWord{13, 11, 358}, + dictWord{145, 11, 51}, + dictWord{ + 5, + 11, + 188, + }, + dictWord{5, 11, 814}, + dictWord{8, 11, 10}, + dictWord{9, 11, 421}, + dictWord{9, 11, 729}, + dictWord{10, 11, 609}, + dictWord{139, 11, 689}, + dictWord{134, 11, 624}, + dictWord{135, 11, 298}, + dictWord{135, 0, 462}, + dictWord{4, 0, 345}, + dictWord{139, 10, 624}, + dictWord{136, 10, 574}, + dictWord{ + 4, + 0, + 385, + }, + dictWord{7, 0, 265}, + dictWord{135, 0, 587}, + dictWord{6, 0, 808}, + dictWord{132, 11, 528}, + dictWord{133, 0, 398}, + dictWord{132, 10, 354}, + dictWord{ + 4, + 0, + 347, + }, + dictWord{5, 0, 423}, + dictWord{5, 0, 996}, + dictWord{135, 0, 1329}, + dictWord{135, 10, 1558}, + dictWord{7, 0, 1259}, + dictWord{9, 0, 125}, + dictWord{ + 139, + 0, + 65, + }, + dictWord{5, 0, 136}, + dictWord{6, 0, 136}, + dictWord{136, 0, 644}, + dictWord{5, 11, 104}, + dictWord{6, 11, 173}, + dictWord{135, 11, 1631}, + dictWord{ + 135, + 0, + 469, + }, + dictWord{133, 10, 830}, + dictWord{4, 0, 278}, + dictWord{5, 0, 465}, + dictWord{135, 0, 1367}, + dictWord{7, 11, 810}, + dictWord{8, 11, 138}, + dictWord{ + 8, + 11, + 342, + }, + dictWord{9, 11, 84}, + dictWord{10, 11, 193}, + dictWord{11, 11, 883}, + dictWord{140, 11, 359}, + dictWord{5, 10, 496}, + dictWord{135, 10, 203}, + dictWord{ + 4, + 0, + 433, + }, + dictWord{133, 0, 719}, + dictWord{6, 11, 95}, + dictWord{134, 10, 547}, + dictWord{5, 10, 88}, + dictWord{137, 10, 239}, + dictWord{6, 11, 406}, + dictWord{ + 10, + 11, + 409, + }, + dictWord{10, 11, 447}, + dictWord{11, 11, 44}, + dictWord{140, 11, 100}, + dictWord{134, 0, 1423}, + dictWord{7, 10, 650}, + dictWord{135, 10, 1310}, + dictWord{134, 0, 749}, + dictWord{135, 11, 1243}, + dictWord{135, 0, 1363}, + dictWord{6, 0, 381}, + dictWord{7, 0, 645}, + dictWord{7, 0, 694}, + dictWord{8, 0, 546}, + dictWord{7, 10, 1076}, + dictWord{9, 10, 80}, + dictWord{11, 10, 78}, + dictWord{11, 10, 421}, + dictWord{11, 10, 534}, + dictWord{140, 10, 545}, + dictWord{ + 134, + 11, + 1636, + }, + dictWord{135, 11, 1344}, + dictWord{12, 0, 277}, + dictWord{7, 10, 274}, + dictWord{11, 10, 479}, + dictWord{139, 10, 507}, + dictWord{6, 0, 705}, + dictWord{ + 6, + 0, + 783, + }, + dictWord{6, 0, 1275}, + dictWord{6, 0, 1481}, + dictWord{4, 11, 282}, + dictWord{7, 11, 1034}, + dictWord{11, 11, 398}, + dictWord{11, 11, 634}, + dictWord{ + 12, + 11, + 1, + }, + dictWord{12, 11, 79}, + dictWord{12, 11, 544}, + dictWord{14, 11, 237}, + dictWord{17, 11, 10}, + dictWord{146, 11, 20}, + dictWord{134, 0, 453}, + dictWord{ + 4, + 0, + 555, + }, + dictWord{8, 0, 536}, + dictWord{10, 0, 288}, + dictWord{11, 0, 1005}, + dictWord{4, 10, 497}, + dictWord{135, 10, 1584}, + dictWord{5, 11, 118}, + dictWord{ + 5, + 11, + 499, + }, + dictWord{6, 11, 476}, + dictWord{7, 11, 600}, + dictWord{7, 11, 888}, + dictWord{135, 11, 1096}, + dictWord{138, 0, 987}, + dictWord{7, 0, 1107}, + dictWord{ + 7, + 10, + 261, + }, + dictWord{7, 10, 1115}, + dictWord{7, 10, 1354}, + dictWord{7, 10, 1588}, + dictWord{7, 10, 1705}, + dictWord{7, 10, 1902}, + dictWord{9, 10, 465}, + dictWord{10, 10, 248}, + dictWord{10, 10, 349}, + dictWord{10, 10, 647}, + dictWord{11, 10, 527}, + dictWord{11, 10, 660}, + dictWord{11, 10, 669}, + dictWord{ + 12, + 10, + 529, + }, + dictWord{141, 10, 305}, + dictWord{7, 11, 296}, + dictWord{7, 11, 596}, + dictWord{8, 11, 560}, + dictWord{8, 11, 586}, + dictWord{9, 11, 612}, + dictWord{ + 11, + 11, + 100, + }, + dictWord{11, 11, 304}, + dictWord{12, 11, 46}, + dictWord{13, 11, 89}, + dictWord{14, 11, 112}, + dictWord{145, 11, 122}, + dictWord{9, 0, 370}, + dictWord{ + 138, + 0, + 90, + }, + dictWord{136, 10, 13}, + dictWord{132, 0, 860}, + dictWord{7, 10, 642}, + dictWord{8, 10, 250}, + dictWord{11, 10, 123}, + dictWord{11, 10, 137}, + dictWord{ + 13, + 10, + 48, + }, + dictWord{142, 10, 95}, + dictWord{135, 10, 1429}, + dictWord{137, 11, 321}, + dictWord{132, 0, 257}, + dictWord{135, 0, 2031}, + dictWord{7, 0, 1768}, + dictWord{7, 11, 1599}, + dictWord{7, 11, 1723}, + dictWord{8, 11, 79}, + dictWord{8, 11, 106}, + dictWord{8, 11, 190}, + dictWord{8, 11, 302}, + dictWord{8, 11, 383}, + dictWord{9, 11, 119}, + dictWord{9, 11, 233}, + dictWord{9, 11, 298}, + dictWord{9, 11, 419}, + dictWord{9, 11, 471}, + dictWord{10, 11, 181}, + dictWord{10, 11, 406}, + dictWord{11, 11, 57}, + dictWord{11, 11, 85}, + dictWord{11, 11, 120}, + dictWord{11, 11, 177}, + dictWord{11, 11, 296}, + dictWord{11, 11, 382}, + dictWord{11, 11, 454}, + dictWord{11, 11, 758}, + dictWord{11, 11, 999}, + dictWord{12, 11, 27}, + dictWord{12, 11, 98}, + dictWord{12, 11, 131}, + dictWord{12, 11, 245}, + dictWord{ + 12, + 11, + 312, + }, + dictWord{12, 11, 446}, + dictWord{12, 11, 454}, + dictWord{13, 11, 25}, + dictWord{13, 11, 98}, + dictWord{13, 11, 426}, + dictWord{13, 11, 508}, + dictWord{ + 14, + 11, + 6, + }, + dictWord{14, 11, 163}, + dictWord{14, 11, 272}, + dictWord{14, 11, 277}, + dictWord{14, 11, 370}, + dictWord{15, 11, 95}, + dictWord{15, 11, 138}, + dictWord{ + 15, + 11, + 167, + }, + dictWord{17, 11, 18}, + dictWord{17, 11, 38}, + dictWord{20, 11, 96}, + dictWord{149, 11, 32}, + dictWord{5, 11, 722}, + dictWord{134, 11, 1759}, + dictWord{145, 11, 16}, + dictWord{6, 0, 1071}, + dictWord{134, 0, 1561}, + dictWord{10, 10, 545}, + dictWord{140, 10, 301}, + dictWord{6, 0, 83}, + dictWord{6, 0, 1733}, + dictWord{135, 0, 1389}, + dictWord{4, 0, 835}, + dictWord{135, 0, 1818}, + dictWord{133, 11, 258}, + dictWord{4, 10, 904}, + dictWord{133, 10, 794}, + dictWord{ + 134, + 0, + 2006, + }, + dictWord{5, 11, 30}, + dictWord{7, 11, 495}, + dictWord{8, 11, 134}, + dictWord{9, 11, 788}, + dictWord{140, 11, 438}, + dictWord{135, 11, 2004}, + dictWord{ + 137, + 0, + 696, + }, + dictWord{5, 11, 50}, + dictWord{6, 11, 439}, + dictWord{7, 11, 780}, + dictWord{135, 11, 1040}, + dictWord{7, 11, 772}, + dictWord{7, 11, 1104}, + dictWord{ + 7, + 11, + 1647, + }, + dictWord{11, 11, 269}, + dictWord{11, 11, 539}, + dictWord{11, 11, 607}, + dictWord{11, 11, 627}, + dictWord{11, 11, 706}, + dictWord{11, 11, 975}, + dictWord{12, 11, 248}, + dictWord{12, 11, 311}, + dictWord{12, 11, 434}, + dictWord{12, 11, 600}, + dictWord{12, 11, 622}, + dictWord{13, 11, 297}, + dictWord{ + 13, + 11, + 367, + }, + dictWord{13, 11, 485}, + dictWord{14, 11, 69}, + dictWord{14, 11, 409}, + dictWord{143, 11, 108}, + dictWord{5, 11, 1}, + dictWord{6, 11, 81}, + dictWord{ + 138, + 11, + 520, + }, + dictWord{7, 0, 1718}, + dictWord{9, 0, 95}, + dictWord{9, 0, 274}, + dictWord{10, 0, 279}, + dictWord{10, 0, 317}, + dictWord{10, 0, 420}, + dictWord{11, 0, 303}, + dictWord{11, 0, 808}, + dictWord{12, 0, 134}, + dictWord{12, 0, 367}, + dictWord{13, 0, 149}, + dictWord{13, 0, 347}, + dictWord{14, 0, 349}, + dictWord{14, 0, 406}, + dictWord{ + 18, + 0, + 22, + }, + dictWord{18, 0, 89}, + dictWord{18, 0, 122}, + dictWord{147, 0, 47}, + dictWord{5, 11, 482}, + dictWord{8, 11, 98}, + dictWord{9, 11, 172}, + dictWord{10, 11, 222}, + dictWord{10, 11, 700}, + dictWord{10, 11, 822}, + dictWord{11, 11, 302}, + dictWord{11, 11, 778}, + dictWord{12, 11, 50}, + dictWord{12, 11, 127}, + dictWord{ + 12, + 11, + 396, + }, + dictWord{13, 11, 62}, + dictWord{13, 11, 328}, + dictWord{14, 11, 122}, + dictWord{147, 11, 72}, + dictWord{7, 10, 386}, + dictWord{138, 10, 713}, + dictWord{ + 6, + 10, + 7, + }, + dictWord{6, 10, 35}, + dictWord{7, 10, 147}, + dictWord{7, 10, 1069}, + dictWord{7, 10, 1568}, + dictWord{7, 10, 1575}, + dictWord{7, 10, 1917}, + dictWord{ + 8, + 10, + 43, + }, + dictWord{8, 10, 208}, + dictWord{9, 10, 128}, + dictWord{9, 10, 866}, + dictWord{10, 10, 20}, + dictWord{11, 10, 981}, + dictWord{147, 10, 33}, + dictWord{ + 133, + 0, + 26, + }, + dictWord{132, 0, 550}, + dictWord{5, 11, 2}, + dictWord{7, 11, 1494}, + dictWord{136, 11, 589}, + dictWord{6, 11, 512}, + dictWord{7, 11, 797}, + dictWord{ + 8, + 11, + 253, + }, + dictWord{9, 11, 77}, + dictWord{10, 11, 1}, + dictWord{10, 11, 129}, + dictWord{10, 11, 225}, + dictWord{11, 11, 118}, + dictWord{11, 11, 226}, + dictWord{ + 11, + 11, + 251, + }, + dictWord{11, 11, 430}, + dictWord{11, 11, 701}, + dictWord{11, 11, 974}, + dictWord{11, 11, 982}, + dictWord{12, 11, 64}, + dictWord{12, 11, 260}, + dictWord{ + 12, + 11, + 488, + }, + dictWord{140, 11, 690}, + dictWord{7, 10, 893}, + dictWord{141, 10, 424}, + dictWord{134, 0, 901}, + dictWord{136, 0, 822}, + dictWord{4, 0, 902}, + dictWord{5, 0, 809}, + dictWord{134, 0, 122}, + dictWord{6, 0, 807}, + dictWord{134, 0, 1366}, + dictWord{7, 0, 262}, + dictWord{5, 11, 748}, + dictWord{134, 11, 553}, + dictWord{133, 0, 620}, + dictWord{4, 0, 34}, + dictWord{5, 0, 574}, + dictWord{7, 0, 279}, + dictWord{7, 0, 1624}, + dictWord{136, 0, 601}, + dictWord{9, 0, 170}, + dictWord{ + 6, + 10, + 322, + }, + dictWord{9, 10, 552}, + dictWord{11, 10, 274}, + dictWord{13, 10, 209}, + dictWord{13, 10, 499}, + dictWord{14, 10, 85}, + dictWord{15, 10, 126}, + dictWord{ + 145, + 10, + 70, + }, + dictWord{132, 0, 537}, + dictWord{4, 11, 12}, + dictWord{7, 11, 420}, + dictWord{7, 11, 522}, + dictWord{7, 11, 809}, + dictWord{8, 11, 797}, + dictWord{ + 141, + 11, + 88, + }, + dictWord{133, 0, 332}, + dictWord{8, 10, 83}, + dictWord{8, 10, 742}, + dictWord{8, 10, 817}, + dictWord{9, 10, 28}, + dictWord{9, 10, 29}, + dictWord{9, 10, 885}, + dictWord{10, 10, 387}, + dictWord{11, 10, 633}, + dictWord{11, 10, 740}, + dictWord{13, 10, 235}, + dictWord{13, 10, 254}, + dictWord{15, 10, 143}, + dictWord{ + 143, + 10, + 146, + }, + dictWord{6, 0, 1909}, + dictWord{9, 0, 964}, + dictWord{12, 0, 822}, + dictWord{12, 0, 854}, + dictWord{12, 0, 865}, + dictWord{12, 0, 910}, + dictWord{12, 0, 938}, + dictWord{15, 0, 169}, + dictWord{15, 0, 208}, + dictWord{15, 0, 211}, + dictWord{18, 0, 205}, + dictWord{18, 0, 206}, + dictWord{18, 0, 220}, + dictWord{18, 0, 223}, + dictWord{152, 0, 24}, + dictWord{140, 10, 49}, + dictWord{5, 11, 528}, + dictWord{135, 11, 1580}, + dictWord{6, 0, 261}, + dictWord{8, 0, 182}, + dictWord{139, 0, 943}, + dictWord{134, 0, 1721}, + dictWord{4, 0, 933}, + dictWord{133, 0, 880}, + dictWord{136, 11, 321}, + dictWord{5, 11, 266}, + dictWord{9, 11, 290}, + dictWord{9, 11, 364}, + dictWord{10, 11, 293}, + dictWord{11, 11, 606}, + dictWord{142, 11, 45}, + dictWord{6, 0, 1609}, + dictWord{4, 11, 50}, + dictWord{6, 11, 510}, + dictWord{6, 11, 594}, + dictWord{9, 11, 121}, + dictWord{10, 11, 49}, + dictWord{10, 11, 412}, + dictWord{139, 11, 834}, + dictWord{7, 0, 895}, + dictWord{136, 11, 748}, + dictWord{132, 11, 466}, + dictWord{4, 10, 110}, + dictWord{10, 10, 415}, + dictWord{10, 10, 597}, + dictWord{142, 10, 206}, + dictWord{133, 0, 812}, + dictWord{135, 11, 281}, + dictWord{ + 6, + 0, + 1890, + }, + dictWord{6, 0, 1902}, + dictWord{6, 0, 1916}, + dictWord{9, 0, 929}, + dictWord{9, 0, 942}, + dictWord{9, 0, 975}, + dictWord{9, 0, 984}, + dictWord{9, 0, 986}, + dictWord{ + 9, + 0, + 1011, + }, + dictWord{9, 0, 1019}, + dictWord{12, 0, 804}, + dictWord{12, 0, 851}, + dictWord{12, 0, 867}, + dictWord{12, 0, 916}, + dictWord{12, 0, 923}, + dictWord{ + 15, + 0, + 194, + }, + dictWord{15, 0, 204}, + dictWord{15, 0, 210}, + dictWord{15, 0, 222}, + dictWord{15, 0, 223}, + dictWord{15, 0, 229}, + dictWord{15, 0, 250}, + dictWord{ + 18, + 0, + 179, + }, + dictWord{18, 0, 186}, + dictWord{18, 0, 192}, + dictWord{7, 10, 205}, + dictWord{135, 10, 2000}, + dictWord{132, 11, 667}, + dictWord{135, 0, 778}, + dictWord{ + 4, + 0, + 137, + }, + dictWord{7, 0, 1178}, + dictWord{135, 0, 1520}, + dictWord{134, 0, 1314}, + dictWord{4, 11, 242}, + dictWord{134, 11, 333}, + dictWord{6, 0, 1661}, + dictWord{7, 0, 1975}, + dictWord{7, 0, 2009}, + dictWord{135, 0, 2011}, + dictWord{134, 0, 1591}, + dictWord{4, 10, 283}, + dictWord{135, 10, 1194}, + dictWord{ + 11, + 0, + 820, + }, + dictWord{150, 0, 51}, + dictWord{4, 11, 39}, + dictWord{5, 11, 36}, + dictWord{7, 11, 1843}, + dictWord{8, 11, 407}, + dictWord{11, 11, 144}, + dictWord{ + 140, + 11, + 523, + }, + dictWord{134, 10, 1720}, + dictWord{4, 11, 510}, + dictWord{7, 11, 29}, + dictWord{7, 11, 66}, + dictWord{7, 11, 1980}, + dictWord{10, 11, 487}, + dictWord{ + 10, + 11, + 809, + }, + dictWord{146, 11, 9}, + dictWord{5, 0, 89}, + dictWord{7, 0, 1915}, + dictWord{9, 0, 185}, + dictWord{9, 0, 235}, + dictWord{10, 0, 64}, + dictWord{10, 0, 270}, + dictWord{10, 0, 403}, + dictWord{10, 0, 469}, + dictWord{10, 0, 529}, + dictWord{10, 0, 590}, + dictWord{11, 0, 140}, + dictWord{11, 0, 860}, + dictWord{13, 0, 1}, + dictWord{ + 13, + 0, + 422, + }, + dictWord{14, 0, 341}, + dictWord{14, 0, 364}, + dictWord{17, 0, 93}, + dictWord{18, 0, 113}, + dictWord{19, 0, 97}, + dictWord{147, 0, 113}, + dictWord{133, 0, 695}, + dictWord{6, 0, 987}, + dictWord{134, 0, 1160}, + dictWord{5, 0, 6}, + dictWord{6, 0, 183}, + dictWord{7, 0, 680}, + dictWord{7, 0, 978}, + dictWord{7, 0, 1013}, + dictWord{ + 7, + 0, + 1055, + }, + dictWord{12, 0, 230}, + dictWord{13, 0, 172}, + dictWord{146, 0, 29}, + dictWord{134, 11, 570}, + dictWord{132, 11, 787}, + dictWord{134, 11, 518}, + dictWord{ + 6, + 0, + 29, + }, + dictWord{139, 0, 63}, + dictWord{132, 11, 516}, + dictWord{136, 11, 821}, + dictWord{132, 0, 311}, + dictWord{134, 0, 1740}, + dictWord{7, 0, 170}, + dictWord{8, 0, 90}, + dictWord{8, 0, 177}, + dictWord{8, 0, 415}, + dictWord{11, 0, 714}, + dictWord{14, 0, 281}, + dictWord{136, 10, 735}, + dictWord{134, 0, 1961}, + dictWord{ + 135, + 11, + 1405, + }, + dictWord{4, 11, 10}, + dictWord{7, 11, 917}, + dictWord{139, 11, 786}, + dictWord{5, 10, 132}, + dictWord{9, 10, 486}, + dictWord{9, 10, 715}, + dictWord{ + 10, + 10, + 458, + }, + dictWord{11, 10, 373}, + dictWord{11, 10, 668}, + dictWord{11, 10, 795}, + dictWord{11, 10, 897}, + dictWord{12, 10, 272}, + dictWord{12, 10, 424}, + dictWord{12, 10, 539}, + dictWord{12, 10, 558}, + dictWord{14, 10, 245}, + dictWord{14, 10, 263}, + dictWord{14, 10, 264}, + dictWord{14, 10, 393}, + dictWord{ + 142, + 10, + 403, + }, + dictWord{11, 0, 91}, + dictWord{13, 0, 129}, + dictWord{15, 0, 101}, + dictWord{145, 0, 125}, + dictWord{135, 0, 1132}, + dictWord{4, 0, 494}, + dictWord{6, 0, 74}, + dictWord{7, 0, 44}, + dictWord{7, 0, 407}, + dictWord{12, 0, 17}, + dictWord{15, 0, 5}, + dictWord{148, 0, 11}, + dictWord{133, 10, 379}, + dictWord{5, 0, 270}, + dictWord{ + 5, + 11, + 684, + }, + dictWord{6, 10, 89}, + dictWord{6, 10, 400}, + dictWord{7, 10, 1569}, + dictWord{7, 10, 1623}, + dictWord{7, 10, 1850}, + dictWord{8, 10, 218}, + dictWord{ + 8, + 10, + 422, + }, + dictWord{9, 10, 570}, + dictWord{138, 10, 626}, + dictWord{4, 0, 276}, + dictWord{133, 0, 296}, + dictWord{6, 0, 1523}, + dictWord{134, 11, 27}, + dictWord{ + 6, + 10, + 387, + }, + dictWord{7, 10, 882}, + dictWord{141, 10, 111}, + dictWord{6, 10, 224}, + dictWord{7, 10, 877}, + dictWord{137, 10, 647}, + dictWord{135, 10, 790}, + dictWord{ + 4, + 0, + 7, + }, + dictWord{5, 0, 90}, + dictWord{5, 0, 158}, + dictWord{6, 0, 542}, + dictWord{7, 0, 221}, + dictWord{7, 0, 1574}, + dictWord{9, 0, 490}, + dictWord{10, 0, 540}, + dictWord{ + 11, + 0, + 443, + }, + dictWord{139, 0, 757}, + dictWord{7, 0, 588}, + dictWord{9, 0, 175}, + dictWord{138, 0, 530}, + dictWord{135, 10, 394}, + dictWord{142, 11, 23}, + dictWord{ + 134, + 0, + 786, + }, + dictWord{135, 0, 580}, + dictWord{7, 0, 88}, + dictWord{136, 0, 627}, + dictWord{5, 0, 872}, + dictWord{6, 0, 57}, + dictWord{7, 0, 471}, + dictWord{9, 0, 447}, + dictWord{137, 0, 454}, + dictWord{6, 11, 342}, + dictWord{6, 11, 496}, + dictWord{8, 11, 275}, + dictWord{137, 11, 206}, + dictWord{4, 11, 909}, + dictWord{133, 11, 940}, + dictWord{6, 0, 735}, + dictWord{132, 11, 891}, + dictWord{8, 0, 845}, + dictWord{8, 0, 916}, + dictWord{135, 10, 1409}, + dictWord{5, 0, 31}, + dictWord{134, 0, 614}, + dictWord{11, 0, 458}, + dictWord{12, 0, 15}, + dictWord{140, 0, 432}, + dictWord{8, 0, 330}, + dictWord{140, 0, 477}, + dictWord{4, 0, 530}, + dictWord{5, 0, 521}, + dictWord{ + 7, + 0, + 1200, + }, + dictWord{10, 0, 460}, + dictWord{132, 11, 687}, + dictWord{6, 0, 424}, + dictWord{135, 0, 1866}, + dictWord{9, 0, 569}, + dictWord{12, 0, 12}, + dictWord{ + 12, + 0, + 81, + }, + dictWord{12, 0, 319}, + dictWord{13, 0, 69}, + dictWord{14, 0, 259}, + dictWord{16, 0, 87}, + dictWord{17, 0, 1}, + dictWord{17, 0, 21}, + dictWord{17, 0, 24}, + dictWord{ + 18, + 0, + 15, + }, + dictWord{18, 0, 56}, + dictWord{18, 0, 59}, + dictWord{18, 0, 127}, + dictWord{18, 0, 154}, + dictWord{19, 0, 19}, + dictWord{148, 0, 31}, + dictWord{7, 0, 1302}, + dictWord{136, 10, 38}, + dictWord{134, 11, 253}, + dictWord{5, 10, 261}, + dictWord{7, 10, 78}, + dictWord{7, 10, 199}, + dictWord{8, 10, 815}, + dictWord{9, 10, 126}, + dictWord{138, 10, 342}, + dictWord{5, 0, 595}, + dictWord{135, 0, 1863}, + dictWord{6, 11, 41}, + dictWord{141, 11, 160}, + dictWord{5, 0, 13}, + dictWord{134, 0, 142}, + dictWord{6, 0, 97}, + dictWord{7, 0, 116}, + dictWord{8, 0, 322}, + dictWord{8, 0, 755}, + dictWord{9, 0, 548}, + dictWord{10, 0, 714}, + dictWord{11, 0, 884}, + dictWord{13, 0, 324}, + dictWord{7, 11, 1304}, + dictWord{138, 11, 477}, + dictWord{132, 10, 628}, + dictWord{134, 11, 1718}, + dictWord{7, 10, 266}, + dictWord{136, 10, 804}, + dictWord{135, 10, 208}, + dictWord{7, 0, 1021}, + dictWord{6, 10, 79}, + dictWord{135, 10, 1519}, + dictWord{7, 0, 1472}, + dictWord{135, 0, 1554}, + dictWord{6, 11, 362}, + dictWord{146, 11, 51}, + dictWord{7, 0, 1071}, + dictWord{7, 0, 1541}, + dictWord{7, 0, 1767}, + dictWord{7, 0, 1806}, + dictWord{11, 0, 162}, + dictWord{11, 0, 242}, + dictWord{11, 0, 452}, + dictWord{12, 0, 605}, + dictWord{15, 0, 26}, + dictWord{144, 0, 44}, + dictWord{136, 10, 741}, + dictWord{133, 11, 115}, + dictWord{145, 0, 115}, + dictWord{134, 10, 376}, + dictWord{6, 0, 1406}, + dictWord{134, 0, 1543}, + dictWord{5, 11, 193}, + dictWord{12, 11, 178}, + dictWord{13, 11, 130}, + dictWord{ + 145, + 11, + 84, + }, + dictWord{135, 0, 1111}, + dictWord{8, 0, 1}, + dictWord{9, 0, 650}, + dictWord{10, 0, 326}, + dictWord{5, 11, 705}, + dictWord{137, 11, 606}, + dictWord{5, 0, 488}, + dictWord{6, 0, 527}, + dictWord{7, 0, 489}, + dictWord{7, 0, 1636}, + dictWord{8, 0, 121}, + dictWord{8, 0, 144}, + dictWord{8, 0, 359}, + dictWord{9, 0, 193}, + dictWord{9, 0, 241}, + dictWord{9, 0, 336}, + dictWord{9, 0, 882}, + dictWord{11, 0, 266}, + dictWord{11, 0, 372}, + dictWord{11, 0, 944}, + dictWord{12, 0, 401}, + dictWord{140, 0, 641}, + dictWord{135, 11, 174}, + dictWord{6, 0, 267}, + dictWord{7, 10, 244}, + dictWord{7, 10, 632}, + dictWord{7, 10, 1609}, + dictWord{8, 10, 178}, + dictWord{8, 10, 638}, + dictWord{141, 10, 58}, + dictWord{134, 0, 1983}, + dictWord{134, 0, 1155}, + dictWord{134, 0, 1575}, + dictWord{134, 0, 1438}, + dictWord{9, 0, 31}, + dictWord{ + 10, + 0, + 244, + }, + dictWord{10, 0, 699}, + dictWord{12, 0, 149}, + dictWord{141, 0, 497}, + dictWord{133, 0, 377}, + dictWord{4, 11, 122}, + dictWord{5, 11, 796}, + dictWord{ + 5, + 11, + 952, + }, + dictWord{6, 11, 1660}, + dictWord{6, 11, 1671}, + dictWord{8, 11, 567}, + dictWord{9, 11, 687}, + dictWord{9, 11, 742}, + dictWord{10, 11, 686}, + dictWord{ + 11, + 11, + 356, + }, + dictWord{11, 11, 682}, + dictWord{140, 11, 281}, + dictWord{145, 0, 101}, + dictWord{11, 11, 0}, + dictWord{144, 11, 78}, + dictWord{5, 11, 179}, + dictWord{ + 5, + 10, + 791, + }, + dictWord{7, 11, 1095}, + dictWord{135, 11, 1213}, + dictWord{8, 11, 372}, + dictWord{9, 11, 122}, + dictWord{138, 11, 175}, + dictWord{7, 10, 686}, + dictWord{8, 10, 33}, + dictWord{8, 10, 238}, + dictWord{10, 10, 616}, + dictWord{11, 10, 467}, + dictWord{11, 10, 881}, + dictWord{13, 10, 217}, + dictWord{13, 10, 253}, + dictWord{142, 10, 268}, + dictWord{9, 0, 476}, + dictWord{4, 11, 66}, + dictWord{7, 11, 722}, + dictWord{135, 11, 904}, + dictWord{7, 11, 352}, + dictWord{137, 11, 684}, + dictWord{135, 0, 2023}, + dictWord{135, 0, 1836}, + dictWord{132, 10, 447}, + dictWord{5, 0, 843}, + dictWord{144, 0, 35}, + dictWord{137, 11, 779}, + dictWord{ + 141, + 11, + 35, + }, + dictWord{4, 10, 128}, + dictWord{5, 10, 415}, + dictWord{6, 10, 462}, + dictWord{7, 10, 294}, + dictWord{7, 10, 578}, + dictWord{10, 10, 710}, + dictWord{ + 139, + 10, + 86, + }, + dictWord{132, 0, 554}, + dictWord{133, 0, 536}, + dictWord{136, 10, 587}, + dictWord{5, 0, 207}, + dictWord{9, 0, 79}, + dictWord{11, 0, 625}, + dictWord{ + 145, + 0, + 7, + }, + dictWord{7, 0, 1371}, + dictWord{6, 10, 427}, + dictWord{138, 10, 692}, + dictWord{4, 0, 424}, + dictWord{4, 10, 195}, + dictWord{135, 10, 802}, + dictWord{ + 8, + 0, + 785, + }, + dictWord{133, 11, 564}, + dictWord{135, 0, 336}, + dictWord{4, 0, 896}, + dictWord{6, 0, 1777}, + dictWord{134, 11, 556}, + dictWord{137, 11, 103}, + dictWord{134, 10, 1683}, + dictWord{7, 11, 544}, + dictWord{8, 11, 719}, + dictWord{138, 11, 61}, + dictWord{138, 10, 472}, + dictWord{4, 11, 5}, + dictWord{5, 11, 498}, + dictWord{136, 11, 637}, + dictWord{7, 0, 750}, + dictWord{9, 0, 223}, + dictWord{11, 0, 27}, + dictWord{11, 0, 466}, + dictWord{12, 0, 624}, + dictWord{14, 0, 265}, + dictWord{ + 146, + 0, + 61, + }, + dictWord{12, 0, 238}, + dictWord{18, 0, 155}, + dictWord{12, 11, 238}, + dictWord{146, 11, 155}, + dictWord{151, 10, 28}, + dictWord{133, 11, 927}, + dictWord{12, 0, 383}, + dictWord{5, 10, 3}, + dictWord{8, 10, 578}, + dictWord{9, 10, 118}, + dictWord{10, 10, 705}, + dictWord{141, 10, 279}, + dictWord{4, 11, 893}, + dictWord{ + 5, + 11, + 780, + }, + dictWord{133, 11, 893}, + dictWord{4, 0, 603}, + dictWord{133, 0, 661}, + dictWord{4, 0, 11}, + dictWord{6, 0, 128}, + dictWord{7, 0, 231}, + dictWord{ + 7, + 0, + 1533, + }, + dictWord{10, 0, 725}, + dictWord{5, 10, 229}, + dictWord{5, 11, 238}, + dictWord{135, 11, 1350}, + dictWord{8, 10, 102}, + dictWord{10, 10, 578}, + dictWord{ + 10, + 10, + 672, + }, + dictWord{12, 10, 496}, + dictWord{13, 10, 408}, + dictWord{14, 10, 121}, + dictWord{145, 10, 106}, + dictWord{132, 0, 476}, + dictWord{134, 0, 1552}, + dictWord{134, 11, 1729}, + dictWord{8, 10, 115}, + dictWord{8, 10, 350}, + dictWord{9, 10, 489}, + dictWord{10, 10, 128}, + dictWord{11, 10, 306}, + dictWord{ + 12, + 10, + 373, + }, + dictWord{14, 10, 30}, + dictWord{17, 10, 79}, + dictWord{19, 10, 80}, + dictWord{150, 10, 55}, + dictWord{135, 0, 1807}, + dictWord{4, 0, 680}, + dictWord{ + 4, + 11, + 60, + }, + dictWord{7, 11, 760}, + dictWord{7, 11, 1800}, + dictWord{8, 11, 314}, + dictWord{9, 11, 700}, + dictWord{139, 11, 487}, + dictWord{4, 10, 230}, + dictWord{ + 5, + 10, + 702, + }, + dictWord{148, 11, 94}, + dictWord{132, 11, 228}, + dictWord{139, 0, 435}, + dictWord{9, 0, 20}, + dictWord{10, 0, 324}, + dictWord{10, 0, 807}, + dictWord{ + 139, + 0, + 488, + }, + dictWord{6, 10, 1728}, + dictWord{136, 11, 419}, + dictWord{4, 10, 484}, + dictWord{18, 10, 26}, + dictWord{19, 10, 42}, + dictWord{20, 10, 43}, + dictWord{ + 21, + 10, + 0, + }, + dictWord{23, 10, 27}, + dictWord{152, 10, 14}, + dictWord{135, 0, 1431}, + dictWord{133, 11, 828}, + dictWord{5, 0, 112}, + dictWord{6, 0, 103}, + dictWord{ + 6, + 0, + 150, + }, + dictWord{7, 0, 1303}, + dictWord{9, 0, 292}, + dictWord{10, 0, 481}, + dictWord{20, 0, 13}, + dictWord{7, 11, 176}, + dictWord{7, 11, 178}, + dictWord{7, 11, 1110}, + dictWord{10, 11, 481}, + dictWord{148, 11, 13}, + dictWord{138, 0, 356}, + dictWord{4, 11, 51}, + dictWord{5, 11, 39}, + dictWord{6, 11, 4}, + dictWord{7, 11, 591}, + dictWord{ + 7, + 11, + 849, + }, + dictWord{7, 11, 951}, + dictWord{7, 11, 1129}, + dictWord{7, 11, 1613}, + dictWord{7, 11, 1760}, + dictWord{7, 11, 1988}, + dictWord{9, 11, 434}, + dictWord{10, 11, 754}, + dictWord{11, 11, 25}, + dictWord{11, 11, 37}, + dictWord{139, 11, 414}, + dictWord{6, 0, 1963}, + dictWord{134, 0, 2000}, + dictWord{ + 132, + 10, + 633, + }, + dictWord{6, 0, 1244}, + dictWord{133, 11, 902}, + dictWord{135, 11, 928}, + dictWord{140, 0, 18}, + dictWord{138, 0, 204}, + dictWord{135, 11, 1173}, + dictWord{134, 0, 867}, + dictWord{4, 0, 708}, + dictWord{8, 0, 15}, + dictWord{9, 0, 50}, + dictWord{9, 0, 386}, + dictWord{11, 0, 18}, + dictWord{11, 0, 529}, + dictWord{140, 0, 228}, + dictWord{134, 11, 270}, + dictWord{4, 0, 563}, + dictWord{7, 0, 109}, + dictWord{7, 0, 592}, + dictWord{7, 0, 637}, + dictWord{7, 0, 770}, + dictWord{8, 0, 463}, + dictWord{ + 9, + 0, + 60, + }, + dictWord{9, 0, 335}, + dictWord{9, 0, 904}, + dictWord{10, 0, 73}, + dictWord{11, 0, 434}, + dictWord{12, 0, 585}, + dictWord{13, 0, 331}, + dictWord{18, 0, 110}, + dictWord{148, 0, 60}, + dictWord{132, 0, 502}, + dictWord{14, 11, 359}, + dictWord{19, 11, 52}, + dictWord{148, 11, 47}, + dictWord{6, 11, 377}, + dictWord{7, 11, 1025}, + dictWord{9, 11, 613}, + dictWord{145, 11, 104}, + dictWord{6, 0, 347}, + dictWord{10, 0, 161}, + dictWord{5, 10, 70}, + dictWord{5, 10, 622}, + dictWord{6, 10, 334}, + dictWord{ + 7, + 10, + 1032, + }, + dictWord{9, 10, 171}, + dictWord{11, 10, 26}, + dictWord{11, 10, 213}, + dictWord{11, 10, 637}, + dictWord{11, 10, 707}, + dictWord{12, 10, 202}, + dictWord{12, 10, 380}, + dictWord{13, 10, 226}, + dictWord{13, 10, 355}, + dictWord{14, 10, 222}, + dictWord{145, 10, 42}, + dictWord{132, 11, 416}, + dictWord{4, 0, 33}, + dictWord{5, 0, 102}, + dictWord{6, 0, 284}, + dictWord{7, 0, 1079}, + dictWord{7, 0, 1423}, + dictWord{7, 0, 1702}, + dictWord{8, 0, 470}, + dictWord{9, 0, 554}, + dictWord{ + 9, + 0, + 723, + }, + dictWord{11, 0, 333}, + dictWord{142, 11, 372}, + dictWord{5, 11, 152}, + dictWord{5, 11, 197}, + dictWord{7, 11, 340}, + dictWord{7, 11, 867}, + dictWord{ + 10, + 11, + 548, + }, + dictWord{10, 11, 581}, + dictWord{11, 11, 6}, + dictWord{12, 11, 3}, + dictWord{12, 11, 19}, + dictWord{14, 11, 110}, + dictWord{142, 11, 289}, + dictWord{ + 7, + 0, + 246, + }, + dictWord{135, 0, 840}, + dictWord{6, 0, 10}, + dictWord{8, 0, 571}, + dictWord{9, 0, 739}, + dictWord{143, 0, 91}, + dictWord{6, 0, 465}, + dictWord{7, 0, 1465}, + dictWord{ + 4, + 10, + 23, + }, + dictWord{4, 10, 141}, + dictWord{5, 10, 313}, + dictWord{5, 10, 1014}, + dictWord{6, 10, 50}, + dictWord{7, 10, 142}, + dictWord{7, 10, 559}, + dictWord{ + 8, + 10, + 640, + }, + dictWord{9, 10, 460}, + dictWord{9, 10, 783}, + dictWord{11, 10, 741}, + dictWord{12, 10, 183}, + dictWord{141, 10, 488}, + dictWord{133, 0, 626}, + dictWord{ + 136, + 0, + 614, + }, + dictWord{138, 0, 237}, + dictWord{7, 11, 34}, + dictWord{7, 11, 190}, + dictWord{8, 11, 28}, + dictWord{8, 11, 141}, + dictWord{8, 11, 444}, + dictWord{ + 8, + 11, + 811, + }, + dictWord{9, 11, 468}, + dictWord{11, 11, 334}, + dictWord{12, 11, 24}, + dictWord{12, 11, 386}, + dictWord{140, 11, 576}, + dictWord{133, 11, 757}, + dictWord{ + 5, + 0, + 18, + }, + dictWord{6, 0, 526}, + dictWord{13, 0, 24}, + dictWord{13, 0, 110}, + dictWord{19, 0, 5}, + dictWord{147, 0, 44}, + dictWord{6, 0, 506}, + dictWord{134, 11, 506}, + dictWord{135, 11, 1553}, + dictWord{4, 0, 309}, + dictWord{5, 0, 462}, + dictWord{7, 0, 970}, + dictWord{7, 0, 1097}, + dictWord{22, 0, 30}, + dictWord{22, 0, 33}, + dictWord{ + 7, + 11, + 1385, + }, + dictWord{11, 11, 582}, + dictWord{11, 11, 650}, + dictWord{11, 11, 901}, + dictWord{11, 11, 949}, + dictWord{12, 11, 232}, + dictWord{12, 11, 236}, + dictWord{13, 11, 413}, + dictWord{13, 11, 501}, + dictWord{146, 11, 116}, + dictWord{9, 0, 140}, + dictWord{5, 10, 222}, + dictWord{138, 10, 534}, + dictWord{6, 0, 1056}, + dictWord{137, 10, 906}, + dictWord{134, 0, 1704}, + dictWord{138, 10, 503}, + dictWord{134, 0, 1036}, + dictWord{5, 10, 154}, + dictWord{7, 10, 1491}, + dictWord{ + 10, + 10, + 379, + }, + dictWord{138, 10, 485}, + dictWord{4, 11, 383}, + dictWord{133, 10, 716}, + dictWord{134, 0, 1315}, + dictWord{5, 0, 86}, + dictWord{7, 0, 743}, + dictWord{ + 9, + 0, + 85, + }, + dictWord{10, 0, 281}, + dictWord{10, 0, 432}, + dictWord{11, 0, 825}, + dictWord{12, 0, 251}, + dictWord{13, 0, 118}, + dictWord{142, 0, 378}, + dictWord{ + 8, + 0, + 264, + }, + dictWord{4, 10, 91}, + dictWord{5, 10, 388}, + dictWord{5, 10, 845}, + dictWord{6, 10, 206}, + dictWord{6, 10, 252}, + dictWord{6, 10, 365}, + dictWord{7, 10, 136}, + dictWord{7, 10, 531}, + dictWord{136, 10, 621}, + dictWord{5, 0, 524}, + dictWord{133, 0, 744}, + dictWord{5, 11, 277}, + dictWord{141, 11, 247}, + dictWord{ + 132, + 11, + 435, + }, + dictWord{10, 0, 107}, + dictWord{140, 0, 436}, + dictWord{132, 0, 927}, + dictWord{10, 0, 123}, + dictWord{12, 0, 670}, + dictWord{146, 0, 94}, + dictWord{ + 7, + 0, + 1149, + }, + dictWord{9, 0, 156}, + dictWord{138, 0, 957}, + dictWord{5, 11, 265}, + dictWord{6, 11, 212}, + dictWord{135, 11, 28}, + dictWord{133, 0, 778}, + dictWord{ + 133, + 0, + 502, + }, + dictWord{8, 0, 196}, + dictWord{10, 0, 283}, + dictWord{139, 0, 406}, + dictWord{135, 10, 576}, + dictWord{136, 11, 535}, + dictWord{134, 0, 1312}, + dictWord{ + 5, + 10, + 771, + }, + dictWord{5, 10, 863}, + dictWord{5, 10, 898}, + dictWord{6, 10, 1632}, + dictWord{6, 10, 1644}, + dictWord{134, 10, 1780}, + dictWord{5, 0, 855}, + dictWord{5, 10, 331}, + dictWord{135, 11, 1487}, + dictWord{132, 11, 702}, + dictWord{5, 11, 808}, + dictWord{135, 11, 2045}, + dictWord{7, 0, 1400}, + dictWord{ + 9, + 0, + 446, + }, + dictWord{138, 0, 45}, + dictWord{140, 10, 632}, + dictWord{132, 0, 1003}, + dictWord{5, 11, 166}, + dictWord{8, 11, 739}, + dictWord{140, 11, 511}, + dictWord{ + 5, + 10, + 107, + }, + dictWord{7, 10, 201}, + dictWord{136, 10, 518}, + dictWord{6, 10, 446}, + dictWord{135, 10, 1817}, + dictWord{134, 0, 1532}, + dictWord{ + 134, + 0, + 1097, + }, + dictWord{4, 11, 119}, + dictWord{5, 11, 170}, + dictWord{5, 11, 447}, + dictWord{7, 11, 1708}, + dictWord{7, 11, 1889}, + dictWord{9, 11, 357}, + dictWord{ + 9, + 11, + 719, + }, + dictWord{12, 11, 486}, + dictWord{140, 11, 596}, + dictWord{9, 10, 851}, + dictWord{141, 10, 510}, + dictWord{7, 0, 612}, + dictWord{8, 0, 545}, + dictWord{ + 8, + 0, + 568, + }, + dictWord{8, 0, 642}, + dictWord{9, 0, 717}, + dictWord{10, 0, 541}, + dictWord{10, 0, 763}, + dictWord{11, 0, 449}, + dictWord{12, 0, 489}, + dictWord{13, 0, 153}, + dictWord{13, 0, 296}, + dictWord{14, 0, 138}, + dictWord{14, 0, 392}, + dictWord{15, 0, 50}, + dictWord{16, 0, 6}, + dictWord{16, 0, 12}, + dictWord{20, 0, 9}, + dictWord{ + 132, + 10, + 504, + }, + dictWord{4, 11, 450}, + dictWord{135, 11, 1158}, + dictWord{11, 0, 54}, + dictWord{13, 0, 173}, + dictWord{13, 0, 294}, + dictWord{5, 10, 883}, + dictWord{ + 5, + 10, + 975, + }, + dictWord{8, 10, 392}, + dictWord{148, 10, 7}, + dictWord{13, 0, 455}, + dictWord{15, 0, 99}, + dictWord{15, 0, 129}, + dictWord{144, 0, 68}, + dictWord{135, 0, 172}, + dictWord{132, 11, 754}, + dictWord{5, 10, 922}, + dictWord{134, 10, 1707}, + dictWord{134, 0, 1029}, + dictWord{17, 11, 39}, + dictWord{148, 11, 36}, + dictWord{ + 4, + 0, + 568, + }, + dictWord{5, 10, 993}, + dictWord{7, 10, 515}, + dictWord{137, 10, 91}, + dictWord{132, 0, 732}, + dictWord{10, 0, 617}, + dictWord{138, 11, 617}, + dictWord{ + 134, + 0, + 974, + }, + dictWord{7, 0, 989}, + dictWord{10, 0, 377}, + dictWord{12, 0, 363}, + dictWord{13, 0, 68}, + dictWord{13, 0, 94}, + dictWord{14, 0, 108}, + dictWord{ + 142, + 0, + 306, + }, + dictWord{136, 0, 733}, + dictWord{132, 0, 428}, + dictWord{7, 0, 1789}, + dictWord{135, 11, 1062}, + dictWord{7, 0, 2015}, + dictWord{140, 0, 665}, + dictWord{135, 10, 1433}, + dictWord{5, 0, 287}, + dictWord{7, 10, 921}, + dictWord{8, 10, 580}, + dictWord{8, 10, 593}, + dictWord{8, 10, 630}, + dictWord{138, 10, 28}, + dictWord{138, 0, 806}, + dictWord{4, 10, 911}, + dictWord{5, 10, 867}, + dictWord{5, 10, 1013}, + dictWord{7, 10, 2034}, + dictWord{8, 10, 798}, + dictWord{136, 10, 813}, + dictWord{134, 0, 1539}, + dictWord{8, 11, 523}, + dictWord{150, 11, 34}, + dictWord{135, 11, 740}, + dictWord{7, 11, 238}, + dictWord{7, 11, 2033}, + dictWord{ + 8, + 11, + 120, + }, + dictWord{8, 11, 188}, + dictWord{8, 11, 659}, + dictWord{9, 11, 598}, + dictWord{10, 11, 466}, + dictWord{12, 11, 342}, + dictWord{12, 11, 588}, + dictWord{ + 13, + 11, + 503, + }, + dictWord{14, 11, 246}, + dictWord{143, 11, 92}, + dictWord{7, 0, 1563}, + dictWord{141, 0, 182}, + dictWord{5, 10, 135}, + dictWord{6, 10, 519}, + dictWord{ + 7, + 10, + 1722, + }, + dictWord{10, 10, 271}, + dictWord{11, 10, 261}, + dictWord{145, 10, 54}, + dictWord{14, 10, 338}, + dictWord{148, 10, 81}, + dictWord{7, 0, 484}, + dictWord{ + 4, + 10, + 300, + }, + dictWord{133, 10, 436}, + dictWord{145, 11, 114}, + dictWord{6, 0, 1623}, + dictWord{134, 0, 1681}, + dictWord{133, 11, 640}, + dictWord{4, 11, 201}, + dictWord{7, 11, 1744}, + dictWord{8, 11, 602}, + dictWord{11, 11, 247}, + dictWord{11, 11, 826}, + dictWord{145, 11, 65}, + dictWord{8, 11, 164}, + dictWord{ + 146, + 11, + 62, + }, + dictWord{6, 0, 1833}, + dictWord{6, 0, 1861}, + dictWord{136, 0, 878}, + dictWord{134, 0, 1569}, + dictWord{8, 10, 357}, + dictWord{10, 10, 745}, + dictWord{ + 14, + 10, + 426, + }, + dictWord{17, 10, 94}, + dictWord{147, 10, 57}, + dictWord{12, 0, 93}, + dictWord{12, 0, 501}, + dictWord{13, 0, 362}, + dictWord{14, 0, 151}, + dictWord{15, 0, 40}, + dictWord{15, 0, 59}, + dictWord{16, 0, 46}, + dictWord{17, 0, 25}, + dictWord{18, 0, 14}, + dictWord{18, 0, 134}, + dictWord{19, 0, 25}, + dictWord{19, 0, 69}, + dictWord{ + 20, + 0, + 16, + }, + dictWord{20, 0, 19}, + dictWord{20, 0, 66}, + dictWord{21, 0, 23}, + dictWord{21, 0, 25}, + dictWord{150, 0, 42}, + dictWord{6, 0, 1748}, + dictWord{8, 0, 715}, + dictWord{ + 9, + 0, + 802, + }, + dictWord{10, 0, 46}, + dictWord{10, 0, 819}, + dictWord{13, 0, 308}, + dictWord{14, 0, 351}, + dictWord{14, 0, 363}, + dictWord{146, 0, 67}, + dictWord{ + 132, + 0, + 994, + }, + dictWord{4, 0, 63}, + dictWord{133, 0, 347}, + dictWord{132, 0, 591}, + dictWord{133, 0, 749}, + dictWord{7, 11, 1577}, + dictWord{10, 11, 304}, + dictWord{ + 10, + 11, + 549, + }, + dictWord{11, 11, 424}, + dictWord{12, 11, 365}, + dictWord{13, 11, 220}, + dictWord{13, 11, 240}, + dictWord{142, 11, 33}, + dictWord{133, 0, 366}, + dictWord{ + 7, + 0, + 557, + }, + dictWord{12, 0, 547}, + dictWord{14, 0, 86}, + dictWord{133, 10, 387}, + dictWord{135, 0, 1747}, + dictWord{132, 11, 907}, + dictWord{5, 11, 100}, + dictWord{10, 11, 329}, + dictWord{12, 11, 416}, + dictWord{149, 11, 29}, + dictWord{4, 10, 6}, + dictWord{5, 10, 708}, + dictWord{136, 10, 75}, + dictWord{7, 10, 1351}, + dictWord{9, 10, 581}, + dictWord{10, 10, 639}, + dictWord{11, 10, 453}, + dictWord{140, 10, 584}, + dictWord{7, 0, 89}, + dictWord{132, 10, 303}, + dictWord{138, 10, 772}, + dictWord{132, 11, 176}, + dictWord{5, 11, 636}, + dictWord{5, 11, 998}, + dictWord{8, 11, 26}, + dictWord{137, 11, 358}, + dictWord{7, 11, 9}, + dictWord{7, 11, 1508}, + dictWord{9, 11, 317}, + dictWord{10, 11, 210}, + dictWord{10, 11, 292}, + dictWord{10, 11, 533}, + dictWord{11, 11, 555}, + dictWord{12, 11, 526}, + dictWord{ + 12, + 11, + 607, + }, + dictWord{13, 11, 263}, + dictWord{13, 11, 459}, + dictWord{142, 11, 271}, + dictWord{134, 0, 1463}, + dictWord{6, 0, 772}, + dictWord{6, 0, 1137}, + dictWord{ + 139, + 11, + 595, + }, + dictWord{7, 0, 977}, + dictWord{139, 11, 66}, + dictWord{138, 0, 893}, + dictWord{20, 0, 48}, + dictWord{148, 11, 48}, + dictWord{5, 0, 824}, + dictWord{ + 133, + 0, + 941, + }, + dictWord{134, 11, 295}, + dictWord{7, 0, 1543}, + dictWord{7, 0, 1785}, + dictWord{10, 0, 690}, + dictWord{4, 10, 106}, + dictWord{139, 10, 717}, + dictWord{ + 7, + 0, + 440, + }, + dictWord{8, 0, 230}, + dictWord{139, 0, 106}, + dictWord{5, 10, 890}, + dictWord{133, 10, 988}, + dictWord{6, 10, 626}, + dictWord{142, 10, 431}, + dictWord{ + 10, + 11, + 127, + }, + dictWord{141, 11, 27}, + dictWord{17, 0, 32}, + dictWord{10, 10, 706}, + dictWord{150, 10, 44}, + dictWord{132, 0, 216}, + dictWord{137, 0, 332}, + dictWord{4, 10, 698}, + dictWord{136, 11, 119}, + dictWord{139, 11, 267}, + dictWord{138, 10, 17}, + dictWord{11, 11, 526}, + dictWord{11, 11, 939}, + dictWord{ + 141, + 11, + 290, + }, + dictWord{7, 11, 1167}, + dictWord{11, 11, 934}, + dictWord{13, 11, 391}, + dictWord{145, 11, 76}, + dictWord{139, 11, 39}, + dictWord{134, 10, 84}, + dictWord{ + 4, + 0, + 914, + }, + dictWord{5, 0, 800}, + dictWord{133, 0, 852}, + dictWord{10, 0, 416}, + dictWord{141, 0, 115}, + dictWord{7, 0, 564}, + dictWord{142, 0, 168}, + dictWord{ + 4, + 0, + 918, + }, + dictWord{133, 0, 876}, + dictWord{134, 0, 1764}, + dictWord{152, 0, 3}, + dictWord{4, 0, 92}, + dictWord{5, 0, 274}, + dictWord{7, 11, 126}, + dictWord{136, 11, 84}, + dictWord{140, 10, 498}, + dictWord{136, 11, 790}, + dictWord{8, 0, 501}, + dictWord{5, 10, 986}, + dictWord{6, 10, 130}, + dictWord{7, 10, 1582}, + dictWord{ + 8, + 10, + 458, + }, + dictWord{10, 10, 101}, + dictWord{10, 10, 318}, + dictWord{138, 10, 823}, + dictWord{6, 11, 64}, + dictWord{12, 11, 377}, + dictWord{141, 11, 309}, + dictWord{ + 5, + 0, + 743, + }, + dictWord{138, 0, 851}, + dictWord{4, 0, 49}, + dictWord{7, 0, 280}, + dictWord{135, 0, 1633}, + dictWord{134, 0, 879}, + dictWord{136, 0, 47}, + dictWord{ + 7, + 10, + 1644, + }, + dictWord{137, 10, 129}, + dictWord{132, 0, 865}, + dictWord{134, 0, 1202}, + dictWord{9, 11, 34}, + dictWord{139, 11, 484}, + dictWord{135, 10, 997}, + dictWord{5, 0, 272}, + dictWord{5, 0, 908}, + dictWord{5, 0, 942}, + dictWord{8, 0, 197}, + dictWord{9, 0, 47}, + dictWord{11, 0, 538}, + dictWord{139, 0, 742}, + dictWord{ + 6, + 11, + 1700, + }, + dictWord{7, 11, 26}, + dictWord{7, 11, 293}, + dictWord{7, 11, 382}, + dictWord{7, 11, 1026}, + dictWord{7, 11, 1087}, + dictWord{7, 11, 2027}, + dictWord{ + 8, + 11, + 24, + }, + dictWord{8, 11, 114}, + dictWord{8, 11, 252}, + dictWord{8, 11, 727}, + dictWord{8, 11, 729}, + dictWord{9, 11, 30}, + dictWord{9, 11, 199}, + dictWord{9, 11, 231}, + dictWord{9, 11, 251}, + dictWord{9, 11, 334}, + dictWord{9, 11, 361}, + dictWord{9, 11, 488}, + dictWord{9, 11, 712}, + dictWord{10, 11, 55}, + dictWord{10, 11, 60}, + dictWord{ + 10, + 11, + 232, + }, + dictWord{10, 11, 332}, + dictWord{10, 11, 384}, + dictWord{10, 11, 396}, + dictWord{10, 11, 504}, + dictWord{10, 11, 542}, + dictWord{10, 11, 652}, + dictWord{11, 11, 20}, + dictWord{11, 11, 48}, + dictWord{11, 11, 207}, + dictWord{11, 11, 291}, + dictWord{11, 11, 298}, + dictWord{11, 11, 342}, + dictWord{ + 11, + 11, + 365, + }, + dictWord{11, 11, 394}, + dictWord{11, 11, 620}, + dictWord{11, 11, 705}, + dictWord{11, 11, 1017}, + dictWord{12, 11, 123}, + dictWord{12, 11, 340}, + dictWord{12, 11, 406}, + dictWord{12, 11, 643}, + dictWord{13, 11, 61}, + dictWord{13, 11, 269}, + dictWord{13, 11, 311}, + dictWord{13, 11, 319}, + dictWord{13, 11, 486}, + dictWord{14, 11, 234}, + dictWord{15, 11, 62}, + dictWord{15, 11, 85}, + dictWord{16, 11, 71}, + dictWord{18, 11, 119}, + dictWord{148, 11, 105}, + dictWord{ + 6, + 0, + 1455, + }, + dictWord{150, 11, 37}, + dictWord{135, 10, 1927}, + dictWord{135, 0, 1911}, + dictWord{137, 0, 891}, + dictWord{7, 10, 1756}, + dictWord{137, 10, 98}, + dictWord{7, 10, 1046}, + dictWord{139, 10, 160}, + dictWord{132, 0, 761}, + dictWord{6, 11, 379}, + dictWord{7, 11, 270}, + dictWord{7, 11, 1116}, + dictWord{ + 8, + 11, + 176, + }, + dictWord{8, 11, 183}, + dictWord{9, 11, 432}, + dictWord{9, 11, 661}, + dictWord{12, 11, 247}, + dictWord{12, 11, 617}, + dictWord{146, 11, 125}, + dictWord{ + 6, + 10, + 45, + }, + dictWord{7, 10, 433}, + dictWord{8, 10, 129}, + dictWord{9, 10, 21}, + dictWord{10, 10, 392}, + dictWord{11, 10, 79}, + dictWord{12, 10, 499}, + dictWord{ + 13, + 10, + 199, + }, + dictWord{141, 10, 451}, + dictWord{4, 0, 407}, + dictWord{5, 11, 792}, + dictWord{133, 11, 900}, + dictWord{132, 0, 560}, + dictWord{135, 0, 183}, + dictWord{ + 13, + 0, + 490, + }, + dictWord{7, 10, 558}, + dictWord{136, 10, 353}, + dictWord{4, 0, 475}, + dictWord{6, 0, 731}, + dictWord{11, 0, 35}, + dictWord{13, 0, 71}, + dictWord{13, 0, 177}, + dictWord{14, 0, 422}, + dictWord{133, 10, 785}, + dictWord{8, 10, 81}, + dictWord{9, 10, 189}, + dictWord{9, 10, 201}, + dictWord{11, 10, 478}, + dictWord{11, 10, 712}, + dictWord{141, 10, 338}, + dictWord{4, 0, 418}, + dictWord{4, 0, 819}, + dictWord{133, 10, 353}, + dictWord{151, 10, 26}, + dictWord{4, 11, 901}, + dictWord{ + 133, + 11, + 776, + }, + dictWord{132, 0, 575}, + dictWord{7, 0, 818}, + dictWord{16, 0, 92}, + dictWord{17, 0, 14}, + dictWord{17, 0, 45}, + dictWord{18, 0, 75}, + dictWord{148, 0, 18}, + dictWord{ + 6, + 0, + 222, + }, + dictWord{7, 0, 636}, + dictWord{7, 0, 1620}, + dictWord{8, 0, 409}, + dictWord{9, 0, 693}, + dictWord{139, 0, 77}, + dictWord{6, 10, 25}, + dictWord{7, 10, 855}, + dictWord{7, 10, 1258}, + dictWord{144, 10, 32}, + dictWord{6, 0, 1880}, + dictWord{6, 0, 1887}, + dictWord{6, 0, 1918}, + dictWord{6, 0, 1924}, + dictWord{9, 0, 967}, + dictWord{9, 0, 995}, + dictWord{9, 0, 1015}, + dictWord{12, 0, 826}, + dictWord{12, 0, 849}, + dictWord{12, 0, 857}, + dictWord{12, 0, 860}, + dictWord{12, 0, 886}, + dictWord{ + 12, + 0, + 932, + }, + dictWord{18, 0, 228}, + dictWord{18, 0, 231}, + dictWord{146, 0, 240}, + dictWord{134, 0, 633}, + dictWord{134, 0, 1308}, + dictWord{4, 11, 37}, + dictWord{ + 5, + 11, + 334, + }, + dictWord{135, 11, 1253}, + dictWord{10, 0, 86}, + dictWord{4, 10, 4}, + dictWord{7, 10, 1118}, + dictWord{7, 10, 1320}, + dictWord{7, 10, 1706}, + dictWord{ + 8, + 10, + 277, + }, + dictWord{9, 10, 622}, + dictWord{11, 10, 724}, + dictWord{12, 10, 350}, + dictWord{12, 10, 397}, + dictWord{13, 10, 28}, + dictWord{13, 10, 159}, + dictWord{ + 15, + 10, + 89, + }, + dictWord{18, 10, 5}, + dictWord{19, 10, 9}, + dictWord{20, 10, 34}, + dictWord{150, 10, 47}, + dictWord{132, 11, 508}, + dictWord{137, 11, 448}, + dictWord{ + 12, + 11, + 107, + }, + dictWord{146, 11, 31}, + dictWord{132, 0, 817}, + dictWord{134, 0, 663}, + dictWord{133, 0, 882}, + dictWord{134, 0, 914}, + dictWord{132, 11, 540}, + dictWord{132, 11, 533}, + dictWord{136, 11, 608}, + dictWord{8, 0, 885}, + dictWord{138, 0, 865}, + dictWord{132, 0, 426}, + dictWord{6, 0, 58}, + dictWord{7, 0, 745}, + dictWord{7, 0, 1969}, + dictWord{8, 0, 399}, + dictWord{8, 0, 675}, + dictWord{9, 0, 479}, + dictWord{9, 0, 731}, + dictWord{10, 0, 330}, + dictWord{10, 0, 593}, + dictWord{ + 10, + 0, + 817, + }, + dictWord{11, 0, 32}, + dictWord{11, 0, 133}, + dictWord{11, 0, 221}, + dictWord{145, 0, 68}, + dictWord{134, 10, 255}, + dictWord{7, 0, 102}, + dictWord{ + 137, + 0, + 538, + }, + dictWord{137, 10, 216}, + dictWord{7, 11, 253}, + dictWord{136, 11, 549}, + dictWord{135, 11, 912}, + dictWord{9, 10, 183}, + dictWord{139, 10, 286}, + dictWord{11, 10, 956}, + dictWord{151, 10, 3}, + dictWord{8, 11, 527}, + dictWord{18, 11, 60}, + dictWord{147, 11, 24}, + dictWord{4, 10, 536}, + dictWord{7, 10, 1141}, + dictWord{10, 10, 723}, + dictWord{139, 10, 371}, + dictWord{133, 11, 920}, + dictWord{7, 0, 876}, + dictWord{135, 10, 285}, + dictWord{135, 10, 560}, + dictWord{ + 132, + 10, + 690, + }, + dictWord{142, 11, 126}, + dictWord{11, 10, 33}, + dictWord{12, 10, 571}, + dictWord{149, 10, 1}, + dictWord{133, 0, 566}, + dictWord{9, 0, 139}, + dictWord{ + 10, + 0, + 399, + }, + dictWord{11, 0, 469}, + dictWord{12, 0, 634}, + dictWord{13, 0, 223}, + dictWord{132, 11, 483}, + dictWord{6, 0, 48}, + dictWord{135, 0, 63}, + dictWord{18, 0, 12}, + dictWord{7, 10, 1862}, + dictWord{12, 10, 491}, + dictWord{12, 10, 520}, + dictWord{13, 10, 383}, + dictWord{142, 10, 244}, + dictWord{135, 11, 1665}, + dictWord{132, 11, 448}, + dictWord{9, 11, 495}, + dictWord{146, 11, 104}, + dictWord{6, 0, 114}, + dictWord{7, 0, 1224}, + dictWord{7, 0, 1556}, + dictWord{136, 0, 3}, + dictWord{ + 4, + 10, + 190, + }, + dictWord{133, 10, 554}, + dictWord{8, 0, 576}, + dictWord{9, 0, 267}, + dictWord{133, 10, 1001}, + dictWord{133, 10, 446}, + dictWord{133, 0, 933}, + dictWord{139, 11, 1009}, + dictWord{8, 11, 653}, + dictWord{13, 11, 93}, + dictWord{147, 11, 14}, + dictWord{6, 0, 692}, + dictWord{6, 0, 821}, + dictWord{134, 0, 1077}, + dictWord{5, 11, 172}, + dictWord{135, 11, 801}, + dictWord{138, 0, 752}, + dictWord{4, 0, 375}, + dictWord{134, 0, 638}, + dictWord{134, 0, 1011}, + dictWord{ + 140, + 11, + 540, + }, + dictWord{9, 0, 96}, + dictWord{133, 11, 260}, + dictWord{139, 11, 587}, + dictWord{135, 10, 1231}, + dictWord{12, 0, 30}, + dictWord{13, 0, 148}, + dictWord{ + 14, + 0, + 87, + }, + dictWord{14, 0, 182}, + dictWord{16, 0, 42}, + dictWord{20, 0, 70}, + dictWord{132, 10, 304}, + dictWord{6, 0, 1398}, + dictWord{7, 0, 56}, + dictWord{7, 0, 1989}, + dictWord{8, 0, 337}, + dictWord{8, 0, 738}, + dictWord{9, 0, 600}, + dictWord{12, 0, 37}, + dictWord{13, 0, 447}, + dictWord{142, 0, 92}, + dictWord{138, 0, 666}, + dictWord{ + 5, + 0, + 394, + }, + dictWord{7, 0, 487}, + dictWord{136, 0, 246}, + dictWord{9, 0, 437}, + dictWord{6, 10, 53}, + dictWord{6, 10, 199}, + dictWord{7, 10, 1408}, + dictWord{8, 10, 32}, + dictWord{8, 10, 93}, + dictWord{10, 10, 397}, + dictWord{10, 10, 629}, + dictWord{11, 10, 593}, + dictWord{11, 10, 763}, + dictWord{13, 10, 326}, + dictWord{145, 10, 35}, + dictWord{134, 10, 105}, + dictWord{9, 0, 320}, + dictWord{10, 0, 506}, + dictWord{138, 10, 794}, + dictWord{7, 11, 57}, + dictWord{8, 11, 167}, + dictWord{8, 11, 375}, + dictWord{9, 11, 82}, + dictWord{9, 11, 561}, + dictWord{10, 11, 620}, + dictWord{10, 11, 770}, + dictWord{11, 10, 704}, + dictWord{141, 10, 396}, + dictWord{6, 0, 1003}, + dictWord{5, 10, 114}, + dictWord{5, 10, 255}, + dictWord{141, 10, 285}, + dictWord{7, 0, 866}, + dictWord{135, 0, 1163}, + dictWord{133, 11, 531}, + dictWord{ + 132, + 0, + 328, + }, + dictWord{7, 10, 2035}, + dictWord{8, 10, 19}, + dictWord{9, 10, 89}, + dictWord{138, 10, 831}, + dictWord{8, 11, 194}, + dictWord{136, 11, 756}, + dictWord{ + 136, + 0, + 1000, + }, + dictWord{5, 11, 453}, + dictWord{134, 11, 441}, + dictWord{4, 0, 101}, + dictWord{5, 0, 833}, + dictWord{7, 0, 1171}, + dictWord{136, 0, 744}, + dictWord{ + 133, + 0, + 726, + }, + dictWord{136, 10, 746}, + dictWord{138, 0, 176}, + dictWord{6, 0, 9}, + dictWord{6, 0, 397}, + dictWord{7, 0, 53}, + dictWord{7, 0, 1742}, + dictWord{10, 0, 632}, + dictWord{11, 0, 828}, + dictWord{140, 0, 146}, + dictWord{135, 11, 22}, + dictWord{145, 11, 64}, + dictWord{132, 0, 839}, + dictWord{11, 0, 417}, + dictWord{12, 0, 223}, + dictWord{140, 0, 265}, + dictWord{4, 11, 102}, + dictWord{7, 11, 815}, + dictWord{7, 11, 1699}, + dictWord{139, 11, 964}, + dictWord{5, 10, 955}, + dictWord{ + 136, + 10, + 814, + }, + dictWord{6, 0, 1931}, + dictWord{6, 0, 2007}, + dictWord{18, 0, 246}, + dictWord{146, 0, 247}, + dictWord{8, 0, 198}, + dictWord{11, 0, 29}, + dictWord{140, 0, 534}, + dictWord{135, 0, 1771}, + dictWord{6, 0, 846}, + dictWord{7, 11, 1010}, + dictWord{11, 11, 733}, + dictWord{11, 11, 759}, + dictWord{12, 11, 563}, + dictWord{ + 13, + 11, + 34, + }, + dictWord{14, 11, 101}, + dictWord{18, 11, 45}, + dictWord{146, 11, 129}, + dictWord{4, 0, 186}, + dictWord{5, 0, 157}, + dictWord{8, 0, 168}, + dictWord{138, 0, 6}, + dictWord{132, 11, 899}, + dictWord{133, 10, 56}, + dictWord{148, 10, 100}, + dictWord{133, 0, 875}, + dictWord{5, 0, 773}, + dictWord{5, 0, 991}, + dictWord{6, 0, 1635}, + dictWord{134, 0, 1788}, + dictWord{6, 0, 1274}, + dictWord{9, 0, 477}, + dictWord{141, 0, 78}, + dictWord{4, 0, 639}, + dictWord{7, 0, 111}, + dictWord{8, 0, 581}, + dictWord{ + 12, + 0, + 177, + }, + dictWord{6, 11, 52}, + dictWord{9, 11, 104}, + dictWord{9, 11, 559}, + dictWord{10, 10, 4}, + dictWord{10, 10, 13}, + dictWord{11, 10, 638}, + dictWord{ + 12, + 11, + 308, + }, + dictWord{19, 11, 87}, + dictWord{148, 10, 57}, + dictWord{132, 11, 604}, + dictWord{4, 11, 301}, + dictWord{133, 10, 738}, + dictWord{133, 10, 758}, + dictWord{134, 0, 1747}, + dictWord{7, 11, 1440}, + dictWord{11, 11, 854}, + dictWord{11, 11, 872}, + dictWord{11, 11, 921}, + dictWord{12, 11, 551}, + dictWord{ + 13, + 11, + 472, + }, + dictWord{142, 11, 367}, + dictWord{7, 0, 1364}, + dictWord{7, 0, 1907}, + dictWord{141, 0, 158}, + dictWord{134, 0, 873}, + dictWord{4, 0, 404}, + dictWord{ + 4, + 0, + 659, + }, + dictWord{7, 0, 552}, + dictWord{135, 0, 675}, + dictWord{135, 10, 1112}, + dictWord{139, 10, 328}, + dictWord{7, 11, 508}, + dictWord{137, 10, 133}, + dictWord{133, 0, 391}, + dictWord{5, 10, 110}, + dictWord{6, 10, 169}, + dictWord{6, 10, 1702}, + dictWord{7, 10, 400}, + dictWord{8, 10, 538}, + dictWord{9, 10, 184}, + dictWord{ + 9, + 10, + 524, + }, + dictWord{140, 10, 218}, + dictWord{6, 11, 310}, + dictWord{7, 11, 1849}, + dictWord{8, 11, 72}, + dictWord{8, 11, 272}, + dictWord{8, 11, 431}, + dictWord{ + 9, + 11, + 12, + }, + dictWord{9, 11, 351}, + dictWord{10, 11, 563}, + dictWord{10, 11, 630}, + dictWord{10, 11, 810}, + dictWord{11, 11, 367}, + dictWord{11, 11, 599}, + dictWord{11, 11, 686}, + dictWord{140, 11, 672}, + dictWord{5, 0, 540}, + dictWord{6, 0, 1697}, + dictWord{136, 0, 668}, + dictWord{132, 0, 883}, + dictWord{134, 0, 78}, + dictWord{12, 0, 628}, + dictWord{18, 0, 79}, + dictWord{6, 10, 133}, + dictWord{9, 10, 353}, + dictWord{139, 10, 993}, + dictWord{6, 11, 181}, + dictWord{7, 11, 537}, + dictWord{ + 8, + 11, + 64, + }, + dictWord{9, 11, 127}, + dictWord{10, 11, 496}, + dictWord{12, 11, 510}, + dictWord{141, 11, 384}, + dictWord{6, 10, 93}, + dictWord{7, 10, 1422}, + dictWord{ + 7, + 10, + 1851, + }, + dictWord{8, 10, 673}, + dictWord{9, 10, 529}, + dictWord{140, 10, 43}, + dictWord{137, 10, 371}, + dictWord{134, 0, 1460}, + dictWord{134, 0, 962}, + dictWord{4, 11, 244}, + dictWord{135, 11, 233}, + dictWord{9, 10, 25}, + dictWord{10, 10, 467}, + dictWord{138, 10, 559}, + dictWord{4, 10, 335}, + dictWord{ + 135, + 10, + 942, + }, + dictWord{133, 0, 460}, + dictWord{135, 11, 334}, + dictWord{134, 11, 1650}, + dictWord{4, 0, 199}, + dictWord{139, 0, 34}, + dictWord{5, 10, 601}, + dictWord{ + 8, + 10, + 39, + }, + dictWord{10, 10, 773}, + dictWord{11, 10, 84}, + dictWord{12, 10, 205}, + dictWord{142, 10, 1}, + dictWord{133, 10, 870}, + dictWord{134, 0, 388}, + dictWord{14, 0, 474}, + dictWord{148, 0, 120}, + dictWord{133, 11, 369}, + dictWord{139, 0, 271}, + dictWord{4, 0, 511}, + dictWord{9, 0, 333}, + dictWord{9, 0, 379}, + dictWord{ + 10, + 0, + 602, + }, + dictWord{11, 0, 441}, + dictWord{11, 0, 723}, + dictWord{11, 0, 976}, + dictWord{12, 0, 357}, + dictWord{132, 10, 181}, + dictWord{134, 0, 608}, + dictWord{134, 10, 1652}, + dictWord{22, 0, 49}, + dictWord{137, 11, 338}, + dictWord{140, 0, 988}, + dictWord{134, 0, 617}, + dictWord{5, 0, 938}, + dictWord{136, 0, 707}, + dictWord{132, 10, 97}, + dictWord{5, 10, 147}, + dictWord{6, 10, 286}, + dictWord{7, 10, 1362}, + dictWord{141, 10, 176}, + dictWord{6, 0, 756}, + dictWord{ + 134, + 0, + 1149, + }, + dictWord{133, 11, 896}, + dictWord{6, 10, 375}, + dictWord{7, 10, 169}, + dictWord{7, 10, 254}, + dictWord{136, 10, 780}, + dictWord{134, 0, 1583}, + dictWord{135, 10, 1447}, + dictWord{139, 0, 285}, + dictWord{7, 11, 1117}, + dictWord{8, 11, 393}, + dictWord{136, 11, 539}, + dictWord{135, 0, 344}, + dictWord{ + 6, + 0, + 469, + }, + dictWord{7, 0, 1709}, + dictWord{138, 0, 515}, + dictWord{5, 10, 629}, + dictWord{135, 10, 1549}, + dictWord{5, 11, 4}, + dictWord{5, 11, 810}, + dictWord{ + 6, + 11, + 13, + }, + dictWord{6, 11, 538}, + dictWord{6, 11, 1690}, + dictWord{6, 11, 1726}, + dictWord{7, 11, 499}, + dictWord{7, 11, 1819}, + dictWord{8, 11, 148}, + dictWord{ + 8, + 11, + 696, + }, + dictWord{8, 11, 791}, + dictWord{12, 11, 125}, + dictWord{13, 11, 54}, + dictWord{143, 11, 9}, + dictWord{135, 11, 1268}, + dictWord{137, 0, 404}, + dictWord{ + 132, + 0, + 500, + }, + dictWord{5, 0, 68}, + dictWord{134, 0, 383}, + dictWord{11, 0, 216}, + dictWord{139, 0, 340}, + dictWord{4, 11, 925}, + dictWord{5, 11, 803}, + dictWord{ + 8, + 11, + 698, + }, + dictWord{138, 11, 828}, + dictWord{4, 0, 337}, + dictWord{6, 0, 353}, + dictWord{7, 0, 1934}, + dictWord{8, 0, 488}, + dictWord{137, 0, 429}, + dictWord{7, 0, 236}, + dictWord{7, 0, 1795}, + dictWord{8, 0, 259}, + dictWord{9, 0, 135}, + dictWord{9, 0, 177}, + dictWord{9, 0, 860}, + dictWord{10, 0, 825}, + dictWord{11, 0, 115}, + dictWord{ + 11, + 0, + 370, + }, + dictWord{11, 0, 405}, + dictWord{11, 0, 604}, + dictWord{12, 0, 10}, + dictWord{12, 0, 667}, + dictWord{12, 0, 669}, + dictWord{13, 0, 76}, + dictWord{14, 0, 310}, + dictWord{15, 0, 76}, + dictWord{15, 0, 147}, + dictWord{148, 0, 23}, + dictWord{4, 0, 15}, + dictWord{4, 0, 490}, + dictWord{5, 0, 22}, + dictWord{6, 0, 244}, + dictWord{7, 0, 40}, + dictWord{7, 0, 200}, + dictWord{7, 0, 906}, + dictWord{7, 0, 1199}, + dictWord{9, 0, 616}, + dictWord{10, 0, 716}, + dictWord{11, 0, 635}, + dictWord{11, 0, 801}, + dictWord{ + 140, + 0, + 458, + }, + dictWord{12, 0, 756}, + dictWord{132, 10, 420}, + dictWord{134, 0, 1504}, + dictWord{6, 0, 757}, + dictWord{133, 11, 383}, + dictWord{6, 0, 1266}, + dictWord{ + 135, + 0, + 1735, + }, + dictWord{5, 0, 598}, + dictWord{7, 0, 791}, + dictWord{8, 0, 108}, + dictWord{9, 0, 123}, + dictWord{7, 10, 1570}, + dictWord{140, 10, 542}, + dictWord{ + 142, + 11, + 410, + }, + dictWord{9, 11, 660}, + dictWord{138, 11, 347}, +} diff --git a/vendor/github.com/andybalholm/brotli/symbol_list.go b/vendor/github.com/andybalholm/brotli/symbol_list.go new file mode 100644 index 0000000000..c5cb49e5a9 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/symbol_list.go @@ -0,0 +1,22 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Utilities for building Huffman decoding tables. */ + +type symbolList struct { + storage []uint16 + offset int +} + +func symbolListGet(sl symbolList, i int) uint16 { + return sl.storage[i+sl.offset] +} + +func symbolListPut(sl symbolList, i int, val uint16) { + sl.storage[i+sl.offset] = val +} diff --git a/vendor/github.com/andybalholm/brotli/transform.go b/vendor/github.com/andybalholm/brotli/transform.go new file mode 100644 index 0000000000..d2c043a622 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/transform.go @@ -0,0 +1,641 @@ +package brotli + +const ( + transformIdentity = 0 + transformOmitLast1 = 1 + transformOmitLast2 = 2 + transformOmitLast3 = 3 + transformOmitLast4 = 4 + transformOmitLast5 = 5 + transformOmitLast6 = 6 + transformOmitLast7 = 7 + transformOmitLast8 = 8 + transformOmitLast9 = 9 + transformUppercaseFirst = 10 + transformUppercaseAll = 11 + transformOmitFirst1 = 12 + transformOmitFirst2 = 13 + transformOmitFirst3 = 14 + transformOmitFirst4 = 15 + transformOmitFirst5 = 16 + transformOmitFirst6 = 17 + transformOmitFirst7 = 18 + transformOmitFirst8 = 19 + transformOmitFirst9 = 20 + transformShiftFirst = 21 + transformShiftAll = 22 + iota - 22 + numTransformTypes +) + +const transformsMaxCutOff = transformOmitLast9 + +type transforms struct { + prefix_suffix_size uint16 + prefix_suffix []byte + prefix_suffix_map []uint16 + num_transforms uint32 + transforms []byte + params []byte + cutOffTransforms [transformsMaxCutOff + 1]int16 +} + +func transformPrefixId(t *transforms, I int) byte { + return t.transforms[(I*3)+0] +} + +func transformType(t *transforms, I int) byte { + return t.transforms[(I*3)+1] +} + +func transformSuffixId(t *transforms, I int) byte { + return t.transforms[(I*3)+2] +} + +func transformPrefix(t *transforms, I int) []byte { + return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:] +} + +func transformSuffix(t *transforms, I int) []byte { + return t.prefix_suffix[t.prefix_suffix_map[transformSuffixId(t, I)]:] +} + +/* RFC 7932 transforms string data */ +const kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000" + +var kPrefixSuffixMap = [50]uint16{ + 0x00, + 0x02, + 0x05, + 0x0E, + 0x13, + 0x16, + 0x18, + 0x1E, + 0x23, + 0x25, + 0x2A, + 0x2D, + 0x2F, + 0x32, + 0x34, + 0x3A, + 0x3E, + 0x45, + 0x47, + 0x4E, + 0x55, + 0x5A, + 0x5C, + 0x63, + 0x68, + 0x6D, + 0x72, + 0x77, + 0x7A, + 0x7C, + 0x80, + 0x83, + 0x88, + 0x8C, + 0x8E, + 0x91, + 0x97, + 0x9F, + 0xA5, + 0xA9, + 0xAD, + 0xB2, + 0xB7, + 0xBD, + 0xC2, + 0xC7, + 0xCA, + 0xCF, + 0xD5, + 0xD8, +} + +/* RFC 7932 transforms */ +var kTransformsData = []byte{ + 49, + transformIdentity, + 49, + 49, + transformIdentity, + 0, + 0, + transformIdentity, + 0, + 49, + transformOmitFirst1, + 49, + 49, + transformUppercaseFirst, + 0, + 49, + transformIdentity, + 47, + 0, + transformIdentity, + 49, + 4, + transformIdentity, + 0, + 49, + transformIdentity, + 3, + 49, + transformUppercaseFirst, + 49, + 49, + transformIdentity, + 6, + 49, + transformOmitFirst2, + 49, + 49, + transformOmitLast1, + 49, + 1, + transformIdentity, + 0, + 49, + transformIdentity, + 1, + 0, + transformUppercaseFirst, + 0, + 49, + transformIdentity, + 7, + 49, + transformIdentity, + 9, + 48, + transformIdentity, + 0, + 49, + transformIdentity, + 8, + 49, + transformIdentity, + 5, + 49, + transformIdentity, + 10, + 49, + transformIdentity, + 11, + 49, + transformOmitLast3, + 49, + 49, + transformIdentity, + 13, + 49, + transformIdentity, + 14, + 49, + transformOmitFirst3, + 49, + 49, + transformOmitLast2, + 49, + 49, + transformIdentity, + 15, + 49, + transformIdentity, + 16, + 0, + transformUppercaseFirst, + 49, + 49, + transformIdentity, + 12, + 5, + transformIdentity, + 49, + 0, + transformIdentity, + 1, + 49, + transformOmitFirst4, + 49, + 49, + transformIdentity, + 18, + 49, + transformIdentity, + 17, + 49, + transformIdentity, + 19, + 49, + transformIdentity, + 20, + 49, + transformOmitFirst5, + 49, + 49, + transformOmitFirst6, + 49, + 47, + transformIdentity, + 49, + 49, + transformOmitLast4, + 49, + 49, + transformIdentity, + 22, + 49, + transformUppercaseAll, + 49, + 49, + transformIdentity, + 23, + 49, + transformIdentity, + 24, + 49, + transformIdentity, + 25, + 49, + transformOmitLast7, + 49, + 49, + transformOmitLast1, + 26, + 49, + transformIdentity, + 27, + 49, + transformIdentity, + 28, + 0, + transformIdentity, + 12, + 49, + transformIdentity, + 29, + 49, + transformOmitFirst9, + 49, + 49, + transformOmitFirst7, + 49, + 49, + transformOmitLast6, + 49, + 49, + transformIdentity, + 21, + 49, + transformUppercaseFirst, + 1, + 49, + transformOmitLast8, + 49, + 49, + transformIdentity, + 31, + 49, + transformIdentity, + 32, + 47, + transformIdentity, + 3, + 49, + transformOmitLast5, + 49, + 49, + transformOmitLast9, + 49, + 0, + transformUppercaseFirst, + 1, + 49, + transformUppercaseFirst, + 8, + 5, + transformIdentity, + 21, + 49, + transformUppercaseAll, + 0, + 49, + transformUppercaseFirst, + 10, + 49, + transformIdentity, + 30, + 0, + transformIdentity, + 5, + 35, + transformIdentity, + 49, + 47, + transformIdentity, + 2, + 49, + transformUppercaseFirst, + 17, + 49, + transformIdentity, + 36, + 49, + transformIdentity, + 33, + 5, + transformIdentity, + 0, + 49, + transformUppercaseFirst, + 21, + 49, + transformUppercaseFirst, + 5, + 49, + transformIdentity, + 37, + 0, + transformIdentity, + 30, + 49, + transformIdentity, + 38, + 0, + transformUppercaseAll, + 0, + 49, + transformIdentity, + 39, + 0, + transformUppercaseAll, + 49, + 49, + transformIdentity, + 34, + 49, + transformUppercaseAll, + 8, + 49, + transformUppercaseFirst, + 12, + 0, + transformIdentity, + 21, + 49, + transformIdentity, + 40, + 0, + transformUppercaseFirst, + 12, + 49, + transformIdentity, + 41, + 49, + transformIdentity, + 42, + 49, + transformUppercaseAll, + 17, + 49, + transformIdentity, + 43, + 0, + transformUppercaseFirst, + 5, + 49, + transformUppercaseAll, + 10, + 0, + transformIdentity, + 34, + 49, + transformUppercaseFirst, + 33, + 49, + transformIdentity, + 44, + 49, + transformUppercaseAll, + 5, + 45, + transformIdentity, + 49, + 0, + transformIdentity, + 33, + 49, + transformUppercaseFirst, + 30, + 49, + transformUppercaseAll, + 30, + 49, + transformIdentity, + 46, + 49, + transformUppercaseAll, + 1, + 49, + transformUppercaseFirst, + 34, + 0, + transformUppercaseFirst, + 33, + 0, + transformUppercaseAll, + 30, + 0, + transformUppercaseAll, + 1, + 49, + transformUppercaseAll, + 33, + 49, + transformUppercaseAll, + 21, + 49, + transformUppercaseAll, + 12, + 0, + transformUppercaseAll, + 5, + 49, + transformUppercaseAll, + 34, + 0, + transformUppercaseAll, + 12, + 0, + transformUppercaseFirst, + 30, + 0, + transformUppercaseAll, + 34, + 0, + transformUppercaseFirst, + 34, +} + +var kBrotliTransforms = transforms{ + 217, + []byte(kPrefixSuffix), + kPrefixSuffixMap[:], + 121, + kTransformsData, + nil, /* no extra parameters */ + [transformsMaxCutOff + 1]int16{0, 12, 27, 23, 42, 63, 56, 48, 59, 64}, +} + +func getTransforms() *transforms { + return &kBrotliTransforms +} + +func toUpperCase(p []byte) int { + if p[0] < 0xC0 { + if p[0] >= 'a' && p[0] <= 'z' { + p[0] ^= 32 + } + + return 1 + } + + /* An overly simplified uppercasing model for UTF-8. */ + if p[0] < 0xE0 { + p[1] ^= 32 + return 2 + } + + /* An arbitrary transform for three byte characters. */ + p[2] ^= 5 + + return 3 +} + +func shiftTransform(word []byte, word_len int, parameter uint16) int { + /* Limited sign extension: scalar < (1 << 24). */ + var scalar uint32 = (uint32(parameter) & 0x7FFF) + (0x1000000 - (uint32(parameter) & 0x8000)) + if word[0] < 0x80 { + /* 1-byte rune / 0sssssss / 7 bit scalar (ASCII). */ + scalar += uint32(word[0]) + + word[0] = byte(scalar & 0x7F) + return 1 + } else if word[0] < 0xC0 { + /* Continuation / 10AAAAAA. */ + return 1 + } else if word[0] < 0xE0 { + /* 2-byte rune / 110sssss AAssssss / 11 bit scalar. */ + if word_len < 2 { + return 1 + } + scalar += uint32(word[1]&0x3F | (word[0]&0x1F)<<6) + word[0] = byte(0xC0 | (scalar>>6)&0x1F) + word[1] = byte(uint32(word[1]&0xC0) | scalar&0x3F) + return 2 + } else if word[0] < 0xF0 { + /* 3-byte rune / 1110ssss AAssssss BBssssss / 16 bit scalar. */ + if word_len < 3 { + return word_len + } + scalar += uint32(word[2])&0x3F | uint32(word[1]&0x3F)<<6 | uint32(word[0]&0x0F)<<12 + word[0] = byte(0xE0 | (scalar>>12)&0x0F) + word[1] = byte(uint32(word[1]&0xC0) | (scalar>>6)&0x3F) + word[2] = byte(uint32(word[2]&0xC0) | scalar&0x3F) + return 3 + } else if word[0] < 0xF8 { + /* 4-byte rune / 11110sss AAssssss BBssssss CCssssss / 21 bit scalar. */ + if word_len < 4 { + return word_len + } + scalar += uint32(word[3])&0x3F | uint32(word[2]&0x3F)<<6 | uint32(word[1]&0x3F)<<12 | uint32(word[0]&0x07)<<18 + word[0] = byte(0xF0 | (scalar>>18)&0x07) + word[1] = byte(uint32(word[1]&0xC0) | (scalar>>12)&0x3F) + word[2] = byte(uint32(word[2]&0xC0) | (scalar>>6)&0x3F) + word[3] = byte(uint32(word[3]&0xC0) | scalar&0x3F) + return 4 + } + + return 1 +} + +func transformDictionaryWord(dst []byte, word []byte, len int, trans *transforms, transform_idx int) int { + var idx int = 0 + var prefix []byte = transformPrefix(trans, transform_idx) + var type_ byte = transformType(trans, transform_idx) + var suffix []byte = transformSuffix(trans, transform_idx) + { + var prefix_len int = int(prefix[0]) + prefix = prefix[1:] + for { + tmp1 := prefix_len + prefix_len-- + if tmp1 == 0 { + break + } + dst[idx] = prefix[0] + idx++ + prefix = prefix[1:] + } + } + { + var t int = int(type_) + var i int = 0 + if t <= transformOmitLast9 { + len -= t + } else if t >= transformOmitFirst1 && t <= transformOmitFirst9 { + var skip int = t - (transformOmitFirst1 - 1) + word = word[skip:] + len -= skip + } + + for i < len { + dst[idx] = word[i] + idx++ + i++ + } + if t == transformUppercaseFirst { + toUpperCase(dst[idx-len:]) + } else if t == transformUppercaseAll { + var uppercase []byte = dst + uppercase = uppercase[idx-len:] + for len > 0 { + var step int = toUpperCase(uppercase) + uppercase = uppercase[step:] + len -= step + } + } else if t == transformShiftFirst { + var param uint16 = uint16(trans.params[transform_idx*2]) + uint16(trans.params[transform_idx*2+1])<<8 + shiftTransform(dst[idx-len:], int(len), param) + } else if t == transformShiftAll { + var param uint16 = uint16(trans.params[transform_idx*2]) + uint16(trans.params[transform_idx*2+1])<<8 + var shift []byte = dst + shift = shift[idx-len:] + for len > 0 { + var step int = shiftTransform(shift, int(len), param) + shift = shift[step:] + len -= step + } + } + } + { + var suffix_len int = int(suffix[0]) + suffix = suffix[1:] + for { + tmp2 := suffix_len + suffix_len-- + if tmp2 == 0 { + break + } + dst[idx] = suffix[0] + idx++ + suffix = suffix[1:] + } + return idx + } +} diff --git a/vendor/github.com/andybalholm/brotli/utf8_util.go b/vendor/github.com/andybalholm/brotli/utf8_util.go new file mode 100644 index 0000000000..3244247eec --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/utf8_util.go @@ -0,0 +1,70 @@ +package brotli + +/* Copyright 2013 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Heuristics for deciding about the UTF8-ness of strings. */ + +const kMinUTF8Ratio float64 = 0.75 + +/* Returns 1 if at least min_fraction of the bytes between pos and + pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise + returns 0. */ +func parseAsUTF8(symbol *int, input []byte, size uint) uint { + /* ASCII */ + if input[0]&0x80 == 0 { + *symbol = int(input[0]) + if *symbol > 0 { + return 1 + } + } + + /* 2-byte UTF8 */ + if size > 1 && input[0]&0xE0 == 0xC0 && input[1]&0xC0 == 0x80 { + *symbol = (int(input[0])&0x1F)<<6 | int(input[1])&0x3F + if *symbol > 0x7F { + return 2 + } + } + + /* 3-byte UFT8 */ + if size > 2 && input[0]&0xF0 == 0xE0 && input[1]&0xC0 == 0x80 && input[2]&0xC0 == 0x80 { + *symbol = (int(input[0])&0x0F)<<12 | (int(input[1])&0x3F)<<6 | int(input[2])&0x3F + if *symbol > 0x7FF { + return 3 + } + } + + /* 4-byte UFT8 */ + if size > 3 && input[0]&0xF8 == 0xF0 && input[1]&0xC0 == 0x80 && input[2]&0xC0 == 0x80 && input[3]&0xC0 == 0x80 { + *symbol = (int(input[0])&0x07)<<18 | (int(input[1])&0x3F)<<12 | (int(input[2])&0x3F)<<6 | int(input[3])&0x3F + if *symbol > 0xFFFF && *symbol <= 0x10FFFF { + return 4 + } + } + + /* Not UTF8, emit a special symbol above the UTF8-code space */ + *symbol = 0x110000 | int(input[0]) + + return 1 +} + +/* Returns 1 if at least min_fraction of the data is UTF8-encoded.*/ +func isMostlyUTF8(data []byte, pos uint, mask uint, length uint, min_fraction float64) bool { + var size_utf8 uint = 0 + var i uint = 0 + for i < length { + var symbol int + current_data := data[(pos+i)&mask:] + var bytes_read uint = parseAsUTF8(&symbol, current_data, length-i) + i += bytes_read + if symbol < 0x110000 { + size_utf8 += bytes_read + } + } + + return float64(size_utf8) > min_fraction*float64(length) +} diff --git a/vendor/github.com/andybalholm/brotli/util.go b/vendor/github.com/andybalholm/brotli/util.go new file mode 100644 index 0000000000..a84553a639 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/util.go @@ -0,0 +1,7 @@ +package brotli + +func assert(cond bool) { + if !cond { + panic("assertion failure") + } +} diff --git a/vendor/github.com/andybalholm/brotli/write_bits.go b/vendor/github.com/andybalholm/brotli/write_bits.go new file mode 100644 index 0000000000..8729901198 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/write_bits.go @@ -0,0 +1,52 @@ +package brotli + +import "encoding/binary" + +/* Copyright 2010 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Write bits into a byte array. */ + +/* This function writes bits into bytes in increasing addresses, and within + a byte least-significant-bit first. + + The function can write up to 56 bits in one go with WriteBits + Example: let's assume that 3 bits (Rs below) have been written already: + + BYTE-0 BYTE+1 BYTE+2 + + 0000 0RRR 0000 0000 0000 0000 + + Now, we could write 5 or less bits in MSB by just sifting by 3 + and OR'ing to BYTE-0. + + For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, + and locate the rest in BYTE+1, BYTE+2, etc. */ +func writeBits(n_bits uint, bits uint64, pos *uint, array []byte) { + /* This branch of the code can write up to 56 bits at a time, + 7 bits are lost by being perhaps already in *p and at least + 1 bit is needed to initialize the bit-stream ahead (i.e. if 7 + bits are in *p and we write 57 bits, then the next write will + access a byte that was never initialized). */ + p := array[*pos>>3:] + v := uint64(p[0]) + v |= bits << (*pos & 7) + binary.LittleEndian.PutUint64(p, v) + *pos += n_bits +} + +func writeSingleBit(bit bool, pos *uint, array []byte) { + if bit { + writeBits(1, 1, pos, array) + } else { + writeBits(1, 0, pos, array) + } +} + +func writeBitsPrepareStorage(pos uint, array []byte) { + assert(pos&7 == 0) + array[pos>>3] = 0 +} diff --git a/vendor/github.com/andybalholm/brotli/writer.go b/vendor/github.com/andybalholm/brotli/writer.go new file mode 100644 index 0000000000..8a688117d1 --- /dev/null +++ b/vendor/github.com/andybalholm/brotli/writer.go @@ -0,0 +1,162 @@ +package brotli + +import ( + "errors" + "io" + + "github.com/andybalholm/brotli/matchfinder" +) + +const ( + BestSpeed = 0 + BestCompression = 11 + DefaultCompression = 6 +) + +// WriterOptions configures Writer. +type WriterOptions struct { + // Quality controls the compression-speed vs compression-density trade-offs. + // The higher the quality, the slower the compression. Range is 0 to 11. + Quality int + // LGWin is the base 2 logarithm of the sliding window size. + // Range is 10 to 24. 0 indicates automatic configuration based on Quality. + LGWin int +} + +var ( + errEncode = errors.New("brotli: encode error") + errWriterClosed = errors.New("brotli: Writer is closed") +) + +// Writes to the returned writer are compressed and written to dst. +// It is the caller's responsibility to call Close on the Writer when done. +// Writes may be buffered and not flushed until Close. +func NewWriter(dst io.Writer) *Writer { + return NewWriterLevel(dst, DefaultCompression) +} + +// NewWriterLevel is like NewWriter but specifies the compression level instead +// of assuming DefaultCompression. +// The compression level can be DefaultCompression or any integer value between +// BestSpeed and BestCompression inclusive. +func NewWriterLevel(dst io.Writer, level int) *Writer { + return NewWriterOptions(dst, WriterOptions{ + Quality: level, + }) +} + +// NewWriterOptions is like NewWriter but specifies WriterOptions +func NewWriterOptions(dst io.Writer, options WriterOptions) *Writer { + w := new(Writer) + w.options = options + w.Reset(dst) + return w +} + +// Reset discards the Writer's state and makes it equivalent to the result of +// its original state from NewWriter or NewWriterLevel, but writing to dst +// instead. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(dst io.Writer) { + encoderInitState(w) + w.params.quality = w.options.Quality + if w.options.LGWin > 0 { + w.params.lgwin = uint(w.options.LGWin) + } + w.dst = dst + w.err = nil +} + +func (w *Writer) writeChunk(p []byte, op int) (n int, err error) { + if w.dst == nil { + return 0, errWriterClosed + } + if w.err != nil { + return 0, w.err + } + + for { + availableIn := uint(len(p)) + nextIn := p + success := encoderCompressStream(w, op, &availableIn, &nextIn) + bytesConsumed := len(p) - int(availableIn) + p = p[bytesConsumed:] + n += bytesConsumed + if !success { + return n, errEncode + } + + if len(p) == 0 || w.err != nil { + return n, w.err + } + } +} + +// Flush outputs encoded data for all input provided to Write. The resulting +// output can be decoded to match all input before Flush, but the stream is +// not yet complete until after Close. +// Flush has a negative impact on compression. +func (w *Writer) Flush() error { + _, err := w.writeChunk(nil, operationFlush) + return err +} + +// Close flushes remaining data to the decorated writer. +func (w *Writer) Close() error { + // If stream is already closed, it is reported by `writeChunk`. + _, err := w.writeChunk(nil, operationFinish) + w.dst = nil + return err +} + +// Write implements io.Writer. Flush or Close must be called to ensure that the +// encoded bytes are actually flushed to the underlying Writer. +func (w *Writer) Write(p []byte) (n int, err error) { + return w.writeChunk(p, operationProcess) +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + +// NewWriterV2 is like NewWriterLevel, but it uses the new implementation +// based on the matchfinder package. It currently supports up to level 7; +// if a higher level is specified, level 7 will be used. +func NewWriterV2(dst io.Writer, level int) *matchfinder.Writer { + var mf matchfinder.MatchFinder + if level < 2 { + mf = matchfinder.M0{Lazy: level == 1} + } else { + hashLen := 6 + if level >= 6 { + hashLen = 5 + } + chainLen := 64 + switch level { + case 2: + chainLen = 0 + case 3: + chainLen = 1 + case 4: + chainLen = 2 + case 5: + chainLen = 4 + case 6: + chainLen = 8 + } + mf = &matchfinder.M4{ + MaxDistance: 1 << 20, + ChainLength: chainLen, + HashLen: hashLen, + DistanceBitCost: 57, + } + } + + return &matchfinder.Writer{ + Dest: dst, + MatchFinder: mf, + Encoder: &Encoder{}, + BlockSize: 1 << 16, + } +} diff --git a/vendor/github.com/axiomhq/hyperloglog/.gitignore b/vendor/github.com/axiomhq/hyperloglog/.gitignore new file mode 100644 index 0000000000..a1338d6851 --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/.gitignore @@ -0,0 +1,14 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ diff --git a/vendor/github.com/axiomhq/hyperloglog/Contributing.md b/vendor/github.com/axiomhq/hyperloglog/Contributing.md new file mode 100644 index 0000000000..d0eee97178 --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/Contributing.md @@ -0,0 +1,41 @@ +## How to Contribute + +๐Ÿ‘๐ŸŽ‰ First of all, thank you for your interest in Axiom-node! We'd love to accept your patches and contributions! ๐ŸŽ‰๐Ÿ‘ + +This project accepts contributions. In order to contribute, you should pay attention to a few guidelines: + +## Reporting Issues + +Bugs, feature requests, and development-related questions should be directed to our [GitHub issue tracker](https://github.com/axiomhq/hyperloglog/issues). + +When reporting a bug, please try and provide as much context as possible such as your operating system, Go version and anything else that might be relevant to the bug. For feature requests, please explain what you're trying to do and how the requested feature would help you do that. + +## Setup + +[Fork](https://github.com/axiomhq/hyperloglog.git), then clone this repository: + +``` +git clone https://github.com/axiomhq/hyperloglog.git +cd hyperloglog +cd demo +go run hyperloglog_demo.go +``` + +## Submitting Modifications + +1. It's generally best to start by opening a new issue describing the bug or feature you're intending to fix. Even if you think it's relatively minor, it's helpful to know what people are working on. Mention in the initial issue that you are planning to work on that bug or feature so that it can be assigned to you. + +2. Follow the normal process of [forking](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/fork-a-repo) the project, and setup a new branch to work in. It's important that each group of changes be done in separate branches in order to ensure that a pull request only includes the commits related to that bug or feature. + +3. Go makes it very simple to ensure properly formatted code, so always run `go fmt` on your code before committing it. + +4. Do your best to have [well-formated commit messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) +for each change. This provides consistency throughout the project and ensures that commit messages are able to be formatted properly by various git tools. + +5. Finally, push the commits to your fork and submit a [pull request](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) + +### Once you've filed the PR: + +- One or more maintainers will use GitHub's review feature to review your PR. +- If the maintainer asks for any changes, edit your changes, push, and ask for another review. +- If the maintainer decides to suggest some improvements or alternatives, modify and make improvements. Once your changes are approved, one of the project maintainers will merge them. \ No newline at end of file diff --git a/vendor/github.com/axiomhq/hyperloglog/LICENSE b/vendor/github.com/axiomhq/hyperloglog/LICENSE new file mode 100644 index 0000000000..1ff7ea96ea --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2021, Axiom, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/axiomhq/hyperloglog/README.md b/vendor/github.com/axiomhq/hyperloglog/README.md new file mode 100644 index 0000000000..5affa52873 --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/README.md @@ -0,0 +1,51 @@ +# HyperLogLog - an algorithm for approximating the number of distinct elements + +[![GoDoc](https://godoc.org/github.com/axiomhq/hyperloglog?status.svg)](https://godoc.org/github.com/axiomhq/hyperloglog) [![Go Report Card](https://goreportcard.com/badge/github.com/axiomhq/hyperloglog)](https://goreportcard.com/report/github.com/axiomhq/hyperloglog) [![CircleCI](https://circleci.com/gh/axiomhq/hyperloglog/tree/master.svg?style=svg)](https://circleci.com/gh/axiomhq/hyperloglog/tree/master) + +An improved version of [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) for the count-distinct problem, approximating the number of distinct elements in a multiset. This implementation offers enhanced performance, flexibility, and simplicity while maintaining accuracy. + +## Note on Implementation History + +The initial version of this work (tagged as v0.1.0) was based on ["Better with fewer bits: Improving the performance of cardinality estimation of large data streams - Qingjun Xiao, You Zhou, Shigang Chen"](http://cse.seu.edu.cn/PersonalPage/csqjxiao/csqjxiao_files/papers/INFOCOM17.pdf). However, the current implementation has evolved significantly from this original basis, notably moving away from the tailcut method. + +## Current Implementation + +The current implementation is based on the LogLog-Beta algorithm, as described in: + +["LogLog-Beta and More: A New Algorithm for Cardinality Estimation Based on LogLog Counting"](https://arxiv.org/pdf/1612.02284) by Jason Qin, Denys Kim, and Yumei Tung (2016). + +Key features of the current implementation: +* **Metro hash** used instead of xxhash +* **Sparse representation** for lower cardinalities (like HyperLogLog++) +* **LogLog-Beta** for dynamic bias correction across all cardinalities +* **8-bit registers** for convenience and simplified implementation +* **Order-independent insertions and merging** for consistent results regardless of data input order +* **Removal of tailcut method** for a more straightforward approach +* **Flexible precision** allowing for 2^4 to 2^18 registers + +This implementation is now more straightforward, efficient, and flexible, while remaining backwards compatible with previous versions. It provides a balance between precision, memory usage, speed, and ease of use. + +## Precision and Memory Usage + +This implementation allows for creating HyperLogLog sketches with arbitrary precision between 2^4 and 2^18 registers. The memory usage scales with the number of registers: + +* Minimum (2^4 registers): 16 bytes +* Default (2^14 registers): 16 KB +* Maximum (2^18 registers): 256 KB + +Users can choose the precision that best fits their use case, balancing memory usage against estimation accuracy. + +## Note +A big thank you to Prof. Shigang Chen and his team at the University of Florida who are actively conducting research around "Big Network Data". + +## Contributing + +Kindly check our [contributing guide](https://github.com/axiomhq/hyperloglog/blob/main/Contributing.md) on how to propose bugfixes and improvements, and submitting pull requests to the project + +## License + +© Axiom, Inc., 2024 + +Distributed under MIT License (`The MIT License`). + +See [LICENSE](LICENSE) for more information. \ No newline at end of file diff --git a/vendor/github.com/axiomhq/hyperloglog/beta.go b/vendor/github.com/axiomhq/hyperloglog/beta.go new file mode 100644 index 0000000000..29d560136a --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/beta.go @@ -0,0 +1,273 @@ +package hyperloglog + +import ( + "fmt" + "math" +) + +var betaMap = map[uint8]func(float64) float64{ + 4: beta4, + 5: beta5, + 6: beta6, + 7: beta7, + 8: beta8, + 9: beta9, + 10: beta10, + 11: beta11, + 12: beta12, + 13: beta13, + 14: beta14, + 15: beta15, + 16: beta16, + 17: beta17, + 18: beta18, +} + +func beta(p uint8, ez float64) float64 { + f, ok := betaMap[p] + if !ok { + panic(fmt.Sprintf("invalid precision %d", p)) + } + return f(ez) +} + +/* +p=4 +[-0.582581413904517,-1.935300357560050,11.07932375 8035073,-22.131357446444323,22.505391846630037,-12 .000723834917984,3.220579408194167,-0.342225302271 235] +*/ +func beta4(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.582581413904517*ez + + -1.935300357560050*zl + + 11.079323758035073*math.Pow(zl, 2) + + -22.131357446444323*math.Pow(zl, 3) + + 22.505391846630037*math.Pow(zl, 4) + + -12.000723834917984*math.Pow(zl, 5) + + 3.220579408194167*math.Pow(zl, 6) + + -0.342225302271235*math.Pow(zl, 7) +} + +/* +p=5 +[-0.7518999460733967,-0.9590030077748760,5.5997371 322141607,-8.2097636999765520,6.5091254894472037,- 2.6830293734323729,0.5612891113138221,-0.046333162 2196545] +*/ +func beta5(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.7518999460733967*ez + + -0.9590030077748760*zl + + 5.5997371322141607*math.Pow(zl, 2) + + -8.2097636999765520*math.Pow(zl, 3) + + 6.5091254894472037*math.Pow(zl, 4) + + -2.6830293734323729*math.Pow(zl, 5) + + 0.5612891113138221*math.Pow(zl, 6) + + -0.0463331622196545*math.Pow(zl, 7) +} + +/* +p=6 +[29.8257900969619634,-31.3287083337725925,-10.5942 523036582283,-11.5720125689099618,3.81887543739074 92,-2.4160130328530811,0.4542208940970826,-0.05751 55452020420] +*/ +func beta6(ez float64) float64 { + zl := math.Log(ez + 1) + return 29.8257900969619634*ez + + -31.3287083337725925*zl + + -10.5942523036582283*math.Pow(zl, 2) + + -11.5720125689099618*math.Pow(zl, 3) + + 3.8188754373907492*math.Pow(zl, 4) + + -2.4160130328530811*math.Pow(zl, 5) + + 0.4542208940970826*math.Pow(zl, 6) + + -0.0575155452020420*math.Pow(zl, 7) +} + +/* +p=7 +[2.8102921290820060,-3.9780498518175995,1.31626800 41351582,-3.9252486335805901,2.0080835753946471,-0 .7527151937556955,0.1265569894242751,-0.0109946438726240] +*/ +func beta7(ez float64) float64 { + zl := math.Log(ez + 1) + return 2.8102921290820060*ez + + -3.9780498518175995*zl + + 1.3162680041351582*math.Pow(zl, 2) + + -3.9252486335805901*math.Pow(zl, 3) + + 2.0080835753946471*math.Pow(zl, 4) + + -0.7527151937556955*math.Pow(zl, 5) + + 0.1265569894242751*math.Pow(zl, 6) + + -0.0109946438726240*math.Pow(zl, 7) +} + +/* +p=8 +[1.00633544887550519,-2.00580666405112407,1.643697 49366514117,-2.70560809940566172,1.392099802442225 98,-0.46470374272183190,0.07384282377269775,-0.00578554885254223] +*/ +func beta8(ez float64) float64 { + zl := math.Log(ez + 1) + return 1.00633544887550519*ez + + -2.00580666405112407*zl + + 1.64369749366514117*math.Pow(zl, 2) + + -2.70560809940566172*math.Pow(zl, 3) + + 1.39209980244222598*math.Pow(zl, 4) + + -0.46470374272183190*math.Pow(zl, 5) + + 0.07384282377269775*math.Pow(zl, 6) + + -0.00578554885254223*math.Pow(zl, 7) +} + +/* +p=9 +[-0.09415657458167959,-0.78130975924550528,1.71514 946750712460,-1.73711250406516338,0.86441508489048 924,-0.23819027465047218,0.03343448400269076,-0.00 207858528178157] +*/ +func beta9(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.09415657458167959*ez + + -0.78130975924550528*zl + + 1.71514946750712460*math.Pow(zl, 2) + + -1.73711250406516338*math.Pow(zl, 3) + + 0.86441508489048924*math.Pow(zl, 4) + + -0.23819027465047218*math.Pow(zl, 5) + + 0.03343448400269076*math.Pow(zl, 6) + + -0.00207858528178157*math.Pow(zl, 7) +} + +/* +p=10 +[-0.25935400670790054,-0.52598301999805808,1.48933 034925876839,-1.29642714084993571,0.62284756217221615,-0.15672326770251041,0.02054415903878563,-0.00 112488483925502] +*/ +func beta10(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.25935400670790054*ez + + -0.52598301999805808*zl + + 1.48933034925876839*math.Pow(zl, 2) + + -1.29642714084993571*math.Pow(zl, 3) + + 0.62284756217221615*math.Pow(zl, 4) + + -0.15672326770251041*math.Pow(zl, 5) + + 0.02054415903878563*math.Pow(zl, 6) + + -0.00112488483925502*math.Pow(zl, 7) +} + +/* +p=11 +[-4.32325553856025e-01,-1.08450736399632e-01,6.091 56550741120e-01,-1.65687801845180e-02,-7.958293410 87617e-02,4.71830602102918e-02,-7.81372902346934e- 03,5.84268708489995e-04] +*/ +func beta11(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.432325553856025*ez + + -0.108450736399632*zl + + 0.609156550741120*math.Pow(zl, 2) + + -0.0165687801845180*math.Pow(zl, 3) + + -0.0795829341087617*math.Pow(zl, 4) + + 0.0471830602102918*math.Pow(zl, 5) + + -0.00781372902346934*math.Pow(zl, 6) + + 0.000584268708489995*math.Pow(zl, 7) +} + +/* +p=12 +[-3.84979202588598e-01,1.83162233114364e-01,1.3039 6688841854e-01,7.04838927629266e-02,-8.95893971464 453e-03,1.13010036741605e-02,-1.94285569591290e-03 ,2.25435774024964e-04] +*/ +func beta12(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.384979202588598*ez + + 0.183162233114364*zl + + 0.130396688841854*math.Pow(zl, 2) + + 0.0704838927629266*math.Pow(zl, 3) + + -0.0089589397146453*math.Pow(zl, 4) + + 0.0113010036741605*math.Pow(zl, 5) + + -0.00194285569591290*math.Pow(zl, 6) + + 0.000225435774024964*math.Pow(zl, 7) +} + +/* +p=13 +[-0.41655270946462997,-0.22146677040685156,0.38862 131236999947,0.45340979746062371,-0.36264738324476 375,0.12304650053558529,-0.01701540384555510,0.001 02750367080838] +*/ +func beta13(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.41655270946462997*ez + + -0.22146677040685156*zl + + 0.38862131236999947*math.Pow(zl, 2) + + 0.45340979746062371*math.Pow(zl, 3) + + -0.36264738324476375*math.Pow(zl, 4) + + 0.12304650053558529*math.Pow(zl, 5) + + -0.01701540384555510*math.Pow(zl, 6) + + 0.00102750367080838*math.Pow(zl, 7) +} + +/* +p=14 +[-3.71009760230692e-01,9.78811941207509e-03,1.8579 6293324165e-01,2.03015527328432e-01,-1.16710521803 686e-01,4.31106699492820e-02,-5.99583540511831e-03 ,4.49704299509437e-04] +*/ + +func beta14(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.371009760230692*ez + + 0.00978811941207509*zl + + 0.185796293324165*math.Pow(zl, 2) + + 0.203015527328432*math.Pow(zl, 3) + + -0.116710521803686*math.Pow(zl, 4) + + 0.0431106699492820*math.Pow(zl, 5) + + -0.00599583540511831*math.Pow(zl, 6) + + 0.000449704299509437*math.Pow(zl, 7) +} + +/* +p=15 +[-0.38215145543875273,-0.89069400536090837,0.37602 335774678869,0.99335977440682377,-0.65577441638318 956,0.18332342129703610,-0.02241529633062872,0.001 21399789330194] +*/ +func beta15(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.38215145543875273*ez + + -0.89069400536090837*zl + + 0.37602335774678869*math.Pow(zl, 2) + + 0.99335977440682377*math.Pow(zl, 3) + + -0.65577441638318956*math.Pow(zl, 4) + + 0.18332342129703610*math.Pow(zl, 5) + + -0.02241529633062872*math.Pow(zl, 6) + + 0.00121399789330194*math.Pow(zl, 7) +} + +/* +p=16 +[-0.37331876643753059,-1.41704077448122989,0.407291 84796612533,1.56152033906584164,-0.99242233534286128,0.26064681399483092,-0.03053811369682807,0.00155770210179105] +*/ +func beta16(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.37331876643753059*ez + + -1.41704077448122989*zl + + 0.40729184796612533*math.Pow(zl, 2) + + 1.56152033906584164*math.Pow(zl, 3) + + -0.99242233534286128*math.Pow(zl, 4) + + 0.26064681399483092*math.Pow(zl, 5) + + -0.03053811369682807*math.Pow(zl, 6) + + 0.00155770210179105*math.Pow(zl, 7) +} + +/* +p=17 +[-0.36775502299404605,0.53831422351377967,0.769702 89278767923,0.55002583586450560,-0.745755882611469 41,0.25711835785821952,-0.03437902606864149,0.0018 5949146371616] +*/ +func beta17(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.36775502299404605*ez + + 0.53831422351377967*zl + + 0.76970289278767923*math.Pow(zl, 2) + + 0.55002583586450560*math.Pow(zl, 3) + + -0.74575588261146941*math.Pow(zl, 4) + + 0.25711835785821952*math.Pow(zl, 5) + + -0.03437902606864149*math.Pow(zl, 6) + + 0.00185949146371616*math.Pow(zl, 7) +} + +/* +p=18 +[-0.36479623325960542,0.99730412328635032,1.553543 86230081221,1.25932677198028919,-1.533259482091101 63,0.47801042200056593,-0.05951025172951174,0.0029 1076804642205] +*/ +func beta18(ez float64) float64 { + zl := math.Log(ez + 1) + return -0.36479623325960542*ez + + 0.99730412328635032*zl + + 1.55354386230081221*math.Pow(zl, 2) + + 1.25932677198028919*math.Pow(zl, 3) + + -1.53325948209110163*math.Pow(zl, 4) + + 0.47801042200056593*math.Pow(zl, 5) + + -0.05951025172951174*math.Pow(zl, 6) + + 0.00291076804642205*math.Pow(zl, 7) +} diff --git a/vendor/github.com/axiomhq/hyperloglog/compressed.go b/vendor/github.com/axiomhq/hyperloglog/compressed.go new file mode 100644 index 0000000000..ba55b76f4b --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/compressed.go @@ -0,0 +1,168 @@ +package hyperloglog + +import ( + "encoding/binary" + "slices" +) + +// Original author of this file is github.com/clarkduvall/hyperloglog + +type iterator struct { + i int + last uint32 + v *compressedList +} + +func (iter *iterator) Next() uint32 { + n, i := iter.v.decode(iter.i, iter.last) + iter.last = n + iter.i = i + return n +} + +func (iter *iterator) Peek() (uint32, int) { + return iter.v.decode(iter.i, iter.last) +} + +func (iter *iterator) Advance(last uint32, i int) { + iter.last = last + iter.i = i +} + +func (iter iterator) HasNext() bool { + return iter.i < iter.v.Len() +} + +type compressedList struct { + count uint32 + last uint32 + b variableLengthList +} + +func (v *compressedList) Clone() *compressedList { + if v == nil { + return nil + } + + newV := &compressedList{ + count: v.count, + last: v.last, + } + + newV.b = make(variableLengthList, len(v.b)) + copy(newV.b, v.b) + return newV +} + +func (v *compressedList) AppendBinary(data []byte) ([]byte, error) { + // At least 4 bytes for the two fixed sized values + data = slices.Grow(data, 4+4) + + // Marshal the count and last values. + data = append(data, + // Number of items in the list. + byte(v.count>>24), + byte(v.count>>16), + byte(v.count>>8), + byte(v.count), + // The last item in the list. + byte(v.last>>24), + byte(v.last>>16), + byte(v.last>>8), + byte(v.last), + ) + + // Append the variableLengthList + return v.b.AppendBinary(data) +} + +func (v *compressedList) UnmarshalBinary(data []byte) error { + if len(data) < 12 { + return ErrorTooShort + } + + // Set the count. + v.count, data = binary.BigEndian.Uint32(data[:4]), data[4:] + + // Set the last value. + v.last, data = binary.BigEndian.Uint32(data[:4]), data[4:] + + // Set the list. + sz, data := binary.BigEndian.Uint32(data[:4]), data[4:] + v.b = make([]uint8, sz) + if uint32(len(data)) < sz { + return ErrorTooShort + } + for i := uint32(0); i < sz; i++ { + v.b[i] = data[i] + } + return nil +} + +func newCompressedList(capacity int) *compressedList { + v := &compressedList{} + v.b = make(variableLengthList, 0, capacity) + return v +} + +func (v *compressedList) Len() int { + return len(v.b) +} + +func (v *compressedList) decode(i int, last uint32) (uint32, int) { + n, i := v.b.decode(i) + return n + last, i +} + +func (v *compressedList) Append(x uint32) { + v.count++ + v.b = v.b.Append(x - v.last) + v.last = x +} + +func (v *compressedList) Iter() iterator { + return iterator{0, 0, v} +} + +type variableLengthList []uint8 + +func (v variableLengthList) AppendBinary(data []byte) ([]byte, error) { + // 4 bytes for the size of the list, and a byte for each element in the + // list. + data = slices.Grow(data, 4+len(v)) + + // Length of the list. We only need 32 bits because the size of the set + // couldn't exceed that on 32 bit architectures. + sz := len(v) + data = append(data, + byte(sz>>24), + byte(sz>>16), + byte(sz>>8), + byte(sz), + ) + + // Marshal each element in the list. + for i := 0; i < sz; i++ { + data = append(data, v[i]) + } + + return data, nil +} + +func (v variableLengthList) decode(i int) (uint32, int) { + var x uint32 + j := i + for ; v[j]&0x80 != 0; j++ { + x |= uint32(v[j]&0x7f) << (uint(j-i) * 7) + } + x |= uint32(v[j]) << (uint(j-i) * 7) + return x, j + 1 +} + +func (v variableLengthList) Append(x uint32) variableLengthList { + for x&0xffffff80 != 0 { + v = append(v, uint8((x&0x7f)|0x80)) + x >>= 7 + } + return append(v, uint8(x&0x7f)) +} diff --git a/vendor/github.com/axiomhq/hyperloglog/hyperloglog.go b/vendor/github.com/axiomhq/hyperloglog/hyperloglog.go new file mode 100644 index 0000000000..0ebfd124f1 --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/hyperloglog.go @@ -0,0 +1,354 @@ +package hyperloglog + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "slices" +) + +const ( + pp = uint8(25) + mp = uint32(1) << pp + version = 2 +) + +type Sketch struct { + p uint8 + m uint32 + alpha float64 + tmpSet set + sparseList *compressedList + regs []uint8 +} + +// New returns a HyperLogLog Sketch with 2^14 registers (precision 14) +func New() *Sketch { return New14() } + +// New14 returns a HyperLogLog Sketch with 2^14 registers (precision 14) +func New14() *Sketch { return newSketchNoError(14, true) } + +// New16 returns a HyperLogLog Sketch with 2^16 registers (precision 16) +func New16() *Sketch { return newSketchNoError(16, true) } + +// NewNoSparse returns a HyperLogLog Sketch with 2^14 registers (precision 14) that will not use a sparse representation +func NewNoSparse() *Sketch { return newSketchNoError(14, false) } + +// New16NoSparse returns a HyperLogLog Sketch with 2^16 registers (precision 16) that will not use a sparse representation +func New16NoSparse() *Sketch { return newSketchNoError(16, false) } + +func newSketchNoError(precision uint8, sparse bool) *Sketch { + sk, _ := NewSketch(precision, sparse) + return sk +} + +func NewSketch(precision uint8, sparse bool) (*Sketch, error) { + if precision < 4 || precision > 18 { + return nil, fmt.Errorf("p has to be >= 4 and <= 18") + } + m := uint32(1) << precision + s := &Sketch{ + m: m, + p: precision, + alpha: alpha(float64(m)), + } + if sparse { + s.tmpSet = makeSet(0) + s.sparseList = newCompressedList(0) + } else { + s.regs = make([]uint8, m) + } + return s, nil +} + +func (sk *Sketch) sparse() bool { return sk.sparseList != nil } + +// Clone returns a deep copy of sk. +func (sk *Sketch) Clone() *Sketch { + clone := *sk + clone.regs = append([]uint8(nil), sk.regs...) + clone.tmpSet = sk.tmpSet.Clone() + clone.sparseList = sk.sparseList.Clone() + return &clone +} + +func (sk *Sketch) maybeToNormal() { + if uint32(sk.tmpSet.Len())*100 > sk.m { + sk.mergeSparse() + if uint32(sk.sparseList.Len()) > sk.m { + sk.toNormal() + } + } +} + +func (sk *Sketch) Merge(other *Sketch) error { + if other == nil { + return nil + } + if sk.p != other.p { + return errors.New("precisions must be equal") + } + + if sk.sparse() && other.sparse() { + sk.mergeSparseSketch(other) + } else { + sk.mergeDenseSketch(other) + } + return nil +} + +func (sk *Sketch) mergeSparseSketch(other *Sketch) { + sk.tmpSet.Merge(other.tmpSet) + for iter := other.sparseList.Iter(); iter.HasNext(); { + sk.tmpSet.add(iter.Next()) + } + sk.maybeToNormal() +} + +func (sk *Sketch) mergeDenseSketch(other *Sketch) { + if sk.sparse() { + sk.toNormal() + } + + if other.sparse() { + other.tmpSet.ForEach(func(k uint32) { + i, r := decodeHash(k, other.p, pp) + sk.insert(i, r) + }) + for iter := other.sparseList.Iter(); iter.HasNext(); { + i, r := decodeHash(iter.Next(), other.p, pp) + sk.insert(i, r) + } + } else { + for i, v := range other.regs { + if v > sk.regs[i] { + sk.regs[i] = v + } + } + } +} + +func (sk *Sketch) toNormal() { + if sk.tmpSet.Len() > 0 { + sk.mergeSparse() + } + + sk.regs = make([]uint8, sk.m) + for iter := sk.sparseList.Iter(); iter.HasNext(); { + i, r := decodeHash(iter.Next(), sk.p, pp) + sk.insert(i, r) + } + + sk.tmpSet = nilSet + sk.sparseList = nil +} + +func (sk *Sketch) insert(i uint32, r uint8) { sk.regs[i] = max(r, sk.regs[i]) } +func (sk *Sketch) Insert(e []byte) { sk.InsertHash(hash(e)) } + +func (sk *Sketch) InsertHash(x uint64) { + if sk.sparse() { + if sk.tmpSet.add(encodeHash(x, sk.p, pp)) { + sk.maybeToNormal() + } + return + } + i, r := getPosVal(x, sk.p) + sk.insert(uint32(i), r) +} + +func (sk *Sketch) Estimate() uint64 { + if sk.sparse() { + sk.mergeSparse() + return uint64(linearCount(mp, mp-sk.sparseList.count)) + } + + sum, ez := sumAndZeros(sk.regs) + m := float64(sk.m) + + est := sk.alpha * m * (m - ez) / (sum + beta(sk.p, ez)) + return uint64(est + 0.5) +} + +func (sk *Sketch) mergeSparse() { + if sk.tmpSet.Len() == 0 { + return + } + + keys := make([]uint32, 0, sk.tmpSet.Len()) + sk.tmpSet.ForEach(func(k uint32) { + keys = append(keys, k) + }) + slices.Sort(keys) + + newList := newCompressedList(4*sk.tmpSet.Len() + sk.sparseList.Len()) + for iter, i := sk.sparseList.Iter(), 0; iter.HasNext() || i < len(keys); { + if !iter.HasNext() { + newList.Append(keys[i]) + i++ + continue + } + + if i >= len(keys) { + newList.Append(iter.Next()) + continue + } + + x1, adv := iter.Peek() + x2 := keys[i] + if x1 == x2 { + newList.Append(x1) + iter.Advance(x1, adv) + i++ + } else if x1 > x2 { + newList.Append(x2) + i++ + } else { + newList.Append(x1) + iter.Advance(x1, adv) + } + } + + sk.sparseList = newList + sk.tmpSet = makeSet(0) +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +// +// When the result will be appended to another buffer, consider using +// AppendBinary to avoid additional allocations and copying. +func (sk *Sketch) MarshalBinary() (data []byte, err error) { + return sk.AppendBinary(nil) +} + +// AppendBinary implements the encoding.BinaryAppender interface. +func (sk *Sketch) AppendBinary(data []byte) ([]byte, error) { + data = slices.Grow(data, 8+len(sk.regs)) + // Marshal a version marker. + data = append(data, version) + // Marshal p. + data = append(data, sk.p) + // Marshal b + data = append(data, 0) + + if sk.sparse() { + // It's using the sparse Sketch. + data = append(data, byte(1)) + + // Add the tmp_set + data, err := sk.tmpSet.AppendBinary(data) + if err != nil { + return nil, err + } + + // Add the sparse Sketch + return sk.sparseList.AppendBinary(data) + } + + // It's using the dense Sketch. + data = append(data, byte(0)) + + // Add the dense sketch Sketch. + sz := len(sk.regs) + data = append(data, + byte(sz>>24), + byte(sz>>16), + byte(sz>>8), + byte(sz), + ) + + // Marshal each element in the list. + for _, v := range sk.regs { + data = append(data, byte(v)) + } + + return data, nil +} + +// ErrorTooShort is an error that UnmarshalBinary try to parse too short +// binary. +var ErrorTooShort = errors.New("too short binary") + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +func (sk *Sketch) UnmarshalBinary(data []byte) error { + if len(data) < 8 { + return ErrorTooShort + } + + // Unmarshal version. We may need this in the future if we make + // non-compatible changes. + v := data[0] + + // Unmarshal p. + p := data[1] + + // Unmarshal b. + b := data[2] + + // Determine if we need a sparse Sketch + sparse := data[3] == byte(1) + + // Make a newSketch Sketch if the precision doesn't match or if the Sketch was used + if sk.p != p || sk.regs != nil || sk.tmpSet.Len() > 0 || (sk.sparseList != nil && sk.sparseList.Len() > 0) { + newh, err := NewSketch(p, sparse) + if err != nil { + return err + } + *sk = *newh + } + + // h is now initialised with the correct p. We just need to fill the + // rest of the details out. + if sparse { + // Using the sparse Sketch. + + // Unmarshal the tmp_set. + tssz := binary.BigEndian.Uint32(data[4:8]) + sk.tmpSet = makeSet(int(tssz)) + + // We need to unmarshal tssz values in total, and each value requires us + // to read 4 bytes. + tsLastByte := int((tssz * 4) + 8) + for i := 8; i < tsLastByte; i += 4 { + k := binary.BigEndian.Uint32(data[i : i+4]) + sk.tmpSet.add(k) + } + + // Unmarshal the sparse Sketch. + return sk.sparseList.UnmarshalBinary(data[tsLastByte:]) + } + + // Using the dense Sketch. + sk.sparseList = nil + sk.tmpSet = nilSet + + if v == 1 { + return sk.unmarshalBinaryV1(data[8:], b) + } + return sk.unmarshalBinaryV2(data) +} + +func sumAndZeros(regs []uint8) (res, ez float64) { + for _, v := range regs { + if v == 0 { + ez++ + } + res += 1.0 / math.Pow(2.0, float64(v)) + } + return res, ez +} + +func (sk *Sketch) unmarshalBinaryV1(data []byte, b uint8) error { + sk.regs = make([]uint8, len(data)*2) + for i, v := range data { + sk.regs[i*2] = uint8((v >> 4)) + b + sk.regs[i*2+1] = uint8((v<<4)>>4) + b + } + return nil +} + +func (sk *Sketch) unmarshalBinaryV2(data []byte) error { + sk.regs = data[8:] + return nil +} diff --git a/vendor/github.com/axiomhq/hyperloglog/sparse.go b/vendor/github.com/axiomhq/hyperloglog/sparse.go new file mode 100644 index 0000000000..487d4083cf --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/sparse.go @@ -0,0 +1,112 @@ +package hyperloglog + +import ( + "math/bits" + "slices" + + "github.com/kamstrup/intmap" +) + +func getIndex(k uint32, p, pp uint8) uint32 { + if k&1 == 1 { + return bextr32(k, 32-p, p) + } + return bextr32(k, pp-p+1, p) +} + +// Encode a hash to be used in the sparse representation. +func encodeHash(x uint64, p, pp uint8) uint32 { + idx := uint32(bextr(x, 64-pp, pp)) + if bextr(x, 64-pp, pp-p) == 0 { + zeros := bits.LeadingZeros64((bextr(x, 0, 64-pp)<>24), + byte(sl>>16), + byte(sl>>8), + byte(sl), + ) + + // Marshal each element in the set. + s.m.ForEach(func(k uint32) bool { + data = append(data, + byte(k>>24), + byte(k>>16), + byte(k>>8), + byte(k), + ) + return true + }) + + return data, nil +} diff --git a/vendor/github.com/axiomhq/hyperloglog/utils.go b/vendor/github.com/axiomhq/hyperloglog/utils.go new file mode 100644 index 0000000000..c032c88d4b --- /dev/null +++ b/vendor/github.com/axiomhq/hyperloglog/utils.go @@ -0,0 +1,46 @@ +package hyperloglog + +import ( + "math" + "math/bits" + + metro "github.com/dgryski/go-metro" +) + +var hash = hashFunc + +func alpha(m float64) float64 { + switch m { + case 16: + return 0.673 + case 32: + return 0.697 + case 64: + return 0.709 + } + return 0.7213 / (1 + 1.079/m) +} + +func getPosVal(x uint64, p uint8) (uint64, uint8) { + i := bextr(x, 64-p, p) // {x63,...,x64-p} + w := x<

> start) & ((1 << length) - 1) +} + +func bextr32(v uint32, start, length uint8) uint32 { + return (v >> start) & ((1 << length) - 1) +} + +func hashFunc(e []byte) uint64 { + return metro.Hash64(e, 1337) +} diff --git a/vendor/github.com/kamstrup/intmap/.gitignore b/vendor/github.com/kamstrup/intmap/.gitignore new file mode 100644 index 0000000000..1377554ebe --- /dev/null +++ b/vendor/github.com/kamstrup/intmap/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/vendor/github.com/kamstrup/intmap/LICENSE b/vendor/github.com/kamstrup/intmap/LICENSE new file mode 100644 index 0000000000..1eac633b0c --- /dev/null +++ b/vendor/github.com/kamstrup/intmap/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2016, Brent Pedersen - Bioinformatics +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/kamstrup/intmap/README.md b/vendor/github.com/kamstrup/intmap/README.md new file mode 100644 index 0000000000..e1a1e7003a --- /dev/null +++ b/vendor/github.com/kamstrup/intmap/README.md @@ -0,0 +1,52 @@ +Fast hashmap with integer keys for Golang + +[![GoDoc](https://godoc.org/github.com/kamstrup/intmap?status.svg)](https://godoc.org/github.com/kamstrup/intmap) +[![Go Report Card](https://goreportcard.com/badge/github.com/kamstrup/intmap)](https://goreportcard.com/report/github.com/kamstrup/intmap) + +# intmap + + import "github.com/kamstrup/intmap" + +Package intmap is a fast hashmap implementation for Golang, specialized for maps with integer type keys. +The values can be of any type. + +It is a full port of https://github.com/brentp/intintmap to use type parameters (aka generics). + +It interleaves keys and values in the same underlying array to improve locality. +This is also known as open addressing with linear probing. + +It is up to 3X faster than the builtin map: +``` +name time/op +Map64Fill-8 201ms ยฑ 5% +IntIntMapFill-8 207ms ยฑ31% +StdMapFill-8 371ms ยฑ11% +Map64Get10PercentHitRate-8 148ยตs ยฑ40% +IntIntMapGet10PercentHitRate-8 171ยตs ยฑ50% +StdMapGet10PercentHitRate-8 171ยตs ยฑ33% +Map64Get100PercentHitRate-8 4.50ms ยฑ 5% +IntIntMapGet100PercentHitRate-8 4.82ms ยฑ 6% +StdMapGet100PercentHitRate-8 15.5ms ยฑ32% +``` + +## Usage + +```go +m := intmap.New[int64,int64](32768) +m.Put(int64(1234), int64(-222)) +m.Put(int64(123), int64(33)) + +v, ok := m.Get(int64(222)) +v, ok := m.Get(int64(333)) + +m.Del(int64(222)) +m.Del(int64(333)) + +fmt.Println(m.Len()) + +m.ForEach(func(k int64, v int64) { + fmt.Printf("key: %d, value: %d\n", k, v) +}) + +m.Clear() // all gone, but buffers kept +``` diff --git a/vendor/github.com/kamstrup/intmap/map64.go b/vendor/github.com/kamstrup/intmap/map64.go new file mode 100644 index 0000000000..20926b95fb --- /dev/null +++ b/vendor/github.com/kamstrup/intmap/map64.go @@ -0,0 +1,442 @@ +// Package intmap contains a fast hashmap implementation for maps with keys of any integer type +package intmap + +import ( + "iter" + "math" +) + +// IntKey is a type constraint for values that can be used as keys in Map +type IntKey interface { + ~int | ~uint | ~int64 | ~uint64 | ~int32 | ~uint32 | ~int16 | ~uint16 | ~int8 | ~uint8 | ~uintptr +} + +type pair[K IntKey, V any] struct { + K K + V V +} + +const fillFactor64 = 0.7 + +func phiMix64(x int) int { + h := int64(x) * int64(0x9E3779B9) + return int(h ^ (h >> 16)) +} + +// Map is a hashmap where the keys are some any integer type. +// It is valid to call methods that read a nil map, similar to a standard Go map. +// Methods valid on a nil map are Has, Get, Len, and ForEach. +type Map[K IntKey, V any] struct { + data []pair[K, V] // key-value pairs + size int + + zeroVal V // value of 'zero' key + hasZeroKey bool // do we have 'zero' key in the map? +} + +// New creates a new map with keys being any integer subtype. +// The map can store up to the given capacity before reallocation and rehashing occurs. +func New[K IntKey, V any](capacity int) *Map[K, V] { + return &Map[K, V]{ + data: make([]pair[K, V], arraySize(capacity, fillFactor64)), + } +} + +// Has checks if the given key exists in the map. +// Calling this method on a nil map will return false. +func (m *Map[K, V]) Has(key K) bool { + if m == nil { + return false + } + + if key == K(0) { + return m.hasZeroKey + } + + idx := m.startIndex(key) + p := m.data[idx] + + if p.K == K(0) { // end of chain already + return false + } + if p.K == key { // we check zero prior to this call + return true + } + + // hash collision, seek next hash match, bailing on first empty + for { + idx = m.nextIndex(idx) + p = m.data[idx] + if p.K == K(0) { + return false + } + if p.K == key { + return true + } + } +} + +// Get returns the value if the key is found. +// If you just need to check for existence it is easier to use Has. +// Calling this method on a nil map will return the zero value for V and false. +func (m *Map[K, V]) Get(key K) (V, bool) { + if m == nil { + var zero V + return zero, false + } + + if key == K(0) { + if m.hasZeroKey { + return m.zeroVal, true + } + var zero V + return zero, false + } + + idx := m.startIndex(key) + p := m.data[idx] + + if p.K == K(0) { // end of chain already + var zero V + return zero, false + } + if p.K == key { // we check zero prior to this call + return p.V, true + } + + // hash collision, seek next hash match, bailing on first empty + for { + idx = m.nextIndex(idx) + p = m.data[idx] + if p.K == K(0) { + var zero V + return zero, false + } + if p.K == key { + return p.V, true + } + } +} + +// Put adds or updates key with value val. +func (m *Map[K, V]) Put(key K, val V) { + if key == K(0) { + if !m.hasZeroKey { + m.size++ + } + m.zeroVal = val + m.hasZeroKey = true + return + } + + idx := m.startIndex(key) + p := &m.data[idx] + + if p.K == K(0) { // end of chain already + p.K = key + p.V = val + if m.size >= m.sizeThreshold() { + m.rehash() + } else { + m.size++ + } + return + } else if p.K == key { // overwrite existing value + p.V = val + return + } + + // hash collision, seek next empty or key match + for { + idx = m.nextIndex(idx) + p = &m.data[idx] + + if p.K == K(0) { + p.K = key + p.V = val + if m.size >= m.sizeThreshold() { + m.rehash() + } else { + m.size++ + } + return + } else if p.K == key { + p.V = val + return + } + } +} + +// PutIfNotExists adds the key-value pair only if the key does not already exist +// in the map, and returns the current value associated with the key and a boolean +// indicating whether the value was newly added or not. +func (m *Map[K, V]) PutIfNotExists(key K, val V) (V, bool) { + if key == K(0) { + if m.hasZeroKey { + return m.zeroVal, false + } + m.zeroVal = val + m.hasZeroKey = true + m.size++ + return val, true + } + + idx := m.startIndex(key) + p := &m.data[idx] + + if p.K == K(0) { // end of chain already + p.K = key + p.V = val + m.size++ + if m.size >= m.sizeThreshold() { + m.rehash() + } + return val, true + } else if p.K == key { + return p.V, false + } + + // hash collision, seek next hash match, bailing on first empty + for { + idx = m.nextIndex(idx) + p = &m.data[idx] + + if p.K == K(0) { + p.K = key + p.V = val + m.size++ + if m.size >= m.sizeThreshold() { + m.rehash() + } + return val, true + } else if p.K == key { + return p.V, false + } + } +} + +// ForEach iterates through key-value pairs in the map while the function f returns true. +// This method returns immediately if invoked on a nil map. +// +// The iteration order of a Map is not defined, so please avoid relying on it. +func (m *Map[K, V]) ForEach(f func(K, V) bool) { + if m == nil { + return + } + + if m.hasZeroKey && !f(K(0), m.zeroVal) { + return + } + forEach64(m.data, f) +} + +// All returns an iterator over key-value pairs from m. +// The iterator returns immediately if invoked on a nil map. +// +// The iteration order of a Map is not defined, so please avoid relying on it. +func (m *Map[K, V]) All() iter.Seq2[K, V] { + return m.ForEach +} + +// Keys returns an iterator over keys in m. +// The iterator returns immediately if invoked on a nil map. +// +// The iteration order of a Map is not defined, so please avoid relying on it. +func (m *Map[K, V]) Keys() iter.Seq[K] { + return func(yield func(k K) bool) { + if m == nil { + return + } + + if m.hasZeroKey && !yield(K(0)) { + return + } + + for _, p := range m.data { + if p.K != K(0) && !yield(p.K) { + return + } + } + } +} + +// Values returns an iterator over values in m. +// The iterator returns immediately if invoked on a nil map. +// +// The iteration order of a Map is not defined, so please avoid relying on it. +func (m *Map[K, V]) Values() iter.Seq[V] { + return func(yield func(v V) bool) { + if m == nil { + return + } + + if m.hasZeroKey && !yield(m.zeroVal) { + return + } + + for _, p := range m.data { + if p.K != K(0) && !yield(p.V) { + return + } + } + } +} + +// Clear removes all items from the map, but keeps the internal buffers for reuse. +func (m *Map[K, V]) Clear() { + var zero V + m.hasZeroKey = false + m.zeroVal = zero + + // compiles down to runtime.memclr() + for i := range m.data { + m.data[i] = pair[K, V]{} + } + + m.size = 0 +} + +func (m *Map[K, V]) rehash() { + oldData := m.data + m.data = make([]pair[K, V], 2*len(m.data)) + + // reset size + if m.hasZeroKey { + m.size = 1 + } else { + m.size = 0 + } + + forEach64(oldData, func(k K, v V) bool { + m.Put(k, v) + return true + }) +} + +// Len returns the number of elements in the map. +// The length of a nil map is defined to be zero. +func (m *Map[K, V]) Len() int { + if m == nil { + return 0 + } + + return m.size +} + +func (m *Map[K, V]) sizeThreshold() int { + return int(math.Floor(float64(len(m.data)) * fillFactor64)) +} + +func (m *Map[K, V]) startIndex(key K) int { + return phiMix64(int(key)) & (len(m.data) - 1) +} + +func (m *Map[K, V]) nextIndex(idx int) int { + return (idx + 1) & (len(m.data) - 1) +} + +func forEach64[K IntKey, V any](pairs []pair[K, V], f func(k K, v V) bool) { + for _, p := range pairs { + if p.K != K(0) && !f(p.K, p.V) { + return + } + } +} + +// Del deletes a key and its value, returning true iff the key was found +func (m *Map[K, V]) Del(key K) bool { + if key == K(0) { + if m.hasZeroKey { + m.hasZeroKey = false + m.size-- + return true + } + return false + } + + idx := m.startIndex(key) + p := m.data[idx] + + if p.K == key { + // any keys that were pushed back needs to be shifted nack into the empty slot + // to avoid breaking the chain + m.shiftKeys(idx) + m.size-- + return true + } else if p.K == K(0) { // end of chain already + return false + } + + for { + idx = m.nextIndex(idx) + p = m.data[idx] + + if p.K == key { + // any keys that were pushed back needs to be shifted nack into the empty slot + // to avoid breaking the chain + m.shiftKeys(idx) + m.size-- + return true + } else if p.K == K(0) { + return false + } + + } +} + +func (m *Map[K, V]) shiftKeys(idx int) int { + // Shift entries with the same hash. + // We need to do this on deletion to ensure we don't have zeroes in the hash chain + for { + var p pair[K, V] + lastIdx := idx + idx = m.nextIndex(idx) + for { + p = m.data[idx] + if p.K == K(0) { + m.data[lastIdx] = pair[K, V]{} + return lastIdx + } + + slot := m.startIndex(p.K) + if lastIdx <= idx { + if lastIdx >= slot || slot > idx { + break + } + } else { + if lastIdx >= slot && slot > idx { + break + } + } + idx = m.nextIndex(idx) + } + m.data[lastIdx] = p + } +} + +func nextPowerOf2(x uint32) uint32 { + if x == math.MaxUint32 { + return x + } + + if x == 0 { + return 1 + } + + x-- + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + + return x + 1 +} + +func arraySize(exp int, fill float64) int { + s := nextPowerOf2(uint32(math.Ceil(float64(exp) / fill))) + if s < 2 { + s = 2 + } + return int(s) +} diff --git a/vendor/github.com/kamstrup/intmap/set.go b/vendor/github.com/kamstrup/intmap/set.go new file mode 100644 index 0000000000..b81ce224b6 --- /dev/null +++ b/vendor/github.com/kamstrup/intmap/set.go @@ -0,0 +1,59 @@ +package intmap + +import "iter" + +// Set is a specialization of Map modelling a set of integers. +// Like Map, methods that read from the set are valid on the nil Set. +// This include Has, Len, and ForEach. +type Set[K IntKey] Map[K, struct{}] + +// NewSet creates a new Set with a given initial capacity. +func NewSet[K IntKey](capacity int) *Set[K] { + return (*Set[K])(New[K, struct{}](capacity)) +} + +// Add an element to the set. Returns true if the element was not already present. +func (s *Set[K]) Add(k K) bool { + _, found := (*Map[K, struct{}])(s).PutIfNotExists(k, struct{}{}) + return found +} + +// Del deletes a key, returning true iff the key was found +func (s *Set[K]) Del(k K) bool { + return (*Map[K, struct{}])(s).Del(k) +} + +// Clear removes all items from the Set, but keeps the internal buffers for reuse. +func (s *Set[K]) Clear() { + (*Map[K, struct{}])(s).Clear() +} + +// Has returns true if the key is in the set. +// If the set is nil this method always return false. +func (s *Set[K]) Has(k K) bool { + return (*Map[K, struct{}])(s).Has(k) +} + +// Len returns the number of elements in the set. +// If the set is nil this method return 0. +func (s *Set[K]) Len() int { + return (*Map[K, struct{}])(s).Len() +} + +// ForEach iterates over the elements in the set while the visit function returns true. +// This method returns immediately if the set is nil. +// +// The iteration order of a Set is not defined, so please avoid relying on it. +func (s *Set[K]) ForEach(visit func(k K) bool) { + (*Map[K, struct{}])(s).ForEach(func(k K, _ struct{}) bool { + return visit(k) + }) +} + +// All returns an iterator over keys from the set. +// The iterator returns immediately if the set is nil. +// +// The iteration order of a Set is not defined, so please avoid relying on it. +func (s *Set[K]) All() iter.Seq[K] { + return s.ForEach +} diff --git a/vendor/github.com/mattn/go-runewidth/LICENSE b/vendor/github.com/mattn/go-runewidth/LICENSE new file mode 100644 index 0000000000..91b5cef30e --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/mattn/go-runewidth/README.md b/vendor/github.com/mattn/go-runewidth/README.md new file mode 100644 index 0000000000..5e2cfd98cb --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/README.md @@ -0,0 +1,27 @@ +go-runewidth +============ + +[![Build Status](https://github.com/mattn/go-runewidth/workflows/test/badge.svg?branch=master)](https://github.com/mattn/go-runewidth/actions?query=workflow%3Atest) +[![Codecov](https://codecov.io/gh/mattn/go-runewidth/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-runewidth) +[![GoDoc](https://godoc.org/github.com/mattn/go-runewidth?status.svg)](http://godoc.org/github.com/mattn/go-runewidth) +[![Go Report Card](https://goreportcard.com/badge/github.com/mattn/go-runewidth)](https://goreportcard.com/report/github.com/mattn/go-runewidth) + +Provides functions to get fixed width of the character or string. + +Usage +----- + +```go +runewidth.StringWidth("ใคใฎใ โ˜†HIRO") == 12 +``` + + +Author +------ + +Yasuhiro Matsumoto + +License +------- + +under the MIT License: http://mattn.mit-license.org/2013 diff --git a/vendor/github.com/mattn/go-runewidth/runewidth.go b/vendor/github.com/mattn/go-runewidth/runewidth.go new file mode 100644 index 0000000000..7dfbb3be91 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth.go @@ -0,0 +1,358 @@ +package runewidth + +import ( + "os" + "strings" + + "github.com/rivo/uniseg" +) + +//go:generate go run script/generate.go + +var ( + // EastAsianWidth will be set true if the current locale is CJK + EastAsianWidth bool + + // StrictEmojiNeutral should be set false if handle broken fonts + StrictEmojiNeutral bool = true + + // DefaultCondition is a condition in current locale + DefaultCondition = &Condition{ + EastAsianWidth: false, + StrictEmojiNeutral: true, + } +) + +func init() { + handleEnv() +} + +func handleEnv() { + env := os.Getenv("RUNEWIDTH_EASTASIAN") + if env == "" { + EastAsianWidth = IsEastAsian() + } else { + EastAsianWidth = env == "1" + } + // update DefaultCondition + if DefaultCondition.EastAsianWidth != EastAsianWidth { + DefaultCondition.EastAsianWidth = EastAsianWidth + if len(DefaultCondition.combinedLut) > 0 { + DefaultCondition.combinedLut = DefaultCondition.combinedLut[:0] + CreateLUT() + } + } +} + +type interval struct { + first rune + last rune +} + +type table []interval + +func inTables(r rune, ts ...table) bool { + for _, t := range ts { + if inTable(r, t) { + return true + } + } + return false +} + +func inTable(r rune, t table) bool { + if r < t[0].first { + return false + } + + bot := 0 + top := len(t) - 1 + for top >= bot { + mid := (bot + top) >> 1 + + switch { + case t[mid].last < r: + bot = mid + 1 + case t[mid].first > r: + top = mid - 1 + default: + return true + } + } + + return false +} + +var private = table{ + {0x00E000, 0x00F8FF}, {0x0F0000, 0x0FFFFD}, {0x100000, 0x10FFFD}, +} + +var nonprint = table{ + {0x0000, 0x001F}, {0x007F, 0x009F}, {0x00AD, 0x00AD}, + {0x070F, 0x070F}, {0x180B, 0x180E}, {0x200B, 0x200F}, + {0x2028, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFE, 0xFFFF}, +} + +// Condition have flag EastAsianWidth whether the current locale is CJK or not. +type Condition struct { + combinedLut []byte + EastAsianWidth bool + StrictEmojiNeutral bool +} + +// NewCondition return new instance of Condition which is current locale. +func NewCondition() *Condition { + return &Condition{ + EastAsianWidth: EastAsianWidth, + StrictEmojiNeutral: StrictEmojiNeutral, + } +} + +// RuneWidth returns the number of cells in r. +// See http://www.unicode.org/reports/tr11/ +func (c *Condition) RuneWidth(r rune) int { + if r < 0 || r > 0x10FFFF { + return 0 + } + if len(c.combinedLut) > 0 { + return int(c.combinedLut[r>>1]>>(uint(r&1)*4)) & 3 + } + // optimized version, verified by TestRuneWidthChecksums() + if !c.EastAsianWidth { + switch { + case r < 0x20: + return 0 + case (r >= 0x7F && r <= 0x9F) || r == 0xAD: // nonprint + return 0 + case r < 0x300: + return 1 + case inTable(r, narrow): + return 1 + case inTables(r, nonprint, combining): + return 0 + case inTable(r, doublewidth): + return 2 + default: + return 1 + } + } else { + switch { + case inTables(r, nonprint, combining): + return 0 + case inTable(r, narrow): + return 1 + case inTables(r, ambiguous, doublewidth): + return 2 + case !c.StrictEmojiNeutral && inTables(r, ambiguous, emoji, narrow): + return 2 + default: + return 1 + } + } +} + +// CreateLUT will create an in-memory lookup table of 557056 bytes for faster operation. +// This should not be called concurrently with other operations on c. +// If options in c is changed, CreateLUT should be called again. +func (c *Condition) CreateLUT() { + const max = 0x110000 + lut := c.combinedLut + if len(c.combinedLut) != 0 { + // Remove so we don't use it. + c.combinedLut = nil + } else { + lut = make([]byte, max/2) + } + for i := range lut { + i32 := int32(i * 2) + x0 := c.RuneWidth(i32) + x1 := c.RuneWidth(i32 + 1) + lut[i] = uint8(x0) | uint8(x1)<<4 + } + c.combinedLut = lut +} + +// StringWidth return width as you can see +func (c *Condition) StringWidth(s string) (width int) { + g := uniseg.NewGraphemes(s) + for g.Next() { + var chWidth int + for _, r := range g.Runes() { + chWidth = c.RuneWidth(r) + if chWidth > 0 { + break // Our best guess at this point is to use the width of the first non-zero-width rune. + } + } + width += chWidth + } + return +} + +// Truncate return string truncated with w cells +func (c *Condition) Truncate(s string, w int, tail string) string { + if c.StringWidth(s) <= w { + return s + } + w -= c.StringWidth(tail) + var width int + pos := len(s) + g := uniseg.NewGraphemes(s) + for g.Next() { + var chWidth int + for _, r := range g.Runes() { + chWidth = c.RuneWidth(r) + if chWidth > 0 { + break // See StringWidth() for details. + } + } + if width+chWidth > w { + pos, _ = g.Positions() + break + } + width += chWidth + } + return s[:pos] + tail +} + +// TruncateLeft cuts w cells from the beginning of the `s`. +func (c *Condition) TruncateLeft(s string, w int, prefix string) string { + if c.StringWidth(s) <= w { + return prefix + } + + var width int + pos := len(s) + + g := uniseg.NewGraphemes(s) + for g.Next() { + var chWidth int + for _, r := range g.Runes() { + chWidth = c.RuneWidth(r) + if chWidth > 0 { + break // See StringWidth() for details. + } + } + + if width+chWidth > w { + if width < w { + _, pos = g.Positions() + prefix += strings.Repeat(" ", width+chWidth-w) + } else { + pos, _ = g.Positions() + } + + break + } + + width += chWidth + } + + return prefix + s[pos:] +} + +// Wrap return string wrapped with w cells +func (c *Condition) Wrap(s string, w int) string { + width := 0 + out := "" + for _, r := range s { + cw := c.RuneWidth(r) + if r == '\n' { + out += string(r) + width = 0 + continue + } else if width+cw > w { + out += "\n" + width = 0 + out += string(r) + width += cw + continue + } + out += string(r) + width += cw + } + return out +} + +// FillLeft return string filled in left by spaces in w cells +func (c *Condition) FillLeft(s string, w int) string { + width := c.StringWidth(s) + count := w - width + if count > 0 { + b := make([]byte, count) + for i := range b { + b[i] = ' ' + } + return string(b) + s + } + return s +} + +// FillRight return string filled in left by spaces in w cells +func (c *Condition) FillRight(s string, w int) string { + width := c.StringWidth(s) + count := w - width + if count > 0 { + b := make([]byte, count) + for i := range b { + b[i] = ' ' + } + return s + string(b) + } + return s +} + +// RuneWidth returns the number of cells in r. +// See http://www.unicode.org/reports/tr11/ +func RuneWidth(r rune) int { + return DefaultCondition.RuneWidth(r) +} + +// IsAmbiguousWidth returns whether is ambiguous width or not. +func IsAmbiguousWidth(r rune) bool { + return inTables(r, private, ambiguous) +} + +// IsNeutralWidth returns whether is neutral width or not. +func IsNeutralWidth(r rune) bool { + return inTable(r, neutral) +} + +// StringWidth return width as you can see +func StringWidth(s string) (width int) { + return DefaultCondition.StringWidth(s) +} + +// Truncate return string truncated with w cells +func Truncate(s string, w int, tail string) string { + return DefaultCondition.Truncate(s, w, tail) +} + +// TruncateLeft cuts w cells from the beginning of the `s`. +func TruncateLeft(s string, w int, prefix string) string { + return DefaultCondition.TruncateLeft(s, w, prefix) +} + +// Wrap return string wrapped with w cells +func Wrap(s string, w int) string { + return DefaultCondition.Wrap(s, w) +} + +// FillLeft return string filled in left by spaces in w cells +func FillLeft(s string, w int) string { + return DefaultCondition.FillLeft(s, w) +} + +// FillRight return string filled in left by spaces in w cells +func FillRight(s string, w int) string { + return DefaultCondition.FillRight(s, w) +} + +// CreateLUT will create an in-memory lookup table of 557055 bytes for faster operation. +// This should not be called concurrently with other operations. +func CreateLUT() { + if len(DefaultCondition.combinedLut) > 0 { + return + } + DefaultCondition.CreateLUT() +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go b/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go new file mode 100644 index 0000000000..84b6528dfe --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go @@ -0,0 +1,9 @@ +//go:build appengine +// +build appengine + +package runewidth + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + return false +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_js.go b/vendor/github.com/mattn/go-runewidth/runewidth_js.go new file mode 100644 index 0000000000..c2abbc2db3 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_js.go @@ -0,0 +1,9 @@ +//go:build js && !appengine +// +build js,!appengine + +package runewidth + +func IsEastAsian() bool { + // TODO: Implement this for the web. Detect east asian in a compatible way, and return true. + return false +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_posix.go b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go new file mode 100644 index 0000000000..5a31d738ec --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go @@ -0,0 +1,81 @@ +//go:build !windows && !js && !appengine +// +build !windows,!js,!appengine + +package runewidth + +import ( + "os" + "regexp" + "strings" +) + +var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`) + +var mblenTable = map[string]int{ + "utf-8": 6, + "utf8": 6, + "jis": 8, + "eucjp": 3, + "euckr": 2, + "euccn": 2, + "sjis": 2, + "cp932": 2, + "cp51932": 2, + "cp936": 2, + "cp949": 2, + "cp950": 2, + "big5": 2, + "gbk": 2, + "gb2312": 2, +} + +func isEastAsian(locale string) bool { + charset := strings.ToLower(locale) + r := reLoc.FindStringSubmatch(locale) + if len(r) == 2 { + charset = strings.ToLower(r[1]) + } + + if strings.HasSuffix(charset, "@cjk_narrow") { + return false + } + + for pos, b := range []byte(charset) { + if b == '@' { + charset = charset[:pos] + break + } + } + max := 1 + if m, ok := mblenTable[charset]; ok { + max = m + } + if max > 1 && (charset[0] != 'u' || + strings.HasPrefix(locale, "ja") || + strings.HasPrefix(locale, "ko") || + strings.HasPrefix(locale, "zh")) { + return true + } + return false +} + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + locale := os.Getenv("LC_ALL") + if locale == "" { + locale = os.Getenv("LC_CTYPE") + } + if locale == "" { + locale = os.Getenv("LANG") + } + + // ignore C locale + if locale == "POSIX" || locale == "C" { + return false + } + if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') { + return false + } + + return isEastAsian(locale) +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_table.go b/vendor/github.com/mattn/go-runewidth/runewidth_table.go new file mode 100644 index 0000000000..e5d890c266 --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_table.go @@ -0,0 +1,439 @@ +// Code generated by script/generate.go. DO NOT EDIT. + +package runewidth + +var combining = table{ + {0x0300, 0x036F}, {0x0483, 0x0489}, {0x07EB, 0x07F3}, + {0x0C00, 0x0C00}, {0x0C04, 0x0C04}, {0x0D00, 0x0D01}, + {0x135D, 0x135F}, {0x1A7F, 0x1A7F}, {0x1AB0, 0x1AC0}, + {0x1B6B, 0x1B73}, {0x1DC0, 0x1DF9}, {0x1DFB, 0x1DFF}, + {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2DE0, 0x2DFF}, + {0x3099, 0x309A}, {0xA66F, 0xA672}, {0xA674, 0xA67D}, + {0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA8E0, 0xA8F1}, + {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, {0x10376, 0x1037A}, + {0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x11300, 0x11301}, + {0x1133B, 0x1133C}, {0x11366, 0x1136C}, {0x11370, 0x11374}, + {0x16AF0, 0x16AF4}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172}, + {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, + {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, + {0x1E8D0, 0x1E8D6}, +} + +var doublewidth = table{ + {0x1100, 0x115F}, {0x231A, 0x231B}, {0x2329, 0x232A}, + {0x23E9, 0x23EC}, {0x23F0, 0x23F0}, {0x23F3, 0x23F3}, + {0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2648, 0x2653}, + {0x267F, 0x267F}, {0x2693, 0x2693}, {0x26A1, 0x26A1}, + {0x26AA, 0x26AB}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5}, + {0x26CE, 0x26CE}, {0x26D4, 0x26D4}, {0x26EA, 0x26EA}, + {0x26F2, 0x26F3}, {0x26F5, 0x26F5}, {0x26FA, 0x26FA}, + {0x26FD, 0x26FD}, {0x2705, 0x2705}, {0x270A, 0x270B}, + {0x2728, 0x2728}, {0x274C, 0x274C}, {0x274E, 0x274E}, + {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797}, + {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x2E80, 0x2E99}, + {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, + {0x3000, 0x303E}, {0x3041, 0x3096}, {0x3099, 0x30FF}, + {0x3105, 0x312F}, {0x3131, 0x318E}, {0x3190, 0x31E3}, + {0x31F0, 0x321E}, {0x3220, 0x3247}, {0x3250, 0x4DBF}, + {0x4E00, 0xA48C}, {0xA490, 0xA4C6}, {0xA960, 0xA97C}, + {0xAC00, 0xD7A3}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, + {0xFE30, 0xFE52}, {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, + {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6}, {0x16FE0, 0x16FE4}, + {0x16FF0, 0x16FF1}, {0x17000, 0x187F7}, {0x18800, 0x18CD5}, + {0x18D00, 0x18D08}, {0x1B000, 0x1B11E}, {0x1B150, 0x1B152}, + {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1F004, 0x1F004}, + {0x1F0CF, 0x1F0CF}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, + {0x1F200, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, + {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F320}, + {0x1F32D, 0x1F335}, {0x1F337, 0x1F37C}, {0x1F37E, 0x1F393}, + {0x1F3A0, 0x1F3CA}, {0x1F3CF, 0x1F3D3}, {0x1F3E0, 0x1F3F0}, + {0x1F3F4, 0x1F3F4}, {0x1F3F8, 0x1F43E}, {0x1F440, 0x1F440}, + {0x1F442, 0x1F4FC}, {0x1F4FF, 0x1F53D}, {0x1F54B, 0x1F54E}, + {0x1F550, 0x1F567}, {0x1F57A, 0x1F57A}, {0x1F595, 0x1F596}, + {0x1F5A4, 0x1F5A4}, {0x1F5FB, 0x1F64F}, {0x1F680, 0x1F6C5}, + {0x1F6CC, 0x1F6CC}, {0x1F6D0, 0x1F6D2}, {0x1F6D5, 0x1F6D7}, + {0x1F6EB, 0x1F6EC}, {0x1F6F4, 0x1F6FC}, {0x1F7E0, 0x1F7EB}, + {0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945}, {0x1F947, 0x1F978}, + {0x1F97A, 0x1F9CB}, {0x1F9CD, 0x1F9FF}, {0x1FA70, 0x1FA74}, + {0x1FA78, 0x1FA7A}, {0x1FA80, 0x1FA86}, {0x1FA90, 0x1FAA8}, + {0x1FAB0, 0x1FAB6}, {0x1FAC0, 0x1FAC2}, {0x1FAD0, 0x1FAD6}, + {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}, +} + +var ambiguous = table{ + {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, + {0x00AA, 0x00AA}, {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, + {0x00B6, 0x00BA}, {0x00BC, 0x00BF}, {0x00C6, 0x00C6}, + {0x00D0, 0x00D0}, {0x00D7, 0x00D8}, {0x00DE, 0x00E1}, + {0x00E6, 0x00E6}, {0x00E8, 0x00EA}, {0x00EC, 0x00ED}, + {0x00F0, 0x00F0}, {0x00F2, 0x00F3}, {0x00F7, 0x00FA}, + {0x00FC, 0x00FC}, {0x00FE, 0x00FE}, {0x0101, 0x0101}, + {0x0111, 0x0111}, {0x0113, 0x0113}, {0x011B, 0x011B}, + {0x0126, 0x0127}, {0x012B, 0x012B}, {0x0131, 0x0133}, + {0x0138, 0x0138}, {0x013F, 0x0142}, {0x0144, 0x0144}, + {0x0148, 0x014B}, {0x014D, 0x014D}, {0x0152, 0x0153}, + {0x0166, 0x0167}, {0x016B, 0x016B}, {0x01CE, 0x01CE}, + {0x01D0, 0x01D0}, {0x01D2, 0x01D2}, {0x01D4, 0x01D4}, + {0x01D6, 0x01D6}, {0x01D8, 0x01D8}, {0x01DA, 0x01DA}, + {0x01DC, 0x01DC}, {0x0251, 0x0251}, {0x0261, 0x0261}, + {0x02C4, 0x02C4}, {0x02C7, 0x02C7}, {0x02C9, 0x02CB}, + {0x02CD, 0x02CD}, {0x02D0, 0x02D0}, {0x02D8, 0x02DB}, + {0x02DD, 0x02DD}, {0x02DF, 0x02DF}, {0x0300, 0x036F}, + {0x0391, 0x03A1}, {0x03A3, 0x03A9}, {0x03B1, 0x03C1}, + {0x03C3, 0x03C9}, {0x0401, 0x0401}, {0x0410, 0x044F}, + {0x0451, 0x0451}, {0x2010, 0x2010}, {0x2013, 0x2016}, + {0x2018, 0x2019}, {0x201C, 0x201D}, {0x2020, 0x2022}, + {0x2024, 0x2027}, {0x2030, 0x2030}, {0x2032, 0x2033}, + {0x2035, 0x2035}, {0x203B, 0x203B}, {0x203E, 0x203E}, + {0x2074, 0x2074}, {0x207F, 0x207F}, {0x2081, 0x2084}, + {0x20AC, 0x20AC}, {0x2103, 0x2103}, {0x2105, 0x2105}, + {0x2109, 0x2109}, {0x2113, 0x2113}, {0x2116, 0x2116}, + {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B}, + {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, + {0x2170, 0x2179}, {0x2189, 0x2189}, {0x2190, 0x2199}, + {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, {0x21D4, 0x21D4}, + {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, + {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, + {0x2211, 0x2211}, {0x2215, 0x2215}, {0x221A, 0x221A}, + {0x221D, 0x2220}, {0x2223, 0x2223}, {0x2225, 0x2225}, + {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, + {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, + {0x2252, 0x2252}, {0x2260, 0x2261}, {0x2264, 0x2267}, + {0x226A, 0x226B}, {0x226E, 0x226F}, {0x2282, 0x2283}, + {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, + {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, + {0x2460, 0x24E9}, {0x24EB, 0x254B}, {0x2550, 0x2573}, + {0x2580, 0x258F}, {0x2592, 0x2595}, {0x25A0, 0x25A1}, + {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, + {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, + {0x25CB, 0x25CB}, {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, + {0x25EF, 0x25EF}, {0x2605, 0x2606}, {0x2609, 0x2609}, + {0x260E, 0x260F}, {0x261C, 0x261C}, {0x261E, 0x261E}, + {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, + {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, + {0x266F, 0x266F}, {0x269E, 0x269F}, {0x26BF, 0x26BF}, + {0x26C6, 0x26CD}, {0x26CF, 0x26D3}, {0x26D5, 0x26E1}, + {0x26E3, 0x26E3}, {0x26E8, 0x26E9}, {0x26EB, 0x26F1}, + {0x26F4, 0x26F4}, {0x26F6, 0x26F9}, {0x26FB, 0x26FC}, + {0x26FE, 0x26FF}, {0x273D, 0x273D}, {0x2776, 0x277F}, + {0x2B56, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF}, + {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, + {0x1F110, 0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F18D}, + {0x1F18F, 0x1F190}, {0x1F19B, 0x1F1AC}, {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, +} +var narrow = table{ + {0x0020, 0x007E}, {0x00A2, 0x00A3}, {0x00A5, 0x00A6}, + {0x00AC, 0x00AC}, {0x00AF, 0x00AF}, {0x27E6, 0x27ED}, + {0x2985, 0x2986}, +} + +var neutral = table{ + {0x0000, 0x001F}, {0x007F, 0x00A0}, {0x00A9, 0x00A9}, + {0x00AB, 0x00AB}, {0x00B5, 0x00B5}, {0x00BB, 0x00BB}, + {0x00C0, 0x00C5}, {0x00C7, 0x00CF}, {0x00D1, 0x00D6}, + {0x00D9, 0x00DD}, {0x00E2, 0x00E5}, {0x00E7, 0x00E7}, + {0x00EB, 0x00EB}, {0x00EE, 0x00EF}, {0x00F1, 0x00F1}, + {0x00F4, 0x00F6}, {0x00FB, 0x00FB}, {0x00FD, 0x00FD}, + {0x00FF, 0x0100}, {0x0102, 0x0110}, {0x0112, 0x0112}, + {0x0114, 0x011A}, {0x011C, 0x0125}, {0x0128, 0x012A}, + {0x012C, 0x0130}, {0x0134, 0x0137}, {0x0139, 0x013E}, + {0x0143, 0x0143}, {0x0145, 0x0147}, {0x014C, 0x014C}, + {0x014E, 0x0151}, {0x0154, 0x0165}, {0x0168, 0x016A}, + {0x016C, 0x01CD}, {0x01CF, 0x01CF}, {0x01D1, 0x01D1}, + {0x01D3, 0x01D3}, {0x01D5, 0x01D5}, {0x01D7, 0x01D7}, + {0x01D9, 0x01D9}, {0x01DB, 0x01DB}, {0x01DD, 0x0250}, + {0x0252, 0x0260}, {0x0262, 0x02C3}, {0x02C5, 0x02C6}, + {0x02C8, 0x02C8}, {0x02CC, 0x02CC}, {0x02CE, 0x02CF}, + {0x02D1, 0x02D7}, {0x02DC, 0x02DC}, {0x02DE, 0x02DE}, + {0x02E0, 0x02FF}, {0x0370, 0x0377}, {0x037A, 0x037F}, + {0x0384, 0x038A}, {0x038C, 0x038C}, {0x038E, 0x0390}, + {0x03AA, 0x03B0}, {0x03C2, 0x03C2}, {0x03CA, 0x0400}, + {0x0402, 0x040F}, {0x0450, 0x0450}, {0x0452, 0x052F}, + {0x0531, 0x0556}, {0x0559, 0x058A}, {0x058D, 0x058F}, + {0x0591, 0x05C7}, {0x05D0, 0x05EA}, {0x05EF, 0x05F4}, + {0x0600, 0x061C}, {0x061E, 0x070D}, {0x070F, 0x074A}, + {0x074D, 0x07B1}, {0x07C0, 0x07FA}, {0x07FD, 0x082D}, + {0x0830, 0x083E}, {0x0840, 0x085B}, {0x085E, 0x085E}, + {0x0860, 0x086A}, {0x08A0, 0x08B4}, {0x08B6, 0x08C7}, + {0x08D3, 0x0983}, {0x0985, 0x098C}, {0x098F, 0x0990}, + {0x0993, 0x09A8}, {0x09AA, 0x09B0}, {0x09B2, 0x09B2}, + {0x09B6, 0x09B9}, {0x09BC, 0x09C4}, {0x09C7, 0x09C8}, + {0x09CB, 0x09CE}, {0x09D7, 0x09D7}, {0x09DC, 0x09DD}, + {0x09DF, 0x09E3}, {0x09E6, 0x09FE}, {0x0A01, 0x0A03}, + {0x0A05, 0x0A0A}, {0x0A0F, 0x0A10}, {0x0A13, 0x0A28}, + {0x0A2A, 0x0A30}, {0x0A32, 0x0A33}, {0x0A35, 0x0A36}, + {0x0A38, 0x0A39}, {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, + {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, + {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E}, {0x0A66, 0x0A76}, + {0x0A81, 0x0A83}, {0x0A85, 0x0A8D}, {0x0A8F, 0x0A91}, + {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0}, {0x0AB2, 0x0AB3}, + {0x0AB5, 0x0AB9}, {0x0ABC, 0x0AC5}, {0x0AC7, 0x0AC9}, + {0x0ACB, 0x0ACD}, {0x0AD0, 0x0AD0}, {0x0AE0, 0x0AE3}, + {0x0AE6, 0x0AF1}, {0x0AF9, 0x0AFF}, {0x0B01, 0x0B03}, + {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10}, {0x0B13, 0x0B28}, + {0x0B2A, 0x0B30}, {0x0B32, 0x0B33}, {0x0B35, 0x0B39}, + {0x0B3C, 0x0B44}, {0x0B47, 0x0B48}, {0x0B4B, 0x0B4D}, + {0x0B55, 0x0B57}, {0x0B5C, 0x0B5D}, {0x0B5F, 0x0B63}, + {0x0B66, 0x0B77}, {0x0B82, 0x0B83}, {0x0B85, 0x0B8A}, + {0x0B8E, 0x0B90}, {0x0B92, 0x0B95}, {0x0B99, 0x0B9A}, + {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F}, {0x0BA3, 0x0BA4}, + {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9}, {0x0BBE, 0x0BC2}, + {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD}, {0x0BD0, 0x0BD0}, + {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BFA}, {0x0C00, 0x0C0C}, + {0x0C0E, 0x0C10}, {0x0C12, 0x0C28}, {0x0C2A, 0x0C39}, + {0x0C3D, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, + {0x0C55, 0x0C56}, {0x0C58, 0x0C5A}, {0x0C60, 0x0C63}, + {0x0C66, 0x0C6F}, {0x0C77, 0x0C8C}, {0x0C8E, 0x0C90}, + {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, + {0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, + {0x0CD5, 0x0CD6}, {0x0CDE, 0x0CDE}, {0x0CE0, 0x0CE3}, + {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2}, {0x0D00, 0x0D0C}, + {0x0D0E, 0x0D10}, {0x0D12, 0x0D44}, {0x0D46, 0x0D48}, + {0x0D4A, 0x0D4F}, {0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, + {0x0D81, 0x0D83}, {0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, + {0x0DB3, 0x0DBB}, {0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, + {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, + {0x0DD8, 0x0DDF}, {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, + {0x0E01, 0x0E3A}, {0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, + {0x0E84, 0x0E84}, {0x0E86, 0x0E8A}, {0x0E8C, 0x0EA3}, + {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EBD}, {0x0EC0, 0x0EC4}, + {0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9}, + {0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0F49, 0x0F6C}, + {0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC}, + {0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FF}, {0x1160, 0x1248}, + {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, + {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, + {0x1290, 0x12B0}, {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, + {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, + {0x135D, 0x137C}, {0x1380, 0x1399}, {0x13A0, 0x13F5}, + {0x13F8, 0x13FD}, {0x1400, 0x169C}, {0x16A0, 0x16F8}, + {0x1700, 0x170C}, {0x170E, 0x1714}, {0x1720, 0x1736}, + {0x1740, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770}, + {0x1772, 0x1773}, {0x1780, 0x17DD}, {0x17E0, 0x17E9}, + {0x17F0, 0x17F9}, {0x1800, 0x180E}, {0x1810, 0x1819}, + {0x1820, 0x1878}, {0x1880, 0x18AA}, {0x18B0, 0x18F5}, + {0x1900, 0x191E}, {0x1920, 0x192B}, {0x1930, 0x193B}, + {0x1940, 0x1940}, {0x1944, 0x196D}, {0x1970, 0x1974}, + {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, + {0x19DE, 0x1A1B}, {0x1A1E, 0x1A5E}, {0x1A60, 0x1A7C}, + {0x1A7F, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA0, 0x1AAD}, + {0x1AB0, 0x1AC0}, {0x1B00, 0x1B4B}, {0x1B50, 0x1B7C}, + {0x1B80, 0x1BF3}, {0x1BFC, 0x1C37}, {0x1C3B, 0x1C49}, + {0x1C4D, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CC7}, + {0x1CD0, 0x1CFA}, {0x1D00, 0x1DF9}, {0x1DFB, 0x1F15}, + {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, + {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB}, + {0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE}, + {0x2000, 0x200F}, {0x2011, 0x2012}, {0x2017, 0x2017}, + {0x201A, 0x201B}, {0x201E, 0x201F}, {0x2023, 0x2023}, + {0x2028, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034}, + {0x2036, 0x203A}, {0x203C, 0x203D}, {0x203F, 0x2064}, + {0x2066, 0x2071}, {0x2075, 0x207E}, {0x2080, 0x2080}, + {0x2085, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8}, + {0x20AA, 0x20AB}, {0x20AD, 0x20BF}, {0x20D0, 0x20F0}, + {0x2100, 0x2102}, {0x2104, 0x2104}, {0x2106, 0x2108}, + {0x210A, 0x2112}, {0x2114, 0x2115}, {0x2117, 0x2120}, + {0x2123, 0x2125}, {0x2127, 0x212A}, {0x212C, 0x2152}, + {0x2155, 0x215A}, {0x215F, 0x215F}, {0x216C, 0x216F}, + {0x217A, 0x2188}, {0x218A, 0x218B}, {0x219A, 0x21B7}, + {0x21BA, 0x21D1}, {0x21D3, 0x21D3}, {0x21D5, 0x21E6}, + {0x21E8, 0x21FF}, {0x2201, 0x2201}, {0x2204, 0x2206}, + {0x2209, 0x220A}, {0x220C, 0x220E}, {0x2210, 0x2210}, + {0x2212, 0x2214}, {0x2216, 0x2219}, {0x221B, 0x221C}, + {0x2221, 0x2222}, {0x2224, 0x2224}, {0x2226, 0x2226}, + {0x222D, 0x222D}, {0x222F, 0x2233}, {0x2238, 0x223B}, + {0x223E, 0x2247}, {0x2249, 0x224B}, {0x224D, 0x2251}, + {0x2253, 0x225F}, {0x2262, 0x2263}, {0x2268, 0x2269}, + {0x226C, 0x226D}, {0x2270, 0x2281}, {0x2284, 0x2285}, + {0x2288, 0x2294}, {0x2296, 0x2298}, {0x229A, 0x22A4}, + {0x22A6, 0x22BE}, {0x22C0, 0x2311}, {0x2313, 0x2319}, + {0x231C, 0x2328}, {0x232B, 0x23E8}, {0x23ED, 0x23EF}, + {0x23F1, 0x23F2}, {0x23F4, 0x2426}, {0x2440, 0x244A}, + {0x24EA, 0x24EA}, {0x254C, 0x254F}, {0x2574, 0x257F}, + {0x2590, 0x2591}, {0x2596, 0x259F}, {0x25A2, 0x25A2}, + {0x25AA, 0x25B1}, {0x25B4, 0x25B5}, {0x25B8, 0x25BB}, + {0x25BE, 0x25BF}, {0x25C2, 0x25C5}, {0x25C9, 0x25CA}, + {0x25CC, 0x25CD}, {0x25D2, 0x25E1}, {0x25E6, 0x25EE}, + {0x25F0, 0x25FC}, {0x25FF, 0x2604}, {0x2607, 0x2608}, + {0x260A, 0x260D}, {0x2610, 0x2613}, {0x2616, 0x261B}, + {0x261D, 0x261D}, {0x261F, 0x263F}, {0x2641, 0x2641}, + {0x2643, 0x2647}, {0x2654, 0x265F}, {0x2662, 0x2662}, + {0x2666, 0x2666}, {0x266B, 0x266B}, {0x266E, 0x266E}, + {0x2670, 0x267E}, {0x2680, 0x2692}, {0x2694, 0x269D}, + {0x26A0, 0x26A0}, {0x26A2, 0x26A9}, {0x26AC, 0x26BC}, + {0x26C0, 0x26C3}, {0x26E2, 0x26E2}, {0x26E4, 0x26E7}, + {0x2700, 0x2704}, {0x2706, 0x2709}, {0x270C, 0x2727}, + {0x2729, 0x273C}, {0x273E, 0x274B}, {0x274D, 0x274D}, + {0x274F, 0x2752}, {0x2756, 0x2756}, {0x2758, 0x2775}, + {0x2780, 0x2794}, {0x2798, 0x27AF}, {0x27B1, 0x27BE}, + {0x27C0, 0x27E5}, {0x27EE, 0x2984}, {0x2987, 0x2B1A}, + {0x2B1D, 0x2B4F}, {0x2B51, 0x2B54}, {0x2B5A, 0x2B73}, + {0x2B76, 0x2B95}, {0x2B97, 0x2C2E}, {0x2C30, 0x2C5E}, + {0x2C60, 0x2CF3}, {0x2CF9, 0x2D25}, {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D70}, + {0x2D7F, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, + {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, + {0x2DE0, 0x2E52}, {0x303F, 0x303F}, {0x4DC0, 0x4DFF}, + {0xA4D0, 0xA62B}, {0xA640, 0xA6F7}, {0xA700, 0xA7BF}, + {0xA7C2, 0xA7CA}, {0xA7F5, 0xA82C}, {0xA830, 0xA839}, + {0xA840, 0xA877}, {0xA880, 0xA8C5}, {0xA8CE, 0xA8D9}, + {0xA8E0, 0xA953}, {0xA95F, 0xA95F}, {0xA980, 0xA9CD}, + {0xA9CF, 0xA9D9}, {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36}, + {0xAA40, 0xAA4D}, {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2}, + {0xAADB, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, + {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, + {0xAB30, 0xAB6B}, {0xAB70, 0xABED}, {0xABF0, 0xABF9}, + {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDFFF}, + {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB36}, + {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, + {0xFB43, 0xFB44}, {0xFB46, 0xFBC1}, {0xFBD3, 0xFD3F}, + {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFD}, + {0xFE20, 0xFE2F}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFC}, {0x10000, 0x1000B}, + {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, + {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, + {0x10100, 0x10102}, {0x10107, 0x10133}, {0x10137, 0x1018E}, + {0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FD}, + {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E0, 0x102FB}, + {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x1037A}, + {0x10380, 0x1039D}, {0x1039F, 0x103C3}, {0x103C8, 0x103D5}, + {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, + {0x1056F, 0x1056F}, {0x10600, 0x10736}, {0x10740, 0x10755}, + {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, + {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, + {0x1083F, 0x10855}, {0x10857, 0x1089E}, {0x108A7, 0x108AF}, + {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, + {0x1091F, 0x10939}, {0x1093F, 0x1093F}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A03}, {0x10A05, 0x10A06}, + {0x10A0C, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, + {0x10A38, 0x10A3A}, {0x10A3F, 0x10A48}, {0x10A50, 0x10A58}, + {0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6}, + {0x10B00, 0x10B35}, {0x10B39, 0x10B55}, {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D27}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, + {0x10E80, 0x10EA9}, {0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1}, + {0x10F00, 0x10F27}, {0x10F30, 0x10F59}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11000, 0x1104D}, {0x11052, 0x1106F}, + {0x1107F, 0x110C1}, {0x110CD, 0x110CD}, {0x110D0, 0x110E8}, + {0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11147}, + {0x11150, 0x11176}, {0x11180, 0x111DF}, {0x111E1, 0x111F4}, + {0x11200, 0x11211}, {0x11213, 0x1123E}, {0x11280, 0x11286}, + {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, + {0x1129F, 0x112A9}, {0x112B0, 0x112EA}, {0x112F0, 0x112F9}, + {0x11300, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, + {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, + {0x11335, 0x11339}, {0x1133B, 0x11344}, {0x11347, 0x11348}, + {0x1134B, 0x1134D}, {0x11350, 0x11350}, {0x11357, 0x11357}, + {0x1135D, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, + {0x11400, 0x1145B}, {0x1145D, 0x11461}, {0x11480, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115B5}, {0x115B8, 0x115DD}, + {0x11600, 0x11644}, {0x11650, 0x11659}, {0x11660, 0x1166C}, + {0x11680, 0x116B8}, {0x116C0, 0x116C9}, {0x11700, 0x1171A}, + {0x1171D, 0x1172B}, {0x11730, 0x1173F}, {0x11800, 0x1183B}, + {0x118A0, 0x118F2}, {0x118FF, 0x11906}, {0x11909, 0x11909}, + {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x11935}, + {0x11937, 0x11938}, {0x1193B, 0x11946}, {0x11950, 0x11959}, + {0x119A0, 0x119A7}, {0x119AA, 0x119D7}, {0x119DA, 0x119E4}, + {0x11A00, 0x11A47}, {0x11A50, 0x11AA2}, {0x11AC0, 0x11AF8}, + {0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, {0x11C38, 0x11C45}, + {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, {0x11C92, 0x11CA7}, + {0x11CA9, 0x11CB6}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, + {0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, + {0x11D3F, 0x11D47}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, + {0x11D67, 0x11D68}, {0x11D6A, 0x11D8E}, {0x11D90, 0x11D91}, + {0x11D93, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF8}, + {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1}, {0x11FFF, 0x12399}, + {0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543}, + {0x13000, 0x1342E}, {0x13430, 0x13438}, {0x14400, 0x14646}, + {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, + {0x16A6E, 0x16A6F}, {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5}, + {0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, + {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E9A}, + {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16F8F, 0x16F9F}, + {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, + {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3}, {0x1D000, 0x1D0F5}, + {0x1D100, 0x1D126}, {0x1D129, 0x1D1E8}, {0x1D200, 0x1D245}, + {0x1D2E0, 0x1D2F3}, {0x1D300, 0x1D356}, {0x1D360, 0x1D378}, + {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, + {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B}, + {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14F}, {0x1E2C0, 0x1E2F9}, + {0x1E2FF, 0x1E2FF}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6}, + {0x1E900, 0x1E94B}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, + {0x1EC71, 0x1ECB4}, {0x1ED01, 0x1ED3D}, {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, + {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, + {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, + {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, + {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, + {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, + {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, + {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, + {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1EEF0, 0x1EEF1}, + {0x1F000, 0x1F003}, {0x1F005, 0x1F02B}, {0x1F030, 0x1F093}, + {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CE}, + {0x1F0D1, 0x1F0F5}, {0x1F10B, 0x1F10F}, {0x1F12E, 0x1F12F}, + {0x1F16A, 0x1F16F}, {0x1F1AD, 0x1F1AD}, {0x1F1E6, 0x1F1FF}, + {0x1F321, 0x1F32C}, {0x1F336, 0x1F336}, {0x1F37D, 0x1F37D}, + {0x1F394, 0x1F39F}, {0x1F3CB, 0x1F3CE}, {0x1F3D4, 0x1F3DF}, + {0x1F3F1, 0x1F3F3}, {0x1F3F5, 0x1F3F7}, {0x1F43F, 0x1F43F}, + {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FE}, {0x1F53E, 0x1F54A}, + {0x1F54F, 0x1F54F}, {0x1F568, 0x1F579}, {0x1F57B, 0x1F594}, + {0x1F597, 0x1F5A3}, {0x1F5A5, 0x1F5FA}, {0x1F650, 0x1F67F}, + {0x1F6C6, 0x1F6CB}, {0x1F6CD, 0x1F6CF}, {0x1F6D3, 0x1F6D4}, + {0x1F6E0, 0x1F6EA}, {0x1F6F0, 0x1F6F3}, {0x1F700, 0x1F773}, + {0x1F780, 0x1F7D8}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, + {0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, + {0x1F8B0, 0x1F8B1}, {0x1F900, 0x1F90B}, {0x1F93B, 0x1F93B}, + {0x1F946, 0x1F946}, {0x1FA00, 0x1FA53}, {0x1FA60, 0x1FA6D}, + {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA}, {0x1FBF0, 0x1FBF9}, + {0xE0001, 0xE0001}, {0xE0020, 0xE007F}, +} + +var emoji = table{ + {0x203C, 0x203C}, {0x2049, 0x2049}, {0x2122, 0x2122}, + {0x2139, 0x2139}, {0x2194, 0x2199}, {0x21A9, 0x21AA}, + {0x231A, 0x231B}, {0x2328, 0x2328}, {0x2388, 0x2388}, + {0x23CF, 0x23CF}, {0x23E9, 0x23F3}, {0x23F8, 0x23FA}, + {0x24C2, 0x24C2}, {0x25AA, 0x25AB}, {0x25B6, 0x25B6}, + {0x25C0, 0x25C0}, {0x25FB, 0x25FE}, {0x2600, 0x2605}, + {0x2607, 0x2612}, {0x2614, 0x2685}, {0x2690, 0x2705}, + {0x2708, 0x2712}, {0x2714, 0x2714}, {0x2716, 0x2716}, + {0x271D, 0x271D}, {0x2721, 0x2721}, {0x2728, 0x2728}, + {0x2733, 0x2734}, {0x2744, 0x2744}, {0x2747, 0x2747}, + {0x274C, 0x274C}, {0x274E, 0x274E}, {0x2753, 0x2755}, + {0x2757, 0x2757}, {0x2763, 0x2767}, {0x2795, 0x2797}, + {0x27A1, 0x27A1}, {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, + {0x2934, 0x2935}, {0x2B05, 0x2B07}, {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x3030, 0x3030}, + {0x303D, 0x303D}, {0x3297, 0x3297}, {0x3299, 0x3299}, + {0x1F000, 0x1F0FF}, {0x1F10D, 0x1F10F}, {0x1F12F, 0x1F12F}, + {0x1F16C, 0x1F171}, {0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E}, + {0x1F191, 0x1F19A}, {0x1F1AD, 0x1F1E5}, {0x1F201, 0x1F20F}, + {0x1F21A, 0x1F21A}, {0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A}, + {0x1F23C, 0x1F23F}, {0x1F249, 0x1F3FA}, {0x1F400, 0x1F53D}, + {0x1F546, 0x1F64F}, {0x1F680, 0x1F6FF}, {0x1F774, 0x1F77F}, + {0x1F7D5, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, + {0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8FF}, + {0x1F90C, 0x1F93A}, {0x1F93C, 0x1F945}, {0x1F947, 0x1FAFF}, + {0x1FC00, 0x1FFFD}, +} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_windows.go b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go new file mode 100644 index 0000000000..5f987a310f --- /dev/null +++ b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go @@ -0,0 +1,28 @@ +//go:build windows && !appengine +// +build windows,!appengine + +package runewidth + +import ( + "syscall" +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32") + procGetConsoleOutputCP = kernel32.NewProc("GetConsoleOutputCP") +) + +// IsEastAsian return true if the current locale is CJK +func IsEastAsian() bool { + r1, _, _ := procGetConsoleOutputCP.Call() + if r1 == 0 { + return false + } + + switch int(r1) { + case 932, 51932, 936, 949, 950: + return true + } + + return false +} diff --git a/vendor/github.com/olekukonko/tablewriter/.gitignore b/vendor/github.com/olekukonko/tablewriter/.gitignore new file mode 100644 index 0000000000..b66cec635a --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/.gitignore @@ -0,0 +1,15 @@ +# Created by .ignore support plugin (hsz.mobi) +### Go template +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + diff --git a/vendor/github.com/olekukonko/tablewriter/.travis.yml b/vendor/github.com/olekukonko/tablewriter/.travis.yml new file mode 100644 index 0000000000..366d48a35c --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/.travis.yml @@ -0,0 +1,22 @@ +language: go +arch: + - ppc64le + - amd64 +go: + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - "1.10" + - tip +jobs: + exclude : + - arch : ppc64le + go : + - 1.3 + - arch : ppc64le + go : + - 1.4 diff --git a/vendor/github.com/olekukonko/tablewriter/LICENSE.md b/vendor/github.com/olekukonko/tablewriter/LICENSE.md new file mode 100644 index 0000000000..a0769b5c15 --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/LICENSE.md @@ -0,0 +1,19 @@ +Copyright (C) 2014 by Oleku Konko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/olekukonko/tablewriter/README.md b/vendor/github.com/olekukonko/tablewriter/README.md new file mode 100644 index 0000000000..f06530d751 --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/README.md @@ -0,0 +1,431 @@ +ASCII Table Writer +========= + +[![Build Status](https://travis-ci.org/olekukonko/tablewriter.png?branch=master)](https://travis-ci.org/olekukonko/tablewriter) +[![Total views](https://img.shields.io/sourcegraph/rrc/github.com/olekukonko/tablewriter.svg)](https://sourcegraph.com/github.com/olekukonko/tablewriter) +[![Godoc](https://godoc.org/github.com/olekukonko/tablewriter?status.svg)](https://godoc.org/github.com/olekukonko/tablewriter) + +Generate ASCII table on the fly ... Installation is simple as + + go get github.com/olekukonko/tablewriter + + +#### Features +- Automatic Padding +- Support Multiple Lines +- Supports Alignment +- Support Custom Separators +- Automatic Alignment of numbers & percentage +- Write directly to http , file etc via `io.Writer` +- Read directly from CSV file +- Optional row line via `SetRowLine` +- Normalise table header +- Make CSV Headers optional +- Enable or disable table border +- Set custom footer support +- Optional identical cells merging +- Set custom caption +- Optional reflowing of paragraphs in multi-line cells. + +#### Example 1 - Basic +```go +data := [][]string{ + []string{"A", "The Good", "500"}, + []string{"B", "The Very very Bad Man", "288"}, + []string{"C", "The Ugly", "120"}, + []string{"D", "The Gopher", "800"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Name", "Sign", "Rating"}) + +for _, v := range data { + table.Append(v) +} +table.Render() // Send output +``` + +##### Output 1 +``` ++------+-----------------------+--------+ +| NAME | SIGN | RATING | ++------+-----------------------+--------+ +| A | The Good | 500 | +| B | The Very very Bad Man | 288 | +| C | The Ugly | 120 | +| D | The Gopher | 800 | ++------+-----------------------+--------+ +``` + +#### Example 2 - Without Border / Footer / Bulk Append +```go +data := [][]string{ + []string{"1/1/2014", "Domain name", "2233", "$10.98"}, + []string{"1/1/2014", "January Hosting", "2233", "$54.95"}, + []string{"1/4/2014", "February Hosting", "2233", "$51.00"}, + []string{"1/4/2014", "February Extra Bandwidth", "2233", "$30.00"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Date", "Description", "CV2", "Amount"}) +table.SetFooter([]string{"", "", "Total", "$146.93"}) // Add Footer +table.SetBorder(false) // Set Border to false +table.AppendBulk(data) // Add Bulk Data +table.Render() +``` + +##### Output 2 +``` + + DATE | DESCRIPTION | CV2 | AMOUNT +-----------+--------------------------+-------+---------- + 1/1/2014 | Domain name | 2233 | $10.98 + 1/1/2014 | January Hosting | 2233 | $54.95 + 1/4/2014 | February Hosting | 2233 | $51.00 + 1/4/2014 | February Extra Bandwidth | 2233 | $30.00 +-----------+--------------------------+-------+---------- + TOTAL | $146 93 + --------+---------- + +``` + + +#### Example 3 - CSV +```go +table, _ := tablewriter.NewCSV(os.Stdout, "testdata/test_info.csv", true) +table.SetAlignment(tablewriter.ALIGN_LEFT) // Set Alignment +table.Render() +``` + +##### Output 3 +``` ++----------+--------------+------+-----+---------+----------------+ +| FIELD | TYPE | NULL | KEY | DEFAULT | EXTRA | ++----------+--------------+------+-----+---------+----------------+ +| user_id | smallint(5) | NO | PRI | NULL | auto_increment | +| username | varchar(10) | NO | | NULL | | +| password | varchar(100) | NO | | NULL | | ++----------+--------------+------+-----+---------+----------------+ +``` + +#### Example 4 - Custom Separator +```go +table, _ := tablewriter.NewCSV(os.Stdout, "testdata/test.csv", true) +table.SetRowLine(true) // Enable row line + +// Change table lines +table.SetCenterSeparator("*") +table.SetColumnSeparator("โ•ช") +table.SetRowSeparator("-") + +table.SetAlignment(tablewriter.ALIGN_LEFT) +table.Render() +``` + +##### Output 4 +``` +*------------*-----------*---------* +โ•ช FIRST NAME โ•ช LAST NAME โ•ช SSN โ•ช +*------------*-----------*---------* +โ•ช John โ•ช Barry โ•ช 123456 โ•ช +*------------*-----------*---------* +โ•ช Kathy โ•ช Smith โ•ช 687987 โ•ช +*------------*-----------*---------* +โ•ช Bob โ•ช McCornick โ•ช 3979870 โ•ช +*------------*-----------*---------* +``` + +#### Example 5 - Markdown Format +```go +data := [][]string{ + []string{"1/1/2014", "Domain name", "2233", "$10.98"}, + []string{"1/1/2014", "January Hosting", "2233", "$54.95"}, + []string{"1/4/2014", "February Hosting", "2233", "$51.00"}, + []string{"1/4/2014", "February Extra Bandwidth", "2233", "$30.00"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Date", "Description", "CV2", "Amount"}) +table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) +table.SetCenterSeparator("|") +table.AppendBulk(data) // Add Bulk Data +table.Render() +``` + +##### Output 5 +``` +| DATE | DESCRIPTION | CV2 | AMOUNT | +|----------|--------------------------|------|--------| +| 1/1/2014 | Domain name | 2233 | $10.98 | +| 1/1/2014 | January Hosting | 2233 | $54.95 | +| 1/4/2014 | February Hosting | 2233 | $51.00 | +| 1/4/2014 | February Extra Bandwidth | 2233 | $30.00 | +``` + +#### Example 6 - Identical cells merging +```go +data := [][]string{ + []string{"1/1/2014", "Domain name", "1234", "$10.98"}, + []string{"1/1/2014", "January Hosting", "2345", "$54.95"}, + []string{"1/4/2014", "February Hosting", "3456", "$51.00"}, + []string{"1/4/2014", "February Extra Bandwidth", "4567", "$30.00"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Date", "Description", "CV2", "Amount"}) +table.SetFooter([]string{"", "", "Total", "$146.93"}) +table.SetAutoMergeCells(true) +table.SetRowLine(true) +table.AppendBulk(data) +table.Render() +``` + +##### Output 6 +``` ++----------+--------------------------+-------+---------+ +| DATE | DESCRIPTION | CV2 | AMOUNT | ++----------+--------------------------+-------+---------+ +| 1/1/2014 | Domain name | 1234 | $10.98 | ++ +--------------------------+-------+---------+ +| | January Hosting | 2345 | $54.95 | ++----------+--------------------------+-------+---------+ +| 1/4/2014 | February Hosting | 3456 | $51.00 | ++ +--------------------------+-------+---------+ +| | February Extra Bandwidth | 4567 | $30.00 | ++----------+--------------------------+-------+---------+ +| TOTAL | $146 93 | ++----------+--------------------------+-------+---------+ +``` + +#### Example 7 - Identical cells merging (specify the column index to merge) +```go +data := [][]string{ + []string{"1/1/2014", "Domain name", "1234", "$10.98"}, + []string{"1/1/2014", "January Hosting", "1234", "$10.98"}, + []string{"1/4/2014", "February Hosting", "3456", "$51.00"}, + []string{"1/4/2014", "February Extra Bandwidth", "4567", "$30.00"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Date", "Description", "CV2", "Amount"}) +table.SetFooter([]string{"", "", "Total", "$146.93"}) +table.SetAutoMergeCellsByColumnIndex([]int{2, 3}) +table.SetRowLine(true) +table.AppendBulk(data) +table.Render() +``` + +##### Output 7 +``` ++----------+--------------------------+-------+---------+ +| DATE | DESCRIPTION | CV2 | AMOUNT | ++----------+--------------------------+-------+---------+ +| 1/1/2014 | Domain name | 1234 | $10.98 | ++----------+--------------------------+ + + +| 1/1/2014 | January Hosting | | | ++----------+--------------------------+-------+---------+ +| 1/4/2014 | February Hosting | 3456 | $51.00 | ++----------+--------------------------+-------+---------+ +| 1/4/2014 | February Extra Bandwidth | 4567 | $30.00 | ++----------+--------------------------+-------+---------+ +| TOTAL | $146.93 | ++----------+--------------------------+-------+---------+ +``` + + +#### Table with color +```go +data := [][]string{ + []string{"1/1/2014", "Domain name", "2233", "$10.98"}, + []string{"1/1/2014", "January Hosting", "2233", "$54.95"}, + []string{"1/4/2014", "February Hosting", "2233", "$51.00"}, + []string{"1/4/2014", "February Extra Bandwidth", "2233", "$30.00"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Date", "Description", "CV2", "Amount"}) +table.SetFooter([]string{"", "", "Total", "$146.93"}) // Add Footer +table.SetBorder(false) // Set Border to false + +table.SetHeaderColor(tablewriter.Colors{tablewriter.Bold, tablewriter.BgGreenColor}, + tablewriter.Colors{tablewriter.FgHiRedColor, tablewriter.Bold, tablewriter.BgBlackColor}, + tablewriter.Colors{tablewriter.BgRedColor, tablewriter.FgWhiteColor}, + tablewriter.Colors{tablewriter.BgCyanColor, tablewriter.FgWhiteColor}) + +table.SetColumnColor(tablewriter.Colors{tablewriter.Bold, tablewriter.FgHiBlackColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgHiRedColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgHiBlackColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgBlackColor}) + +table.SetFooterColor(tablewriter.Colors{}, tablewriter.Colors{}, + tablewriter.Colors{tablewriter.Bold}, + tablewriter.Colors{tablewriter.FgHiRedColor}) + +table.AppendBulk(data) +table.Render() +``` + +#### Table with color Output +![Table with Color](https://cloud.githubusercontent.com/assets/6460392/21101956/bbc7b356-c0a1-11e6-9f36-dba694746efc.png) + +#### Example - 8 Table Cells with Color + +Individual Cell Colors from `func Rich` take precedence over Column Colors + +```go +data := [][]string{ + []string{"Test1Merge", "HelloCol2 - 1", "HelloCol3 - 1", "HelloCol4 - 1"}, + []string{"Test1Merge", "HelloCol2 - 2", "HelloCol3 - 2", "HelloCol4 - 2"}, + []string{"Test1Merge", "HelloCol2 - 3", "HelloCol3 - 3", "HelloCol4 - 3"}, + []string{"Test2Merge", "HelloCol2 - 4", "HelloCol3 - 4", "HelloCol4 - 4"}, + []string{"Test2Merge", "HelloCol2 - 5", "HelloCol3 - 5", "HelloCol4 - 5"}, + []string{"Test2Merge", "HelloCol2 - 6", "HelloCol3 - 6", "HelloCol4 - 6"}, + []string{"Test2Merge", "HelloCol2 - 7", "HelloCol3 - 7", "HelloCol4 - 7"}, + []string{"Test3Merge", "HelloCol2 - 8", "HelloCol3 - 8", "HelloCol4 - 8"}, + []string{"Test3Merge", "HelloCol2 - 9", "HelloCol3 - 9", "HelloCol4 - 9"}, + []string{"Test3Merge", "HelloCol2 - 10", "HelloCol3 -10", "HelloCol4 - 10"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Col1", "Col2", "Col3", "Col4"}) +table.SetFooter([]string{"", "", "Footer3", "Footer4"}) +table.SetBorder(false) + +table.SetHeaderColor(tablewriter.Colors{tablewriter.Bold, tablewriter.BgGreenColor}, + tablewriter.Colors{tablewriter.FgHiRedColor, tablewriter.Bold, tablewriter.BgBlackColor}, + tablewriter.Colors{tablewriter.BgRedColor, tablewriter.FgWhiteColor}, + tablewriter.Colors{tablewriter.BgCyanColor, tablewriter.FgWhiteColor}) + +table.SetColumnColor(tablewriter.Colors{tablewriter.Bold, tablewriter.FgHiBlackColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgHiRedColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgHiBlackColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgBlackColor}) + +table.SetFooterColor(tablewriter.Colors{}, tablewriter.Colors{}, + tablewriter.Colors{tablewriter.Bold}, + tablewriter.Colors{tablewriter.FgHiRedColor}) + +colorData1 := []string{"TestCOLOR1Merge", "HelloCol2 - COLOR1", "HelloCol3 - COLOR1", "HelloCol4 - COLOR1"} +colorData2 := []string{"TestCOLOR2Merge", "HelloCol2 - COLOR2", "HelloCol3 - COLOR2", "HelloCol4 - COLOR2"} + +for i, row := range data { + if i == 4 { + table.Rich(colorData1, []tablewriter.Colors{tablewriter.Colors{}, tablewriter.Colors{tablewriter.Normal, tablewriter.FgCyanColor}, tablewriter.Colors{tablewriter.Bold, tablewriter.FgWhiteColor}, tablewriter.Colors{}}) + table.Rich(colorData2, []tablewriter.Colors{tablewriter.Colors{tablewriter.Normal, tablewriter.FgMagentaColor}, tablewriter.Colors{}, tablewriter.Colors{tablewriter.Bold, tablewriter.BgRedColor}, tablewriter.Colors{tablewriter.FgHiGreenColor, tablewriter.Italic, tablewriter.BgHiCyanColor}}) + } + table.Append(row) +} + +table.SetAutoMergeCells(true) +table.Render() + +``` + +##### Table cells with color Output +![Table cells with Color](https://user-images.githubusercontent.com/9064687/63969376-bcd88d80-ca6f-11e9-9466-c3d954700b25.png) + +#### Example 9 - Set table caption +```go +data := [][]string{ + []string{"A", "The Good", "500"}, + []string{"B", "The Very very Bad Man", "288"}, + []string{"C", "The Ugly", "120"}, + []string{"D", "The Gopher", "800"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Name", "Sign", "Rating"}) +table.SetCaption(true, "Movie ratings.") + +for _, v := range data { + table.Append(v) +} +table.Render() // Send output +``` + +Note: Caption text will wrap with total width of rendered table. + +##### Output 9 +``` ++------+-----------------------+--------+ +| NAME | SIGN | RATING | ++------+-----------------------+--------+ +| A | The Good | 500 | +| B | The Very very Bad Man | 288 | +| C | The Ugly | 120 | +| D | The Gopher | 800 | ++------+-----------------------+--------+ +Movie ratings. +``` + +#### Example 10 - Set NoWhiteSpace and TablePadding option +```go +data := [][]string{ + {"node1.example.com", "Ready", "compute", "1.11"}, + {"node2.example.com", "Ready", "compute", "1.11"}, + {"node3.example.com", "Ready", "compute", "1.11"}, + {"node4.example.com", "NotReady", "compute", "1.11"}, +} + +table := tablewriter.NewWriter(os.Stdout) +table.SetHeader([]string{"Name", "Status", "Role", "Version"}) +table.SetAutoWrapText(false) +table.SetAutoFormatHeaders(true) +table.SetHeaderAlignment(ALIGN_LEFT) +table.SetAlignment(ALIGN_LEFT) +table.SetCenterSeparator("") +table.SetColumnSeparator("") +table.SetRowSeparator("") +table.SetHeaderLine(false) +table.SetBorder(false) +table.SetTablePadding("\t") // pad with tabs +table.SetNoWhiteSpace(true) +table.AppendBulk(data) // Add Bulk Data +table.Render() +``` + +##### Output 10 +``` +NAME STATUS ROLE VERSION +node1.example.com Ready compute 1.11 +node2.example.com Ready compute 1.11 +node3.example.com Ready compute 1.11 +node4.example.com NotReady compute 1.11 +``` + +#### Render table into a string + +Instead of rendering the table to `io.Stdout` you can also render it into a string. Go 1.10 introduced the `strings.Builder` type which implements the `io.Writer` interface and can therefore be used for this task. Example: + +```go +package main + +import ( + "strings" + "fmt" + + "github.com/olekukonko/tablewriter" +) + +func main() { + tableString := &strings.Builder{} + table := tablewriter.NewWriter(tableString) + + /* + * Code to fill the table + */ + + table.Render() + + fmt.Println(tableString.String()) +} +``` + +#### TODO +- ~~Import Directly from CSV~~ - `done` +- ~~Support for `SetFooter`~~ - `done` +- ~~Support for `SetBorder`~~ - `done` +- ~~Support table with uneven rows~~ - `done` +- ~~Support custom alignment~~ +- General Improvement & Optimisation +- `NewHTML` Parse table from HTML diff --git a/vendor/github.com/olekukonko/tablewriter/csv.go b/vendor/github.com/olekukonko/tablewriter/csv.go new file mode 100644 index 0000000000..98878303bc --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/csv.go @@ -0,0 +1,52 @@ +// Copyright 2014 Oleku Konko All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +// This module is a Table Writer API for the Go Programming Language. +// The protocols were written in pure Go and works on windows and unix systems + +package tablewriter + +import ( + "encoding/csv" + "io" + "os" +) + +// Start A new table by importing from a CSV file +// Takes io.Writer and csv File name +func NewCSV(writer io.Writer, fileName string, hasHeader bool) (*Table, error) { + file, err := os.Open(fileName) + if err != nil { + return &Table{}, err + } + defer file.Close() + csvReader := csv.NewReader(file) + t, err := NewCSVReader(writer, csvReader, hasHeader) + return t, err +} + +// Start a New Table Writer with csv.Reader +// This enables customisation such as reader.Comma = ';' +// See http://golang.org/src/pkg/encoding/csv/reader.go?s=3213:3671#L94 +func NewCSVReader(writer io.Writer, csvReader *csv.Reader, hasHeader bool) (*Table, error) { + t := NewWriter(writer) + if hasHeader { + // Read the first row + headers, err := csvReader.Read() + if err != nil { + return &Table{}, err + } + t.SetHeader(headers) + } + for { + record, err := csvReader.Read() + if err == io.EOF { + break + } else if err != nil { + return &Table{}, err + } + t.Append(record) + } + return t, nil +} diff --git a/vendor/github.com/olekukonko/tablewriter/table.go b/vendor/github.com/olekukonko/tablewriter/table.go new file mode 100644 index 0000000000..f913149c69 --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/table.go @@ -0,0 +1,967 @@ +// Copyright 2014 Oleku Konko All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +// This module is a Table Writer API for the Go Programming Language. +// The protocols were written in pure Go and works on windows and unix systems + +// Create & Generate text based table +package tablewriter + +import ( + "bytes" + "fmt" + "io" + "regexp" + "strings" +) + +const ( + MAX_ROW_WIDTH = 30 +) + +const ( + CENTER = "+" + ROW = "-" + COLUMN = "|" + SPACE = " " + NEWLINE = "\n" +) + +const ( + ALIGN_DEFAULT = iota + ALIGN_CENTER + ALIGN_RIGHT + ALIGN_LEFT +) + +var ( + decimal = regexp.MustCompile(`^-?(?:\d{1,3}(?:,\d{3})*|\d+)(?:\.\d+)?$`) + percent = regexp.MustCompile(`^-?\d+\.?\d*$%$`) +) + +type Border struct { + Left bool + Right bool + Top bool + Bottom bool +} + +type Table struct { + out io.Writer + rows [][]string + lines [][][]string + cs map[int]int + rs map[int]int + headers [][]string + footers [][]string + caption bool + captionText string + autoFmt bool + autoWrap bool + reflowText bool + mW int + pCenter string + pRow string + pColumn string + tColumn int + tRow int + hAlign int + fAlign int + align int + newLine string + rowLine bool + autoMergeCells bool + columnsToAutoMergeCells map[int]bool + noWhiteSpace bool + tablePadding string + hdrLine bool + borders Border + colSize int + headerParams []string + columnsParams []string + footerParams []string + columnsAlign []int +} + +// Start New Table +// Take io.Writer Directly +func NewWriter(writer io.Writer) *Table { + t := &Table{ + out: writer, + rows: [][]string{}, + lines: [][][]string{}, + cs: make(map[int]int), + rs: make(map[int]int), + headers: [][]string{}, + footers: [][]string{}, + caption: false, + captionText: "Table caption.", + autoFmt: true, + autoWrap: true, + reflowText: true, + mW: MAX_ROW_WIDTH, + pCenter: CENTER, + pRow: ROW, + pColumn: COLUMN, + tColumn: -1, + tRow: -1, + hAlign: ALIGN_DEFAULT, + fAlign: ALIGN_DEFAULT, + align: ALIGN_DEFAULT, + newLine: NEWLINE, + rowLine: false, + hdrLine: true, + borders: Border{Left: true, Right: true, Bottom: true, Top: true}, + colSize: -1, + headerParams: []string{}, + columnsParams: []string{}, + footerParams: []string{}, + columnsAlign: []int{}} + return t +} + +// Render table output +func (t *Table) Render() { + if t.borders.Top { + t.printLine(true) + } + t.printHeading() + if t.autoMergeCells { + t.printRowsMergeCells() + } else { + t.printRows() + } + if !t.rowLine && t.borders.Bottom { + t.printLine(true) + } + t.printFooter() + + if t.caption { + t.printCaption() + } +} + +const ( + headerRowIdx = -1 + footerRowIdx = -2 +) + +// Set table header +func (t *Table) SetHeader(keys []string) { + t.colSize = len(keys) + for i, v := range keys { + lines := t.parseDimension(v, i, headerRowIdx) + t.headers = append(t.headers, lines) + } +} + +// Set table Footer +func (t *Table) SetFooter(keys []string) { + //t.colSize = len(keys) + for i, v := range keys { + lines := t.parseDimension(v, i, footerRowIdx) + t.footers = append(t.footers, lines) + } +} + +// Set table Caption +func (t *Table) SetCaption(caption bool, captionText ...string) { + t.caption = caption + if len(captionText) == 1 { + t.captionText = captionText[0] + } +} + +// Turn header autoformatting on/off. Default is on (true). +func (t *Table) SetAutoFormatHeaders(auto bool) { + t.autoFmt = auto +} + +// Turn automatic multiline text adjustment on/off. Default is on (true). +func (t *Table) SetAutoWrapText(auto bool) { + t.autoWrap = auto +} + +// Turn automatic reflowing of multiline text when rewrapping. Default is on (true). +func (t *Table) SetReflowDuringAutoWrap(auto bool) { + t.reflowText = auto +} + +// Set the Default column width +func (t *Table) SetColWidth(width int) { + t.mW = width +} + +// Set the minimal width for a column +func (t *Table) SetColMinWidth(column int, width int) { + t.cs[column] = width +} + +// Set the Column Separator +func (t *Table) SetColumnSeparator(sep string) { + t.pColumn = sep +} + +// Set the Row Separator +func (t *Table) SetRowSeparator(sep string) { + t.pRow = sep +} + +// Set the center Separator +func (t *Table) SetCenterSeparator(sep string) { + t.pCenter = sep +} + +// Set Header Alignment +func (t *Table) SetHeaderAlignment(hAlign int) { + t.hAlign = hAlign +} + +// Set Footer Alignment +func (t *Table) SetFooterAlignment(fAlign int) { + t.fAlign = fAlign +} + +// Set Table Alignment +func (t *Table) SetAlignment(align int) { + t.align = align +} + +// Set No White Space +func (t *Table) SetNoWhiteSpace(allow bool) { + t.noWhiteSpace = allow +} + +// Set Table Padding +func (t *Table) SetTablePadding(padding string) { + t.tablePadding = padding +} + +func (t *Table) SetColumnAlignment(keys []int) { + for _, v := range keys { + switch v { + case ALIGN_CENTER: + break + case ALIGN_LEFT: + break + case ALIGN_RIGHT: + break + default: + v = ALIGN_DEFAULT + } + t.columnsAlign = append(t.columnsAlign, v) + } +} + +// Set New Line +func (t *Table) SetNewLine(nl string) { + t.newLine = nl +} + +// Set Header Line +// This would enable / disable a line after the header +func (t *Table) SetHeaderLine(line bool) { + t.hdrLine = line +} + +// Set Row Line +// This would enable / disable a line on each row of the table +func (t *Table) SetRowLine(line bool) { + t.rowLine = line +} + +// Set Auto Merge Cells +// This would enable / disable the merge of cells with identical values +func (t *Table) SetAutoMergeCells(auto bool) { + t.autoMergeCells = auto +} + +// Set Auto Merge Cells By Column Index +// This would enable / disable the merge of cells with identical values for specific columns +// If cols is empty, it is the same as `SetAutoMergeCells(true)`. +func (t *Table) SetAutoMergeCellsByColumnIndex(cols []int) { + t.autoMergeCells = true + + if len(cols) > 0 { + m := make(map[int]bool) + for _, col := range cols { + m[col] = true + } + t.columnsToAutoMergeCells = m + } +} + +// Set Table Border +// This would enable / disable line around the table +func (t *Table) SetBorder(border bool) { + t.SetBorders(Border{border, border, border, border}) +} + +func (t *Table) SetBorders(border Border) { + t.borders = border +} + +// Append row to table +func (t *Table) Append(row []string) { + rowSize := len(t.headers) + if rowSize > t.colSize { + t.colSize = rowSize + } + + n := len(t.lines) + line := [][]string{} + for i, v := range row { + + // Detect string width + // Detect String height + // Break strings into words + out := t.parseDimension(v, i, n) + + // Append broken words + line = append(line, out) + } + t.lines = append(t.lines, line) +} + +// Append row to table with color attributes +func (t *Table) Rich(row []string, colors []Colors) { + rowSize := len(t.headers) + if rowSize > t.colSize { + t.colSize = rowSize + } + + n := len(t.lines) + line := [][]string{} + for i, v := range row { + + // Detect string width + // Detect String height + // Break strings into words + out := t.parseDimension(v, i, n) + + if len(colors) > i { + color := colors[i] + out[0] = format(out[0], color) + } + + // Append broken words + line = append(line, out) + } + t.lines = append(t.lines, line) +} + +// Allow Support for Bulk Append +// Eliminates repeated for loops +func (t *Table) AppendBulk(rows [][]string) { + for _, row := range rows { + t.Append(row) + } +} + +// NumLines to get the number of lines +func (t *Table) NumLines() int { + return len(t.lines) +} + +// Clear rows +func (t *Table) ClearRows() { + t.lines = [][][]string{} +} + +// Clear footer +func (t *Table) ClearFooter() { + t.footers = [][]string{} +} + +// Center based on position and border. +func (t *Table) center(i int) string { + if i == -1 && !t.borders.Left { + return t.pRow + } + + if i == len(t.cs)-1 && !t.borders.Right { + return t.pRow + } + + return t.pCenter +} + +// Print line based on row width +func (t *Table) printLine(nl bool) { + fmt.Fprint(t.out, t.center(-1)) + for i := 0; i < len(t.cs); i++ { + v := t.cs[i] + fmt.Fprintf(t.out, "%s%s%s%s", + t.pRow, + strings.Repeat(string(t.pRow), v), + t.pRow, + t.center(i)) + } + if nl { + fmt.Fprint(t.out, t.newLine) + } +} + +// Print line based on row width with our without cell separator +func (t *Table) printLineOptionalCellSeparators(nl bool, displayCellSeparator []bool) { + fmt.Fprint(t.out, t.pCenter) + for i := 0; i < len(t.cs); i++ { + v := t.cs[i] + if i > len(displayCellSeparator) || displayCellSeparator[i] { + // Display the cell separator + fmt.Fprintf(t.out, "%s%s%s%s", + t.pRow, + strings.Repeat(string(t.pRow), v), + t.pRow, + t.pCenter) + } else { + // Don't display the cell separator for this cell + fmt.Fprintf(t.out, "%s%s", + strings.Repeat(" ", v+2), + t.pCenter) + } + } + if nl { + fmt.Fprint(t.out, t.newLine) + } +} + +// Return the PadRight function if align is left, PadLeft if align is right, +// and Pad by default +func pad(align int) func(string, string, int) string { + padFunc := Pad + switch align { + case ALIGN_LEFT: + padFunc = PadRight + case ALIGN_RIGHT: + padFunc = PadLeft + } + return padFunc +} + +// Print heading information +func (t *Table) printHeading() { + // Check if headers is available + if len(t.headers) < 1 { + return + } + + // Identify last column + end := len(t.cs) - 1 + + // Get pad function + padFunc := pad(t.hAlign) + + // Checking for ANSI escape sequences for header + is_esc_seq := false + if len(t.headerParams) > 0 { + is_esc_seq = true + } + + // Maximum height. + max := t.rs[headerRowIdx] + + // Print Heading + for x := 0; x < max; x++ { + // Check if border is set + // Replace with space if not set + if !t.noWhiteSpace { + fmt.Fprint(t.out, ConditionString(t.borders.Left, t.pColumn, SPACE)) + } + + for y := 0; y <= end; y++ { + v := t.cs[y] + h := "" + + if y < len(t.headers) && x < len(t.headers[y]) { + h = t.headers[y][x] + } + if t.autoFmt { + h = Title(h) + } + pad := ConditionString((y == end && !t.borders.Left), SPACE, t.pColumn) + if t.noWhiteSpace { + pad = ConditionString((y == end && !t.borders.Left), SPACE, t.tablePadding) + } + if is_esc_seq { + if !t.noWhiteSpace { + fmt.Fprintf(t.out, " %s %s", + format(padFunc(h, SPACE, v), + t.headerParams[y]), pad) + } else { + fmt.Fprintf(t.out, "%s %s", + format(padFunc(h, SPACE, v), + t.headerParams[y]), pad) + } + } else { + if !t.noWhiteSpace { + fmt.Fprintf(t.out, " %s %s", + padFunc(h, SPACE, v), + pad) + } else { + // the spaces between breaks the kube formatting + fmt.Fprintf(t.out, "%s%s", + padFunc(h, SPACE, v), + pad) + } + } + } + // Next line + fmt.Fprint(t.out, t.newLine) + } + if t.hdrLine { + t.printLine(true) + } +} + +// Print heading information +func (t *Table) printFooter() { + // Check if headers is available + if len(t.footers) < 1 { + return + } + + // Only print line if border is not set + if !t.borders.Bottom { + t.printLine(true) + } + + // Identify last column + end := len(t.cs) - 1 + + // Get pad function + padFunc := pad(t.fAlign) + + // Checking for ANSI escape sequences for header + is_esc_seq := false + if len(t.footerParams) > 0 { + is_esc_seq = true + } + + // Maximum height. + max := t.rs[footerRowIdx] + + // Print Footer + erasePad := make([]bool, len(t.footers)) + for x := 0; x < max; x++ { + // Check if border is set + // Replace with space if not set + fmt.Fprint(t.out, ConditionString(t.borders.Bottom, t.pColumn, SPACE)) + + for y := 0; y <= end; y++ { + v := t.cs[y] + f := "" + if y < len(t.footers) && x < len(t.footers[y]) { + f = t.footers[y][x] + } + if t.autoFmt { + f = Title(f) + } + pad := ConditionString((y == end && !t.borders.Top), SPACE, t.pColumn) + + if erasePad[y] || (x == 0 && len(f) == 0) { + pad = SPACE + erasePad[y] = true + } + + if is_esc_seq { + fmt.Fprintf(t.out, " %s %s", + format(padFunc(f, SPACE, v), + t.footerParams[y]), pad) + } else { + fmt.Fprintf(t.out, " %s %s", + padFunc(f, SPACE, v), + pad) + } + + //fmt.Fprintf(t.out, " %s %s", + // padFunc(f, SPACE, v), + // pad) + } + // Next line + fmt.Fprint(t.out, t.newLine) + //t.printLine(true) + } + + hasPrinted := false + + for i := 0; i <= end; i++ { + v := t.cs[i] + pad := t.pRow + center := t.pCenter + length := len(t.footers[i][0]) + + if length > 0 { + hasPrinted = true + } + + // Set center to be space if length is 0 + if length == 0 && !t.borders.Right { + center = SPACE + } + + // Print first junction + if i == 0 { + if length > 0 && !t.borders.Left { + center = t.pRow + } + fmt.Fprint(t.out, center) + } + + // Pad With space of length is 0 + if length == 0 { + pad = SPACE + } + // Ignore left space as it has printed before + if hasPrinted || t.borders.Left { + pad = t.pRow + center = t.pCenter + } + + // Change Center end position + if center != SPACE { + if i == end && !t.borders.Right { + center = t.pRow + } + } + + // Change Center start position + if center == SPACE { + if i < end && len(t.footers[i+1][0]) != 0 { + if !t.borders.Left { + center = t.pRow + } else { + center = t.pCenter + } + } + } + + // Print the footer + fmt.Fprintf(t.out, "%s%s%s%s", + pad, + strings.Repeat(string(pad), v), + pad, + center) + + } + + fmt.Fprint(t.out, t.newLine) +} + +// Print caption text +func (t Table) printCaption() { + width := t.getTableWidth() + paragraph, _ := WrapString(t.captionText, width) + for linecount := 0; linecount < len(paragraph); linecount++ { + fmt.Fprintln(t.out, paragraph[linecount]) + } +} + +// Calculate the total number of characters in a row +func (t Table) getTableWidth() int { + var chars int + for _, v := range t.cs { + chars += v + } + + // Add chars, spaces, seperators to calculate the total width of the table. + // ncols := t.colSize + // spaces := ncols * 2 + // seps := ncols + 1 + + return (chars + (3 * t.colSize) + 2) +} + +func (t Table) printRows() { + for i, lines := range t.lines { + t.printRow(lines, i) + } +} + +func (t *Table) fillAlignment(num int) { + if len(t.columnsAlign) < num { + t.columnsAlign = make([]int, num) + for i := range t.columnsAlign { + t.columnsAlign[i] = t.align + } + } +} + +// Print Row Information +// Adjust column alignment based on type + +func (t *Table) printRow(columns [][]string, rowIdx int) { + // Get Maximum Height + max := t.rs[rowIdx] + total := len(columns) + + // TODO Fix uneven col size + // if total < t.colSize { + // for n := t.colSize - total; n < t.colSize ; n++ { + // columns = append(columns, []string{SPACE}) + // t.cs[n] = t.mW + // } + //} + + // Pad Each Height + pads := []int{} + + // Checking for ANSI escape sequences for columns + is_esc_seq := false + if len(t.columnsParams) > 0 { + is_esc_seq = true + } + t.fillAlignment(total) + + for i, line := range columns { + length := len(line) + pad := max - length + pads = append(pads, pad) + for n := 0; n < pad; n++ { + columns[i] = append(columns[i], " ") + } + } + //fmt.Println(max, "\n") + for x := 0; x < max; x++ { + for y := 0; y < total; y++ { + + // Check if border is set + if !t.noWhiteSpace { + fmt.Fprint(t.out, ConditionString((!t.borders.Left && y == 0), SPACE, t.pColumn)) + fmt.Fprintf(t.out, SPACE) + } + + str := columns[y][x] + + // Embedding escape sequence with column value + if is_esc_seq { + str = format(str, t.columnsParams[y]) + } + + // This would print alignment + // Default alignment would use multiple configuration + switch t.columnsAlign[y] { + case ALIGN_CENTER: // + fmt.Fprintf(t.out, "%s", Pad(str, SPACE, t.cs[y])) + case ALIGN_RIGHT: + fmt.Fprintf(t.out, "%s", PadLeft(str, SPACE, t.cs[y])) + case ALIGN_LEFT: + fmt.Fprintf(t.out, "%s", PadRight(str, SPACE, t.cs[y])) + default: + if decimal.MatchString(strings.TrimSpace(str)) || percent.MatchString(strings.TrimSpace(str)) { + fmt.Fprintf(t.out, "%s", PadLeft(str, SPACE, t.cs[y])) + } else { + fmt.Fprintf(t.out, "%s", PadRight(str, SPACE, t.cs[y])) + + // TODO Custom alignment per column + //if max == 1 || pads[y] > 0 { + // fmt.Fprintf(t.out, "%s", Pad(str, SPACE, t.cs[y])) + //} else { + // fmt.Fprintf(t.out, "%s", PadRight(str, SPACE, t.cs[y])) + //} + + } + } + if !t.noWhiteSpace { + fmt.Fprintf(t.out, SPACE) + } else { + fmt.Fprintf(t.out, t.tablePadding) + } + } + // Check if border is set + // Replace with space if not set + if !t.noWhiteSpace { + fmt.Fprint(t.out, ConditionString(t.borders.Left, t.pColumn, SPACE)) + } + fmt.Fprint(t.out, t.newLine) + } + + if t.rowLine { + t.printLine(true) + } +} + +// Print the rows of the table and merge the cells that are identical +func (t *Table) printRowsMergeCells() { + var previousLine []string + var displayCellBorder []bool + var tmpWriter bytes.Buffer + for i, lines := range t.lines { + // We store the display of the current line in a tmp writer, as we need to know which border needs to be print above + previousLine, displayCellBorder = t.printRowMergeCells(&tmpWriter, lines, i, previousLine) + if i > 0 { //We don't need to print borders above first line + if t.rowLine { + t.printLineOptionalCellSeparators(true, displayCellBorder) + } + } + tmpWriter.WriteTo(t.out) + } + //Print the end of the table + if t.rowLine { + t.printLine(true) + } +} + +// Print Row Information to a writer and merge identical cells. +// Adjust column alignment based on type + +func (t *Table) printRowMergeCells(writer io.Writer, columns [][]string, rowIdx int, previousLine []string) ([]string, []bool) { + // Get Maximum Height + max := t.rs[rowIdx] + total := len(columns) + + // Pad Each Height + pads := []int{} + + // Checking for ANSI escape sequences for columns + is_esc_seq := false + if len(t.columnsParams) > 0 { + is_esc_seq = true + } + for i, line := range columns { + length := len(line) + pad := max - length + pads = append(pads, pad) + for n := 0; n < pad; n++ { + columns[i] = append(columns[i], " ") + } + } + + var displayCellBorder []bool + t.fillAlignment(total) + for x := 0; x < max; x++ { + for y := 0; y < total; y++ { + + // Check if border is set + fmt.Fprint(writer, ConditionString((!t.borders.Left && y == 0), SPACE, t.pColumn)) + + fmt.Fprintf(writer, SPACE) + + str := columns[y][x] + + // Embedding escape sequence with column value + if is_esc_seq { + str = format(str, t.columnsParams[y]) + } + + if t.autoMergeCells { + var mergeCell bool + if t.columnsToAutoMergeCells != nil { + // Check to see if the column index is in columnsToAutoMergeCells. + if t.columnsToAutoMergeCells[y] { + mergeCell = true + } + } else { + // columnsToAutoMergeCells was not set. + mergeCell = true + } + //Store the full line to merge mutli-lines cells + fullLine := strings.TrimRight(strings.Join(columns[y], " "), " ") + if len(previousLine) > y && fullLine == previousLine[y] && fullLine != "" && mergeCell { + // If this cell is identical to the one above but not empty, we don't display the border and keep the cell empty. + displayCellBorder = append(displayCellBorder, false) + str = "" + } else { + // First line or different content, keep the content and print the cell border + displayCellBorder = append(displayCellBorder, true) + } + } + + // This would print alignment + // Default alignment would use multiple configuration + switch t.columnsAlign[y] { + case ALIGN_CENTER: // + fmt.Fprintf(writer, "%s", Pad(str, SPACE, t.cs[y])) + case ALIGN_RIGHT: + fmt.Fprintf(writer, "%s", PadLeft(str, SPACE, t.cs[y])) + case ALIGN_LEFT: + fmt.Fprintf(writer, "%s", PadRight(str, SPACE, t.cs[y])) + default: + if decimal.MatchString(strings.TrimSpace(str)) || percent.MatchString(strings.TrimSpace(str)) { + fmt.Fprintf(writer, "%s", PadLeft(str, SPACE, t.cs[y])) + } else { + fmt.Fprintf(writer, "%s", PadRight(str, SPACE, t.cs[y])) + } + } + fmt.Fprintf(writer, SPACE) + } + // Check if border is set + // Replace with space if not set + fmt.Fprint(writer, ConditionString(t.borders.Left, t.pColumn, SPACE)) + fmt.Fprint(writer, t.newLine) + } + + //The new previous line is the current one + previousLine = make([]string, total) + for y := 0; y < total; y++ { + previousLine[y] = strings.TrimRight(strings.Join(columns[y], " "), " ") //Store the full line for multi-lines cells + } + //Returns the newly added line and wether or not a border should be displayed above. + return previousLine, displayCellBorder +} + +func (t *Table) parseDimension(str string, colKey, rowKey int) []string { + var ( + raw []string + maxWidth int + ) + + raw = getLines(str) + maxWidth = 0 + for _, line := range raw { + if w := DisplayWidth(line); w > maxWidth { + maxWidth = w + } + } + + // If wrapping, ensure that all paragraphs in the cell fit in the + // specified width. + if t.autoWrap { + // If there's a maximum allowed width for wrapping, use that. + if maxWidth > t.mW { + maxWidth = t.mW + } + + // In the process of doing so, we need to recompute maxWidth. This + // is because perhaps a word in the cell is longer than the + // allowed maximum width in t.mW. + newMaxWidth := maxWidth + newRaw := make([]string, 0, len(raw)) + + if t.reflowText { + // Make a single paragraph of everything. + raw = []string{strings.Join(raw, " ")} + } + for i, para := range raw { + paraLines, _ := WrapString(para, maxWidth) + for _, line := range paraLines { + if w := DisplayWidth(line); w > newMaxWidth { + newMaxWidth = w + } + } + if i > 0 { + newRaw = append(newRaw, " ") + } + newRaw = append(newRaw, paraLines...) + } + raw = newRaw + maxWidth = newMaxWidth + } + + // Store the new known maximum width. + v, ok := t.cs[colKey] + if !ok || v < maxWidth || v == 0 { + t.cs[colKey] = maxWidth + } + + // Remember the number of lines for the row printer. + h := len(raw) + v, ok = t.rs[rowKey] + + if !ok || v < h || v == 0 { + t.rs[rowKey] = h + } + //fmt.Printf("Raw %+v %d\n", raw, len(raw)) + return raw +} diff --git a/vendor/github.com/olekukonko/tablewriter/table_with_color.go b/vendor/github.com/olekukonko/tablewriter/table_with_color.go new file mode 100644 index 0000000000..ae7a364aed --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/table_with_color.go @@ -0,0 +1,136 @@ +package tablewriter + +import ( + "fmt" + "strconv" + "strings" +) + +const ESC = "\033" +const SEP = ";" + +const ( + BgBlackColor int = iota + 40 + BgRedColor + BgGreenColor + BgYellowColor + BgBlueColor + BgMagentaColor + BgCyanColor + BgWhiteColor +) + +const ( + FgBlackColor int = iota + 30 + FgRedColor + FgGreenColor + FgYellowColor + FgBlueColor + FgMagentaColor + FgCyanColor + FgWhiteColor +) + +const ( + BgHiBlackColor int = iota + 100 + BgHiRedColor + BgHiGreenColor + BgHiYellowColor + BgHiBlueColor + BgHiMagentaColor + BgHiCyanColor + BgHiWhiteColor +) + +const ( + FgHiBlackColor int = iota + 90 + FgHiRedColor + FgHiGreenColor + FgHiYellowColor + FgHiBlueColor + FgHiMagentaColor + FgHiCyanColor + FgHiWhiteColor +) + +const ( + Normal = 0 + Bold = 1 + UnderlineSingle = 4 + Italic +) + +type Colors []int + +func startFormat(seq string) string { + return fmt.Sprintf("%s[%sm", ESC, seq) +} + +func stopFormat() string { + return fmt.Sprintf("%s[%dm", ESC, Normal) +} + +// Making the SGR (Select Graphic Rendition) sequence. +func makeSequence(codes []int) string { + codesInString := []string{} + for _, code := range codes { + codesInString = append(codesInString, strconv.Itoa(code)) + } + return strings.Join(codesInString, SEP) +} + +// Adding ANSI escape sequences before and after string +func format(s string, codes interface{}) string { + var seq string + + switch v := codes.(type) { + + case string: + seq = v + case []int: + seq = makeSequence(v) + case Colors: + seq = makeSequence(v) + default: + return s + } + + if len(seq) == 0 { + return s + } + return startFormat(seq) + s + stopFormat() +} + +// Adding header colors (ANSI codes) +func (t *Table) SetHeaderColor(colors ...Colors) { + if t.colSize != len(colors) { + panic("Number of header colors must be equal to number of headers.") + } + for i := 0; i < len(colors); i++ { + t.headerParams = append(t.headerParams, makeSequence(colors[i])) + } +} + +// Adding column colors (ANSI codes) +func (t *Table) SetColumnColor(colors ...Colors) { + if t.colSize != len(colors) { + panic("Number of column colors must be equal to number of headers.") + } + for i := 0; i < len(colors); i++ { + t.columnsParams = append(t.columnsParams, makeSequence(colors[i])) + } +} + +// Adding column colors (ANSI codes) +func (t *Table) SetFooterColor(colors ...Colors) { + if len(t.footers) != len(colors) { + panic("Number of footer colors must be equal to number of footer.") + } + for i := 0; i < len(colors); i++ { + t.footerParams = append(t.footerParams, makeSequence(colors[i])) + } +} + +func Color(colors ...int) []int { + return colors +} diff --git a/vendor/github.com/olekukonko/tablewriter/util.go b/vendor/github.com/olekukonko/tablewriter/util.go new file mode 100644 index 0000000000..380e7ab35b --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/util.go @@ -0,0 +1,93 @@ +// Copyright 2014 Oleku Konko All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +// This module is a Table Writer API for the Go Programming Language. +// The protocols were written in pure Go and works on windows and unix systems + +package tablewriter + +import ( + "math" + "regexp" + "strings" + + "github.com/mattn/go-runewidth" +) + +var ansi = regexp.MustCompile("\033\\[(?:[0-9]{1,3}(?:;[0-9]{1,3})*)?[m|K]") + +func DisplayWidth(str string) int { + return runewidth.StringWidth(ansi.ReplaceAllLiteralString(str, "")) +} + +// Simple Condition for string +// Returns value based on condition +func ConditionString(cond bool, valid, inValid string) string { + if cond { + return valid + } + return inValid +} + +func isNumOrSpace(r rune) bool { + return ('0' <= r && r <= '9') || r == ' ' +} + +// Format Table Header +// Replace _ , . and spaces +func Title(name string) string { + origLen := len(name) + rs := []rune(name) + for i, r := range rs { + switch r { + case '_': + rs[i] = ' ' + case '.': + // ignore floating number 0.0 + if (i != 0 && !isNumOrSpace(rs[i-1])) || (i != len(rs)-1 && !isNumOrSpace(rs[i+1])) { + rs[i] = ' ' + } + } + } + name = string(rs) + name = strings.TrimSpace(name) + if len(name) == 0 && origLen > 0 { + // Keep at least one character. This is important to preserve + // empty lines in multi-line headers/footers. + name = " " + } + return strings.ToUpper(name) +} + +// Pad String +// Attempts to place string in the center +func Pad(s, pad string, width int) string { + gap := width - DisplayWidth(s) + if gap > 0 { + gapLeft := int(math.Ceil(float64(gap / 2))) + gapRight := gap - gapLeft + return strings.Repeat(string(pad), gapLeft) + s + strings.Repeat(string(pad), gapRight) + } + return s +} + +// Pad String Right position +// This would place string at the left side of the screen +func PadRight(s, pad string, width int) string { + gap := width - DisplayWidth(s) + if gap > 0 { + return s + strings.Repeat(string(pad), gap) + } + return s +} + +// Pad String Left position +// This would place string at the right side of the screen +func PadLeft(s, pad string, width int) string { + gap := width - DisplayWidth(s) + if gap > 0 { + return strings.Repeat(string(pad), gap) + s + } + return s +} diff --git a/vendor/github.com/olekukonko/tablewriter/wrap.go b/vendor/github.com/olekukonko/tablewriter/wrap.go new file mode 100644 index 0000000000..a092ee1f75 --- /dev/null +++ b/vendor/github.com/olekukonko/tablewriter/wrap.go @@ -0,0 +1,99 @@ +// Copyright 2014 Oleku Konko All rights reserved. +// Use of this source code is governed by a MIT +// license that can be found in the LICENSE file. + +// This module is a Table Writer API for the Go Programming Language. +// The protocols were written in pure Go and works on windows and unix systems + +package tablewriter + +import ( + "math" + "strings" + + "github.com/mattn/go-runewidth" +) + +var ( + nl = "\n" + sp = " " +) + +const defaultPenalty = 1e5 + +// Wrap wraps s into a paragraph of lines of length lim, with minimal +// raggedness. +func WrapString(s string, lim int) ([]string, int) { + words := strings.Split(strings.Replace(s, nl, sp, -1), sp) + var lines []string + max := 0 + for _, v := range words { + max = runewidth.StringWidth(v) + if max > lim { + lim = max + } + } + for _, line := range WrapWords(words, 1, lim, defaultPenalty) { + lines = append(lines, strings.Join(line, sp)) + } + return lines, lim +} + +// WrapWords is the low-level line-breaking algorithm, useful if you need more +// control over the details of the text wrapping process. For most uses, +// WrapString will be sufficient and more convenient. +// +// WrapWords splits a list of words into lines with minimal "raggedness", +// treating each rune as one unit, accounting for spc units between adjacent +// words on each line, and attempting to limit lines to lim units. Raggedness +// is the total error over all lines, where error is the square of the +// difference of the length of the line and lim. Too-long lines (which only +// happen when a single word is longer than lim units) have pen penalty units +// added to the error. +func WrapWords(words []string, spc, lim, pen int) [][]string { + n := len(words) + + length := make([][]int, n) + for i := 0; i < n; i++ { + length[i] = make([]int, n) + length[i][i] = runewidth.StringWidth(words[i]) + for j := i + 1; j < n; j++ { + length[i][j] = length[i][j-1] + spc + runewidth.StringWidth(words[j]) + } + } + nbrk := make([]int, n) + cost := make([]int, n) + for i := range cost { + cost[i] = math.MaxInt32 + } + for i := n - 1; i >= 0; i-- { + if length[i][n-1] <= lim { + cost[i] = 0 + nbrk[i] = n + } else { + for j := i + 1; j < n; j++ { + d := lim - length[i][j-1] + c := d*d + cost[j] + if length[i][j-1] > lim { + c += pen // too-long lines get a worse penalty + } + if c < cost[i] { + cost[i] = c + nbrk[i] = j + } + } + } + } + var lines [][]string + i := 0 + for i < n { + lines = append(lines, words[i:nbrk[i]]) + i = nbrk[i] + } + return lines +} + +// getLines decomposes a multiline string into a slice of strings. +func getLines(s string) []string { + return strings.Split(s, nl) +} diff --git a/vendor/github.com/parquet-go/parquet-go/.gitattributes b/vendor/github.com/parquet-go/parquet-go/.gitattributes new file mode 100644 index 0000000000..44d8f56ba0 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/.gitattributes @@ -0,0 +1 @@ +internal/gen-go/* linguist-generated=true \ No newline at end of file diff --git a/vendor/github.com/parquet-go/parquet-go/.gitignore b/vendor/github.com/parquet-go/parquet-go/.gitignore new file mode 100644 index 0000000000..b3584c8d4d --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/.gitignore @@ -0,0 +1,21 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.py + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Emacs +*~ +#*# +.# diff --git a/vendor/github.com/parquet-go/parquet-go/.mailmap b/vendor/github.com/parquet-go/parquet-go/.mailmap new file mode 100644 index 0000000000..09d89739a5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/.mailmap @@ -0,0 +1,2 @@ +Achille Roussel Achille +Thomas Pelletier Thomas Pelletier diff --git a/vendor/github.com/parquet-go/parquet-go/.words b/vendor/github.com/parquet-go/parquet-go/.words new file mode 100644 index 0000000000..d7ff365560 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/.words @@ -0,0 +1,27 @@ + +RowType +Twilio +bottlenecked +decompressors +int96 +millis +nanos +reindexing +repositions +schemas +ColumnPages +PageIndex +Zstandard +xxHash +cardinality +enums +32bit +dic +Blart +Versenwald +purego +stdlib +unscaled +cespare +bitset +checksumming diff --git a/vendor/github.com/parquet-go/parquet-go/AUTHORS.txt b/vendor/github.com/parquet-go/parquet-go/AUTHORS.txt new file mode 100644 index 0000000000..26669538c1 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/AUTHORS.txt @@ -0,0 +1,5 @@ +Achille Roussel +Frederic Branczyk +Julien Fabre +Kevin Burke +Thomas Pelletier diff --git a/vendor/github.com/parquet-go/parquet-go/CHANGELOG.md b/vendor/github.com/parquet-go/parquet-go/CHANGELOG.md new file mode 100644 index 0000000000..987597104c --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/CHANGELOG.md @@ -0,0 +1,16 @@ + # v0.17.0 + + ## Breaking Changes + + - migrate to module github.com/parquet-go/parquet-go [#3](https://github.com/parquet-go/parquet-go/pull/3) @kevinburke + - drop support for `go1.17` [#16](https://github.com/parquet-go/parquet-go/pull/16) @gernest + + ## Bug fixes + + - fix error handling when reading from io.ReaderAt [#18](https://github.com/parquet-go/parquet-go/pull/18) @gernest + - fix zero value of nested field point [#9](https://github.com/parquet-go/parquet-go/pull/9) @gernest + - fix memory corruption in `MergeRowGroups` [#31](https://github.com/parquet-go/parquet-go/pull/31) @gernest + + ## Enhancements + - performance improvement on GenericReader [#17](https://github.com/parquet-go/parquet-go/pull/17) @gernest, @zolstein + - stabilize flakey `TestOpenFile` [#11](https://github.com/parquet-go/parquet-go/pull/11) @gernest diff --git a/vendor/github.com/parquet-go/parquet-go/CODEOWNERS b/vendor/github.com/parquet-go/parquet-go/CODEOWNERS new file mode 100644 index 0000000000..45632dfe36 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/CODEOWNERS @@ -0,0 +1 @@ +* @achille-roussel @fpetkovski @joe-elliott @thorfour diff --git a/vendor/github.com/parquet-go/parquet-go/CODE_OF_CONDUCT.md b/vendor/github.com/parquet-go/parquet-go/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..2f0727ed54 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/CODE_OF_CONDUCT.md @@ -0,0 +1,73 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at open-source@twilio.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org diff --git a/vendor/github.com/parquet-go/parquet-go/CONTRIBUTING.md b/vendor/github.com/parquet-go/parquet-go/CONTRIBUTING.md new file mode 100644 index 0000000000..185d958118 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing to segmentio/parquet + +## Code of Conduct + +Help us keep the project open and inclusive. Please be kind to and +considerate of other developers, as we all have the same goal: make +the project as good as it can be. + +* [Code of Conduct](./CODE_OF_CONDUCT.md) + +## Licensing + +All third party contributors acknowledge that any contributions they provide +will be made under the same open source license that the open source project +is provided under. + +## Contributing + +* Open an Issue to report bugs or discuss non-trivial changes. +* Open a Pull Request to submit a code change for review. + +### Guidelines for code review + +It's good to do code review but we are busy and it's bad to wait for consensus +or opinions that might not arrive. Here are some guidelines + +#### Changes where code review is optional + +- Documentation changes +- Bug fixes where a reproducible test case exists +- Keeping the lights on style work (compat with new Parquet versions, new Go + versions, for example) +- Updates to the CI environment +- Adding additional benchmarks or test cases +- Pull requests that have been open for 30 days, where an attempt has been made + to contact another code owner, and no one has expressly requested changes + +#### Changes that should get at least one code review from an owner + +- Changes that may affect the performance of core library functionality + (serializing, deserializing Parquet data) by more than 2% +- Behavior changes +- New API or changes to existing API + +### Coding Rules + +To ensure consistency throughout the source code, keep these rules in mind +when submitting contributions: + +* All features or bug fixes must be tested by one or more tests. +* All exported types, functions, and symbols must be documented. +* All code must be formatted with `go fmt`. diff --git a/vendor/github.com/parquet-go/parquet-go/LICENSE b/vendor/github.com/parquet-go/parquet-go/LICENSE new file mode 100644 index 0000000000..251c7e5154 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/LICENSE @@ -0,0 +1,213 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023 Twilio, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Parquet. + +* deprecated/parquet.go is based on Apache Parquet's thrift file +* format/parquet.go is based on Apache Parquet's thrift file + +Copyright: 2014 The Apache Software Foundation. +Home page: https://github.com/apache/parquet-format +License: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/vendor/github.com/parquet-go/parquet-go/Makefile b/vendor/github.com/parquet-go/parquet-go/Makefile new file mode 100644 index 0000000000..e3bfd8bc2d --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/Makefile @@ -0,0 +1,15 @@ +.PHONY: format + +AUTHORS.txt: .mailmap + go install github.com/kevinburke/write_mailmap@latest + write_mailmap > AUTHORS.txt + +tools: + go mod tidy -modfile go.tools.mod + +format: tools + go fmt ./... + go tool -modfile go.tools.mod modernize -fix -test ./... + +test: + go test -v -trimpath -race -cover -tags= ./... diff --git a/vendor/github.com/parquet-go/parquet-go/README.md b/vendor/github.com/parquet-go/parquet-go/README.md new file mode 100644 index 0000000000..49341ae5aa --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/README.md @@ -0,0 +1,581 @@ +

+ +
+

parquet-go/parquet-go

+

+High-performance Go library to manipulate parquet files, initially developed at +Twilio Segment. +

+ +
+ +## Motivation + +Parquet has been established as a powerful solution to represent columnar data +on persistent storage mediums, achieving levels of compression and query +performance that enable managing data sets at scales that reach the petabytes. +In addition, having intensive data applications sharing a common format creates +opportunities for interoperation in our tool kits, providing greater leverage +and value to engineers maintaining and operating those systems. + +The creation and evolution of large scale data management systems, combined with +realtime expectations come with challenging maintenance and performance +requirements, that existing solutions to use parquet with Go were not addressing. + +The `parquet-go/parquet-go` package was designed and developed to respond to those +challenges, offering high level APIs to read and write parquet files, while +keeping a low compute and memory footprint in order to be used in environments +where data volumes and cost constraints require software to achieve high levels +of efficiency. + +## Specification + +Columnar storage allows Parquet to store data more efficiently than, say, +using JSON or Protobuf. For more information, refer to the [Parquet Format Specification](https://github.com/apache/parquet-format). + +## Installation + +The package is distributed as a standard Go module that programs can take a +dependency on and install with the following command: + +```bash +go get github.com/parquet-go/parquet-go +``` + +Go 1.21 or later is required to use the package. + +### Compatibility Guarantees + +The package is currently released as a pre-v1 version, which gives maintainers +the freedom to break backward compatibility to help improve the APIs as we learn +which initial design decisions would need to be revisited to better support the +use cases that the library solves for. These occurrences are expected to be rare +in frequency and documentation will be produce to guide users on how to adapt +their programs to breaking changes. + +## Usage + +The following sections describe how to use APIs exposed by the library, +highlighting the use cases with code examples to demonstrate how they are used +in practice. + +### Writing Parquet Files: [parquet.GenericWriter[T]](https://pkg.go.dev/github.com/parquet-go/parquet-go#GenericWriter) + +A parquet file is a collection of rows sharing the same schema, arranged in +columns to support faster scan operations on subsets of the data set. + +For simple use cases, the `parquet.WriteFile[T]` function allows the creation +of parquet files on the file system from a slice of Go values representing the +rows to write to the file. + +```go +type RowType struct { FirstName, LastName string } + +if err := parquet.WriteFile("file.parquet", []RowType{ + {FirstName: "Bob"}, + {FirstName: "Alice"}, +}); err != nil { + ... +} +``` + +The `parquet.GenericWriter[T]` type denormalizes rows into columns, then encodes +the columns into a parquet file, generating row groups, column chunks, and pages +based on configurable heuristics. + +```go +type RowType struct { FirstName, LastName string } + +writer := parquet.NewGenericWriter[RowType](output) + +_, err := writer.Write([]RowType{ + ... +}) +if err != nil { + ... +} + +// Closing the writer is necessary to flush buffers and write the file footer. +if err := writer.Close(); err != nil { + ... +} +``` + +Explicit declaration of the parquet schema on a writer is useful when the +application needs to ensure that data written to a file adheres to a predefined +schema, which may differ from the schema derived from the writer's type +parameter. The `parquet.Schema` type is a in-memory representation of the schema +of parquet rows, translated from the type of Go values, and can be used for this +purpose. + +```go +schema := parquet.SchemaOf(new(RowType)) +writer := parquet.NewGenericWriter[any](output, schema) +... +``` + +### Reading Parquet Files: [parquet.GenericReader[T]](https://pkg.go.dev/github.com/parquet-go/parquet-go#GenericReader) + +For simple use cases where the data set fits in memory and the program will +read most rows of the file, the `parquet.ReadFile[T]` function returns a slice +of Go values representing the rows read from the file. + +```go +type RowType struct { FirstName, LastName string } + +rows, err := parquet.ReadFile[RowType]("file.parquet") +if err != nil { + ... +} + +for _, c := range rows { + fmt.Printf("%+v\n", c) +} +``` + +The expected schema of rows can be explicitly declared when the reader is +constructed, which is useful to ensure that the program receives rows matching +an specific format; for example, when dealing with files from remote storage +sources that applications cannot trust to have used an expected schema. + +Configuring the schema of a reader is done by passing a `parquet.Schema` +instance as argument when constructing a reader. When the schema is declared, +conversion rules implemented by the package are applied to ensure that rows +read by the application match the desired format (see **Evolving Parquet Schemas**). + +```go +schema := parquet.SchemaOf(new(RowType)) +reader := parquet.NewReader(file, schema) +... +``` + +### Inspecting Parquet Files: [parquet.File](https://pkg.go.dev/github.com/parquet-go/parquet-go#File) + +Sometimes, lower-level APIs can be useful to leverage the columnar layout of +parquet files. The `parquet.File` type is intended to provide such features to +Go applications, by exposing APIs to iterate over the various parts of a +parquet file. + +```go +f, err := parquet.OpenFile(file, size) +if err != nil { + ... +} + +for _, rowGroup := range f.RowGroups() { + for _, columnChunk := range rowGroup.ColumnChunks() { + ... + } +} +``` + +### Evolving Parquet Schemas: [parquet.Convert](https://pkg.go.dev/github.com/parquet-go/parquet-go#Convert) + +Parquet files embed all the metadata necessary to interpret their content, +including a description of the schema of the tables represented by the rows and +columns they contain. + +Parquet files are also immutable; once written, there is not mechanism for +_updating_ a file. If their contents need to be changed, rows must be read, +modified, and written to a new file. + +Because applications evolve, the schema written to parquet files also tend to +evolve over time. Those requirements creating challenges when applications need +to operate on parquet files with heterogenous schemas: algorithms that expect +new columns to exist may have issues dealing with rows that come from files with +mismatching schema versions. + +To help build applications that can handle evolving schemas, `parquet-go/parquet-go` +implements conversion rules that create views of row groups to translate between +schema versions. + +The `parquet.Convert` function is the low-level routine constructing conversion +rules from a source to a target schema. The function is used to build converted +views of `parquet.RowReader` or `parquet.RowGroup`, for example: + +```go +type RowTypeV1 struct { ID int64; FirstName string } +type RowTypeV2 struct { ID int64; FirstName, LastName string } + +source := parquet.SchemaOf(RowTypeV1{}) +target := parquet.SchemaOf(RowTypeV2{}) + +conversion, err := parquet.Convert(target, source) +if err != nil { + ... +} + +targetRowGroup := parquet.ConvertRowGroup(sourceRowGroup, conversion) +... +``` + +Conversion rules are automatically applied by the `parquet.CopyRows` function +when the reader and writers passed to the function also implement the +`parquet.RowReaderWithSchema` and `parquet.RowWriterWithSchema` interfaces. +The copy determines whether the reader and writer schemas can be converted from +one to the other, and automatically applies the conversion rules to facilitate +the translation between schemas. + +At this time, conversion rules only supports adding or removing columns from +the schemas, there are no type conversions performed, nor ways to rename +columns, etc... More advanced conversion rules may be added in the future. + +### Sorting Row Groups: [parquet.GenericBuffer[T]](https://pkg.go.dev/github.com/parquet-go/parquet-go#Buffer) + +The `parquet.GenericWriter[T]` type is optimized for minimal memory usage, +keeping the order of rows unchanged and flushing pages as soon as they are filled. + +Parquet supports expressing columns by which rows are sorted through the +declaration of _sorting columns_ on row groups. Sorting row groups requires +buffering all rows before ordering and writing them to a parquet file. + +To help with those use cases, the `parquet-go/parquet-go` package exposes the +`parquet.GenericBuffer[T]` type which acts as a buffer of rows and implements +`sort.Interface` to allow applications to sort rows prior to writing them +to a file. + +The columns that rows are ordered by are configured when creating +`parquet.GenericBuffer[T]` instances using the `parquet.SortingColumns` function +to construct row group options configuring the buffer. The type of parquet +columns defines how values are compared, see [Parquet Logical Types](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md) +for details. + +When written to a file, the buffer is materialized into a single row group with +the declared sorting columns. After being written, buffers can be reused by +calling their `Reset` method. + +The following example shows how to use a `parquet.GenericBuffer[T]` to order rows +written to a parquet file: + +```go +type RowType struct { FirstName, LastName string } + +buffer := parquet.NewGenericBuffer[RowType]( + parquet.SortingRowGroupConfig( + parquet.SortingColumns( + parquet.Ascending("LastName"), + parquet.Ascending("FistName"), + ), + ), +) + +buffer.Write([]RowType{ + {FirstName: "Luke", LastName: "Skywalker"}, + {FirstName: "Han", LastName: "Solo"}, + {FirstName: "Anakin", LastName: "Skywalker"}, +}) + +sort.Sort(buffer) + +writer := parquet.NewGenericWriter[RowType](output) +_, err := parquet.CopyRows(writer, buffer.Rows()) +if err != nil { + ... +} +if err := writer.Close(); err != nil { + ... +} +``` + +### Merging Row Groups: [parquet.MergeRowGroups](https://pkg.go.dev/github.com/parquet-go/parquet-go#MergeRowGroups) + +Parquet files are often used as part of the underlying engine for data +processing or storage layers, in which cases merging multiple row groups +into one that contains more rows can be a useful operation to improve query +performance; for example, bloom filters in parquet files are stored for each +row group, the larger the row group, the fewer filters need to be stored and +the more effective they become. + +The `parquet-go/parquet-go` package supports creating merged views of row groups, +where the view contains all the rows of the merged groups, maintaining the order +defined by the sorting columns of the groups. + +There are a few constraints when merging row groups: + +- The sorting columns of all the row groups must be the same, or the merge + operation must be explicitly configured a set of sorting columns which are + a prefix of the sorting columns of all merged row groups. + +- The schemas of row groups must all be equal, or the merge operation must + be explicitly configured with a schema that all row groups can be converted + to, in which case the limitations of schema conversions apply. + +Once a merged view is created, it may be written to a new parquet file or buffer +in order to create a larger row group: + +```go +merge, err := parquet.MergeRowGroups(rowGroups) +if err != nil { + ... +} + +writer := parquet.NewGenericWriter[RowType](output) +_, err := parquet.CopyRows(writer, merge.Rows()) +if err != nil { + ... +} +if err := writer.Close(); err != nil { + ... +} +``` + +### Using Bloom Filters: [parquet.BloomFilter](https://pkg.go.dev/github.com/parquet-go/parquet-go#BloomFilter) + +Parquet files can embed bloom filters to help improve the performance of point +lookups in the files. The format of parquet bloom filters is documented in +the parquet specification: [Parquet Bloom Filter](https://github.com/apache/parquet-format/blob/master/BloomFilter.md) + +By default, no bloom filters are created in parquet files, but applications can +configure the list of columns to create filters for using the `parquet.BloomFilters` +option when instantiating writers; for example: + +```go +type RowType struct { + FirstName string `parquet:"first_name"` + LastName string `parquet:"last_name"` +} + +const filterBitsPerValue = 10 +writer := parquet.NewGenericWriter[RowType](output, + parquet.BloomFilters( + // Configures the write to generate split-block bloom filters for the + // "first_name" and "last_name" columns of the parquet schema of rows + // witten by the application. + parquet.SplitBlockFilter(filterBitsPerValue, "first_name"), + parquet.SplitBlockFilter(filterBitsPerValue, "last_name"), + ), +) +... +``` + +Generating bloom filters requires to know how many values exist in a column +chunk in order to properly size the filter, which requires buffering all the +values written to the column in memory. Because of it, the memory footprint +of `parquet.GenericWriter[T]` increases linearly with the number of columns +that the writer needs to generate filters for. This extra cost is optimized +away when rows are copied from a `parquet.GenericBuffer[T]` to a writer, since +in this case the number of values per column in known since the buffer already +holds all the values in memory. + +When reading parquet files, column chunks expose the generated bloom filters +with the `parquet.ColumnChunk.BloomFilter` method, returning a +`parquet.BloomFilter` instance if a filter was available, or `nil` when there +were no filters. + +Using bloom filters in parquet files is useful when performing point-lookups in +parquet files; searching for column rows matching a given value. Programs can +quickly eliminate column chunks that they know does not contain the value they +search for by checking the filter first, which is often multiple orders of +magnitude faster than scanning the column. + +The following code snippet hilights how filters are typically used: + +```go +var candidateChunks []parquet.ColumnChunk + +for _, rowGroup := range file.RowGroups() { + columnChunk := rowGroup.ColumnChunks()[columnIndex] + bloomFilter := columnChunk.BloomFilter() + + if bloomFilter != nil { + if ok, err := bloomFilter.Check(value); err != nil { + ... + } else if !ok { + // Bloom filters may return false positives, but never return false + // negatives, we know this column chunk does not contain the value. + continue + } + } + + candidateChunks = append(candidateChunks, columnChunk) +} +``` + +## Optimizations + +The following sections describe common optimization techniques supported by the +library. + +### Optimizing Reads + +Lower level APIs used to read parquet files offer more efficient ways to access +column values. Consecutive sequences of values are grouped into pages which are +represented by the `parquet.Page` interface. + +A column chunk may contain multiple pages, each holding a section of the column +values. Applications can retrieve the column values either by reading them into +buffers of `parquet.Value`, or type asserting the pages to read arrays of +primitive Go values. The following example demonstrates how to use both +mechanisms to read column values: + +```go +pages := column.Pages() +defer func() { + checkErr(pages.Close()) +}() + +for { + p, err := pages.ReadPage() + if err != nil { + ... // io.EOF when there are no more pages + } + + switch page := p.Values().(type) { + case parquet.Int32Reader: + values := make([]int32, page.NumValues()) + _, err := page.ReadInt32s(values) + ... + case parquet.Int64Reader: + values := make([]int64, page.NumValues()) + _, err := page.ReadInt64s(values) + ... + default: + values := make([]parquet.Value, page.NumValues()) + _, err := page.ReadValues(values) + ... + } +} +``` + +Reading arrays of typed values is often preferable when performing aggregations +on the values as this model offers a more compact representation of the values +in memory, and pairs well with the use of optimizations like SIMD vectorization. + +### Optimizing Writes + +Applications that deal with columnar storage are sometimes designed to work with +columnar data throughout the abstraction layers; it then becomes possible to +write columns of values directly instead of reconstructing rows from the column +values. The package offers two main mechanisms to satisfy those use cases: + +#### A. Writing Columns of Typed Arrays + +The first solution assumes that the program works with in-memory arrays of typed +values, for example slices of primitive Go types like `[]float32`; this would be +the case if the application is built on top of a framework like +[Apache Arrow](https://pkg.go.dev/github.com/apache/arrow/go/arrow). + +`parquet.GenericBuffer[T]` is an implementation of the `parquet.RowGroup` +interface which maintains in-memory buffers of column values. Rows can be +written by either boxing primitive values into arrays of `parquet.Value`, +or type asserting the columns to a access specialized versions of the write +methods accepting arrays of Go primitive types. + +When using either of these models, the application is responsible for ensuring +that the same number of rows are written to each column or the resulting parquet +file will be malformed. + +The following examples demonstrate how to use these two models to write columns +of Go values: + +```go +type RowType struct { FirstName, LastName string } + +func writeColumns(buffer *parquet.GenericBuffer[RowType], firstNames []string) error { + values := make([]parquet.Value, len(firstNames)) + for i := range firstNames { + values[i] = parquet.ValueOf(firstNames[i]) + } + _, err := buffer.ColumnBuffers()[0].WriteValues(values) + return err +} +``` + +```go +type RowType struct { ID int64; Value float32 } + +func writeColumns(buffer *parquet.GenericBuffer[RowType], ids []int64, values []float32) error { + if len(ids) != len(values) { + return fmt.Errorf("number of ids and values mismatch: ids=%d values=%d", len(ids), len(values)) + } + columns := buffer.ColumnBuffers() + if err := columns[0].(parquet.Int64Writer).WriteInt64s(ids); err != nil { + return err + } + if err := columns[1].(parquet.FloatWriter).WriteFloats(values); err != nil { + return err + } + return nil +} +``` + +The latter is more efficient as it does not require boxing the input into an +intermediary array of `parquet.Value`. However, it may not always be the right +model depending on the situation, sometimes the generic abstraction can be a +more expressive model. + +#### B. Implementing parquet.RowGroup + +Programs that need full control over the construction of row groups can choose +to provide their own implementation of the `parquet.RowGroup` interface, which +includes defining implementations of `parquet.ColumnChunk` and `parquet.Page` +to expose column values of the row group. + +This model can be preferable when the underlying storage or in-memory +representation of the data needs to be optimized further than what can be +achieved by using an intermediary buffering layer with `parquet.GenericBuffer[T]`. + +See [parquet.RowGroup](https://pkg.go.dev/github.com/parquet-go/parquet-go#RowGroup) +for the full interface documentation. + +#### C. Using on-disk page buffers + +When generating parquet files, the writer needs to buffer all pages before it +can create the row group. This may require significant amounts of memory as the +entire file content must be buffered prior to generating it. In some cases, the +files might even be larger than the amount of memory available to the program. + +The `parquet.GenericWriter[T]` can be configured to use disk storage instead as +a scratch buffer when generating files, by configuring a different page buffer +pool using the `parquet.ColumnPageBuffers` option and `parquet.PageBufferPool` +interface. + +The `parquet-go/parquet-go` package provides an implementation of the interface +which uses temporary files to store pages while a file is generated, allowing +programs to use local storage as swap space to hold pages and keep memory +utilization to a minimum. The following example demonstrates how to configure +a parquet writer to use on-disk page buffers: + +```go +type RowType struct { ... } + +writer := parquet.NewGenericWriter[RowType](output, + parquet.ColumnPageBuffers( + parquet.NewFileBufferPool("", "buffers.*"), + ), +) +``` + +When a row group is complete, pages buffered to disk need to be copied back to +the output file. This results in doubling I/O operations and storage space +requirements (the system needs to have enough free disk space to hold two copies +of the file). The resulting write amplification can often be optimized away by +the kernel if the file system supports copy-on-write of disk pages since copies +between `os.File` instances are optimized using `copy_file_range(2)` (on linux). + +See [parquet.PageBufferPool](https://pkg.go.dev/github.com/parquet-go/parquet-go#PageBufferPool) +for the full interface documentation. + +## Maintenance + +While initial design and development occurred at Twilio Segment, the project is now maintained by the open source community. We welcome external contributors. +to participate in the form of discussions or code changes. Please review to the +[Contribution](./CONTRIBUTING.md) guidelines as well as the [Code of Conduct](./CODE_OF_CONDUCT.md) +before submitting contributions. + +### Continuous Integration + +The project uses [Github Actions](https://github.com/parquet-go/parquet-go/actions) for CI. + +### Debugging + +The package has debugging capabilities built in which can be turned on using the +`PARQUETGODEBUG` environment variable. The value follows a model similar to +`GODEBUG`, it must be formatted as a comma-separated list of `key=value` pairs. + +The following debug flag are currently supported: + +- `tracebuf=1` turns on tracing of internal buffers, which validates that + reference counters are set to zero when buffers are reclaimed by the garbage + collector. When the package detects that a buffer was leaked, it logs an error + message along with the stack trace captured when the buffer was last used. diff --git a/vendor/github.com/parquet-go/parquet-go/allocator.go b/vendor/github.com/parquet-go/parquet-go/allocator.go new file mode 100644 index 0000000000..d1500643b3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/allocator.go @@ -0,0 +1,64 @@ +package parquet + +import ( + "unsafe" + + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type allocator struct{ buffer []byte } + +func (a *allocator) makeBytes(n int) []byte { + if free := cap(a.buffer) - len(a.buffer); free < n { + newCap := 2 * cap(a.buffer) + if newCap == 0 { + newCap = 4096 + } + for newCap < n { + newCap *= 2 + } + a.buffer = make([]byte, 0, newCap) + } + + i := len(a.buffer) + j := len(a.buffer) + n + a.buffer = a.buffer[:j] + return a.buffer[i:j:j] +} + +func (a *allocator) copyBytes(v []byte) []byte { + b := a.makeBytes(len(v)) + copy(b, v) + return b +} + +func (a *allocator) copyString(v string) string { + b := a.makeBytes(len(v)) + copy(b, v) + return unsafecast.String(b) +} + +func (a *allocator) reset() { + a.buffer = a.buffer[:0] +} + +// rowAllocator is a memory allocator used to make a copy of rows referencing +// memory buffers that parquet-go does not have ownership of. +// +// This type is used in the implementation of various readers and writers that +// need to capture rows passed to the ReadRows/WriteRows methods. Copies to a +// local buffer is necessary in those cases to repect the reader/writer +// contracts that do not allow the implementations to retain the rows they +// are passed as arguments. +// +// See: RowBuffer, NewRowGroupRowReader, NewColumnChunkRowReader +type rowAllocator struct{ allocator } + +func (a *rowAllocator) capture(row Row) { + for i, v := range row { + switch v.Kind() { + case ByteArray, FixedLenByteArray: + row[i].ptr = unsafe.SliceData(a.copyBytes(v.byteArray())) + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/array.go b/vendor/github.com/parquet-go/parquet-go/array.go new file mode 100644 index 0000000000..774e6f85ff --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/array.go @@ -0,0 +1,50 @@ +package parquet + +import ( + "unsafe" + + "github.com/parquet-go/parquet-go/sparse" +) + +func makeArrayValue(values []Value, offset uintptr) sparse.Array { + ptr := sliceData(values) + return sparse.UnsafeArray(unsafe.Add(ptr, offset), len(values), unsafe.Sizeof(Value{})) +} + +func makeArrayString(values []string) sparse.Array { + str := "" + ptr := sliceData(values) + return sparse.UnsafeArray(ptr, len(values), unsafe.Sizeof(str)) +} + +func makeArrayBE128(values []*[16]byte) sparse.Array { + ptr := sliceData(values) + return sparse.UnsafeArray(ptr, len(values), unsafe.Sizeof((*[16]byte)(nil))) +} + +func makeArray(base unsafe.Pointer, length int, offset uintptr) sparse.Array { + return sparse.UnsafeArray(base, length, offset) +} + +func makeArrayOf[T any](s []T) sparse.Array { + var model T + return makeArray(sliceData(s), len(s), unsafe.Sizeof(model)) +} + +func makeSlice[T any](a sparse.Array) []T { + return slice[T](a.Index(0), a.Len()) +} + +func slice[T any](p unsafe.Pointer, n int) []T { + return unsafe.Slice((*T)(p), n) +} + +func sliceData[T any](s []T) unsafe.Pointer { + return unsafe.Pointer(unsafe.SliceData(s)) +} + +type sliceHeader struct { + base unsafe.Pointer + len int + cap int +} diff --git a/vendor/github.com/parquet-go/parquet-go/bitmap.go b/vendor/github.com/parquet-go/parquet-go/bitmap.go new file mode 100644 index 0000000000..108cc8b4e3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bitmap.go @@ -0,0 +1,43 @@ +package parquet + +import "sync" + +type bitmap struct { + bits []uint64 +} + +func (m *bitmap) reset(size int) { + size = (size + 63) / 64 + if cap(m.bits) < size { + m.bits = make([]uint64, size, 2*size) + } else { + m.bits = m.bits[:size] + m.clear() + } +} + +func (m *bitmap) clear() { + for i := range m.bits { + m.bits[i] = 0 + } +} + +var ( + bitmapPool sync.Pool // *bitmap +) + +func acquireBitmap(n int) *bitmap { + b, _ := bitmapPool.Get().(*bitmap) + if b == nil { + b = &bitmap{bits: make([]uint64, n, 2*n)} + } else { + b.reset(n) + } + return b +} + +func releaseBitmap(b *bitmap) { + if b != nil { + bitmapPool.Put(b) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom.go b/vendor/github.com/parquet-go/parquet-go/bloom.go new file mode 100644 index 0000000000..911de08212 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom.go @@ -0,0 +1,280 @@ +package parquet + +import ( + "io" + + "github.com/parquet-go/parquet-go/bloom" + "github.com/parquet-go/parquet-go/bloom/xxhash" + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// BloomFilter is an interface allowing applications to test whether a key +// exists in a bloom filter. +type BloomFilter interface { + // Implement the io.ReaderAt interface as a mechanism to allow reading the + // raw bits of the filter. + io.ReaderAt + + // Returns the size of the bloom filter (in bytes). + Size() int64 + + // Tests whether the given value is present in the filter. + // + // A non-nil error may be returned if reading the filter failed. This may + // happen if the filter was lazily loaded from a storage medium during the + // call to Check for example. Applications that can guarantee that the + // filter was in memory at the time Check was called can safely ignore the + // error, which would always be nil in this case. + Check(value Value) (bool, error) +} + +type errorBloomFilter struct{ err error } + +func (f *errorBloomFilter) Size() int64 { return 0 } +func (f *errorBloomFilter) ReadAt([]byte, int64) (int, error) { return 0, f.err } +func (f *errorBloomFilter) Check(Value) (bool, error) { return false, f.err } + +type FileBloomFilter struct { + io.SectionReader + hash bloom.Hash + check func(io.ReaderAt, int64, uint64) (bool, error) +} + +func (f *FileBloomFilter) Check(v Value) (bool, error) { + return f.check(&f.SectionReader, f.Size(), v.hash(f.hash)) +} + +func (v Value) hash(h bloom.Hash) uint64 { + switch v.Kind() { + case Boolean: + return h.Sum64Uint8(v.byte()) + case Int32, Float: + return h.Sum64Uint32(v.uint32()) + case Int64, Double: + return h.Sum64Uint64(v.uint64()) + default: // Int96, ByteArray, FixedLenByteArray, or null + return h.Sum64(v.byteArray()) + } +} + +func newBloomFilter(file io.ReaderAt, offset int64, header *format.BloomFilterHeader) *FileBloomFilter { + if header.Algorithm.Block != nil { + if header.Hash.XxHash != nil { + if header.Compression.Uncompressed != nil { + return &FileBloomFilter{ + SectionReader: *io.NewSectionReader(file, offset, int64(header.NumBytes)), + hash: bloom.XXH64{}, + check: bloom.CheckSplitBlock, + } + } + } + } + return nil +} + +// The BloomFilterColumn interface is a declarative representation of bloom filters +// used when configuring filters on a parquet writer. +type BloomFilterColumn interface { + // Returns the path of the column that the filter applies to. + Path() []string + + // Returns the hashing algorithm used when inserting values into a bloom + // filter. + Hash() bloom.Hash + + // Returns an encoding which can be used to write columns of values to the + // filter. + Encoding() encoding.Encoding + + // Returns the size of the filter needed to encode values in the filter, + // assuming each value will be encoded with the given number of bits. + Size(numValues int64) int +} + +// SplitBlockFilter constructs a split block bloom filter object for the column +// at the given path, with the given bitsPerValue. +// +// If you are unsure what number of bitsPerValue to use, 10 is a reasonable +// tradeoff between size and error rate for common datasets. +// +// For more information on the tradeoff between size and error rate, consult +// this website: https://hur.st/bloomfilter/?n=4000&p=0.1&m=&k=1 +func SplitBlockFilter(bitsPerValue uint, path ...string) BloomFilterColumn { + return splitBlockFilter{ + bitsPerValue: bitsPerValue, + path: path, + } +} + +type splitBlockFilter struct { + bitsPerValue uint + path []string +} + +func (f splitBlockFilter) Path() []string { return f.path } +func (f splitBlockFilter) Hash() bloom.Hash { return bloom.XXH64{} } +func (f splitBlockFilter) Encoding() encoding.Encoding { return splitBlockEncoding{} } + +func (f splitBlockFilter) Size(numValues int64) int { + return bloom.BlockSize * bloom.NumSplitBlocksOf(numValues, f.bitsPerValue) +} + +// Creates a header from the given bloom filter. +// +// For now there is only one type of filter supported, but we provide this +// function to suggest a model for extending the implementation if new filters +// are added to the parquet specs. +func bloomFilterHeader(filter BloomFilterColumn) (header format.BloomFilterHeader) { + switch filter.(type) { + case splitBlockFilter: + header.Algorithm.Block = &format.SplitBlockAlgorithm{} + } + switch filter.Hash().(type) { + case bloom.XXH64: + header.Hash.XxHash = &format.XxHash{} + } + header.Compression.Uncompressed = &format.BloomFilterUncompressed{} + return header +} + +func searchBloomFilterColumn(filters []BloomFilterColumn, path columnPath) BloomFilterColumn { + for _, f := range filters { + if path.equal(f.Path()) { + return f + } + } + return nil +} + +const ( + // Size of the stack buffer used to perform bulk operations on bloom filters. + // + // This value was determined as being a good default empirically, + // 128 x uint64 makes a 1KiB buffer which amortizes the cost of calling + // methods of bloom filters while not causing too much stack growth either. + filterEncodeBufferSize = 128 +) + +type splitBlockEncoding struct { + encoding.NotSupported +} + +func (splitBlockEncoding) EncodeBoolean(dst []byte, src []byte) ([]byte, error) { + splitBlockEncodeUint8(bloom.MakeSplitBlockFilter(dst), src) + return dst, nil +} + +func (splitBlockEncoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + splitBlockEncodeUint32(bloom.MakeSplitBlockFilter(dst), unsafecast.Slice[uint32](src)) + return dst, nil +} + +func (splitBlockEncoding) EncodeInt64(dst []byte, src []int64) ([]byte, error) { + splitBlockEncodeUint64(bloom.MakeSplitBlockFilter(dst), unsafecast.Slice[uint64](src)) + return dst, nil +} + +func (e splitBlockEncoding) EncodeInt96(dst []byte, src []deprecated.Int96) ([]byte, error) { + splitBlockEncodeFixedLenByteArray(bloom.MakeSplitBlockFilter(dst), unsafecastInt96ToBytes(src), 12) + return dst, nil +} + +func (splitBlockEncoding) EncodeFloat(dst []byte, src []float32) ([]byte, error) { + splitBlockEncodeUint32(bloom.MakeSplitBlockFilter(dst), unsafecast.Slice[uint32](src)) + return dst, nil +} + +func (splitBlockEncoding) EncodeDouble(dst []byte, src []float64) ([]byte, error) { + splitBlockEncodeUint64(bloom.MakeSplitBlockFilter(dst), unsafecast.Slice[uint64](src)) + return dst, nil +} + +func (splitBlockEncoding) EncodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, error) { + filter := bloom.MakeSplitBlockFilter(dst) + buffer := make([]uint64, 0, filterEncodeBufferSize) + baseOffset := offsets[0] + + for _, endOffset := range offsets[1:] { + value := src[baseOffset:endOffset:endOffset] + baseOffset = endOffset + + if len(buffer) == cap(buffer) { + filter.InsertBulk(buffer) + buffer = buffer[:0] + } + + buffer = append(buffer, xxhash.Sum64(value)) + } + + filter.InsertBulk(buffer) + return dst, nil +} + +func (splitBlockEncoding) EncodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + filter := bloom.MakeSplitBlockFilter(dst) + if size == 16 { + splitBlockEncodeUint128(filter, unsafecast.Slice[[16]byte](src)) + } else { + splitBlockEncodeFixedLenByteArray(filter, src, size) + } + return dst, nil +} + +func splitBlockEncodeFixedLenByteArray(filter bloom.SplitBlockFilter, data []byte, size int) { + buffer := make([]uint64, 0, filterEncodeBufferSize) + + for i, j := 0, size; j <= len(data); { + if len(buffer) == cap(buffer) { + filter.InsertBulk(buffer) + buffer = buffer[:0] + } + buffer = append(buffer, xxhash.Sum64(data[i:j])) + i += size + j += size + } + + filter.InsertBulk(buffer) +} + +func splitBlockEncodeUint8(filter bloom.SplitBlockFilter, values []uint8) { + buffer := make([]uint64, filterEncodeBufferSize) + + for i := 0; i < len(values); { + n := xxhash.MultiSum64Uint8(buffer, values[i:]) + filter.InsertBulk(buffer[:n]) + i += n + } +} + +func splitBlockEncodeUint32(filter bloom.SplitBlockFilter, values []uint32) { + buffer := make([]uint64, filterEncodeBufferSize) + + for i := 0; i < len(values); { + n := xxhash.MultiSum64Uint32(buffer, values[i:]) + filter.InsertBulk(buffer[:n]) + i += n + } +} + +func splitBlockEncodeUint64(filter bloom.SplitBlockFilter, values []uint64) { + buffer := make([]uint64, filterEncodeBufferSize) + + for i := 0; i < len(values); { + n := xxhash.MultiSum64Uint64(buffer, values[i:]) + filter.InsertBulk(buffer[:n]) + i += n + } +} + +func splitBlockEncodeUint128(filter bloom.SplitBlockFilter, values [][16]byte) { + buffer := make([]uint64, filterEncodeBufferSize) + + for i := 0; i < len(values); { + n := xxhash.MultiSum64Uint128(buffer, values[i:]) + filter.InsertBulk(buffer[:n]) + i += n + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/block.go b/vendor/github.com/parquet-go/parquet-go/bloom/block.go new file mode 100644 index 0000000000..193dec721d --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/block.go @@ -0,0 +1,28 @@ +package bloom + +import "unsafe" + +// Word represents 32 bits words of bloom filter blocks. +type Word uint32 + +// Block represents bloom filter blocks which contain eight 32 bits words. +type Block [8]Word + +// Bytes returns b as a byte slice. +func (b *Block) Bytes() []byte { + return unsafe.Slice((*byte)(unsafe.Pointer(b)), BlockSize) +} + +const ( + // BlockSize is the size of bloom filter blocks in bytes. + BlockSize = 32 + + salt0 = 0x47b6137b + salt1 = 0x44974d91 + salt2 = 0x8824ad5b + salt3 = 0xa2b7289d + salt4 = 0x705495c7 + salt5 = 0x2df1424b + salt6 = 0x9efc4947 + salt7 = 0x5c6bfb31 +) diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.go b/vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.go new file mode 100644 index 0000000000..5edf9d5d90 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.go @@ -0,0 +1,39 @@ +//go:build !purego + +package bloom + +import "golang.org/x/sys/cpu" + +// The functions in this file are SIMD-optimized versions of the functions +// declared in block_optimized.go for x86 targets. +// +// The optimization yields measurable improvements over the pure Go versions: +// +// goos: darwin +// goarch: amd64 +// pkg: github.com/parquet-go/parquet-go/bloom +// cpu: Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz +// +// name old time/op new time/op delta +// BlockInsert 11.6ns ยฑ 4% 2.0ns ยฑ 3% -82.37% (p=0.000 n=8+8) +// BlockCheck 12.6ns ยฑ28% 2.1ns ยฑ 4% -83.12% (p=0.000 n=10+8) +// +// name old speed new speed delta +// BlockInsert 2.73GB/s ยฑ13% 15.70GB/s ยฑ 3% +475.96% (p=0.000 n=9+8) +// BlockCheck 2.59GB/s ยฑ23% 15.06GB/s ยฑ 4% +482.25% (p=0.000 n=10+8) +// +// Note that the numbers above are a comparison to the routines implemented in +// block_optimized.go; the delta comparing to functions in block_default.go is +// significantly larger but not very interesting since those functions have no +// practical use cases. +var hasAVX2 = cpu.X86.HasAVX2 + +//go:noescape +func blockInsert(b *Block, x uint32) + +//go:noescape +func blockCheck(b *Block, x uint32) bool + +func (b *Block) Insert(x uint32) { blockInsert(b, x) } + +func (b *Block) Check(x uint32) bool { return blockCheck(b, x) } diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.s b/vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.s new file mode 100644 index 0000000000..713aea2cd7 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/block_amd64.s @@ -0,0 +1,129 @@ +//go:build !purego + +#include "textflag.h" + +#define salt0 0x47b6137b +#define salt1 0x44974d91 +#define salt2 0x8824ad5b +#define salt3 0xa2b7289d +#define salt4 0x705495c7 +#define salt5 0x2df1424b +#define salt6 0x9efc4947 +#define salt7 0x5c6bfb31 + +DATA ones+0(SB)/4, $1 +DATA ones+4(SB)/4, $1 +DATA ones+8(SB)/4, $1 +DATA ones+12(SB)/4, $1 +DATA ones+16(SB)/4, $1 +DATA ones+20(SB)/4, $1 +DATA ones+24(SB)/4, $1 +DATA ones+28(SB)/4, $1 +GLOBL ones(SB), RODATA|NOPTR, $32 + +DATA salt+0(SB)/4, $salt0 +DATA salt+4(SB)/4, $salt1 +DATA salt+8(SB)/4, $salt2 +DATA salt+12(SB)/4, $salt3 +DATA salt+16(SB)/4, $salt4 +DATA salt+20(SB)/4, $salt5 +DATA salt+24(SB)/4, $salt6 +DATA salt+28(SB)/4, $salt7 +GLOBL salt(SB), RODATA|NOPTR, $32 + +// This initial block is a SIMD implementation of the mask function declared in +// block_default.go and block_optimized.go. For each of the 8 x 32 bits words of +// the bloom filter block, the operation performed is: +// +// block[i] = 1 << ((x * salt[i]) >> 27) +// +// Arguments +// --------- +// +// * src is a memory location where the value to use when computing the mask is +// located. The memory location is not modified. +// +// * tmp is a YMM register used as scratch space to hold intermediary results in +// the algorithm. +// +// * dst is a YMM register where the final mask is written. +// +#define generateMask(src, tmp, dst) \ + VMOVDQA ones(SB), dst \ + VPBROADCASTD src, tmp \ + VPMULLD salt(SB), tmp, tmp \ + VPSRLD $27, tmp, tmp \ + VPSLLVD tmp, dst, dst + +#define insert(salt, src, dst) \ + MOVL src, CX \ + IMULL salt, CX \ + SHRL $27, CX \ + MOVL $1, DX \ + SHLL CX, DX \ + ORL DX, dst + +#define check(salt, b, x) \ + MOVL b, CX \ + MOVL x, DX \ + IMULL salt, DX \ + SHRL $27, DX \ + BTL DX, CX \ + JAE notfound + +// func blockInsert(b *Block, x uint32) +TEXT ยทblockInsert(SB), NOSPLIT, $0-16 + MOVQ b+0(FP), AX + CMPB ยทhasAVX2(SB), $0 + JE fallback +avx2: + generateMask(x+8(FP), Y1, Y0) + // Set all 1 bits of the mask in the bloom filter block. + VPOR (AX), Y0, Y0 + VMOVDQU Y0, (AX) + VZEROUPPER + RET +fallback: + MOVL x+8(FP), BX + insert($salt0, BX, 0(AX)) + insert($salt1, BX, 4(AX)) + insert($salt2, BX, 8(AX)) + insert($salt3, BX, 12(AX)) + insert($salt4, BX, 16(AX)) + insert($salt5, BX, 20(AX)) + insert($salt6, BX, 24(AX)) + insert($salt7, BX, 28(AX)) + RET + +// func blockCheck(b *Block, x uint32) bool +TEXT ยทblockCheck(SB), NOSPLIT, $0-17 + MOVQ b+0(FP), AX + CMPB ยทhasAVX2(SB), $0 + JE fallback +avx2: + generateMask(x+8(FP), Y1, Y0) + // Compare the 1 bits of the mask with the bloom filter block, then compare + // the result with the mask, expecting equality if the value `x` was present + // in the block. + VPAND (AX), Y0, Y1 // Y0 = block & mask + VPTEST Y0, Y1 // if (Y0 & ^Y1) != 0 { CF = 1 } + SETCS ret+16(FP) // return CF == 1 + VZEROUPPER + RET +fallback: + MOVL x+8(FP), BX + check($salt0, 0(AX), BX) + check($salt1, 4(AX), BX) + check($salt2, 8(AX), BX) + check($salt3, 12(AX), BX) + check($salt4, 16(AX), BX) + check($salt5, 20(AX), BX) + check($salt6, 24(AX), BX) + check($salt7, 28(AX), BX) + MOVB $1, CX + JMP done +notfound: + XORB CX, CX +done: + MOVB CX, ret+16(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/block_default.go b/vendor/github.com/parquet-go/parquet-go/bloom/block_default.go new file mode 100644 index 0000000000..016cb73b44 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/block_default.go @@ -0,0 +1,65 @@ +//go:build purego && parquet.bloom.no_unroll + +package bloom + +// This file contains direct translation of the algorithms described in the +// parquet bloom filter spec: +// https://github.com/apache/parquet-format/blob/master/BloomFilter.md +// +// There are no practical reasons to eable the parquet.bloom.no_unroll build +// tag, the code is left here as a reference to ensure that the optimized +// implementations of block operations behave the same as the functions in this +// file. + +var salt = [8]uint32{ + 0: salt0, + 1: salt1, + 2: salt2, + 3: salt3, + 4: salt4, + 5: salt5, + 6: salt6, + 7: salt7, +} + +func (w *Word) set(i uint) { + *w |= Word(1 << i) +} + +func (w Word) has(i uint) bool { + return ((w >> Word(i)) & 1) != 0 +} + +func mask(x uint32) Block { + var b Block + for i := uint(0); i < 8; i++ { + y := x * salt[i] + b[i].set(uint(y) >> 27) + } + return b +} + +func (b *Block) Insert(x uint32) { + masked := mask(x) + for i := uint(0); i < 8; i++ { + for j := uint(0); j < 32; j++ { + if masked[i].has(j) { + b[i].set(j) + } + } + } +} + +func (b *Block) Check(x uint32) bool { + masked := mask(x) + for i := uint(0); i < 8; i++ { + for j := uint(0); j < 32; j++ { + if masked[i].has(j) { + if !b[i].has(j) { + return false + } + } + } + } + return true +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/block_optimized.go b/vendor/github.com/parquet-go/parquet-go/bloom/block_optimized.go new file mode 100644 index 0000000000..20a9768121 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/block_optimized.go @@ -0,0 +1,53 @@ +//go:build (!amd64 || purego) && !parquet.bloom.no_unroll + +package bloom + +// The functions in this file are optimized versions of the algorithms described +// in https://github.com/apache/parquet-format/blob/master/BloomFilter.md +// +// The functions are manual unrolling of the loops, which yield significant +// performance improvements: +// +// goos: darwin +// goarch: amd64 +// pkg: github.com/parquet-go/parquet-go/bloom +// cpu: Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz +// +// name old time/op new time/op delta +// BlockInsert 327ns ยฑ 1% 12ns ยฑ 4% -96.47% (p=0.000 n=9+8) +// BlockCheck 240ns ยฑ 4% 13ns ยฑ28% -94.75% (p=0.000 n=8+10) +// +// name old speed new speed delta +// BlockInsert 97.8MB/s ยฑ 1% 2725.0MB/s ยฑ13% +2686.59% (p=0.000 n=9+9) +// BlockCheck 133MB/s ยฑ 4% 2587MB/s ยฑ23% +1838.46% (p=0.000 n=8+10) +// +// The benchmarks measure throughput based on the byte size of a bloom filter +// block. + +func (b *Block) Insert(x uint32) { + b[0] |= 1 << ((x * salt0) >> 27) + b[1] |= 1 << ((x * salt1) >> 27) + b[2] |= 1 << ((x * salt2) >> 27) + b[3] |= 1 << ((x * salt3) >> 27) + b[4] |= 1 << ((x * salt4) >> 27) + b[5] |= 1 << ((x * salt5) >> 27) + b[6] |= 1 << ((x * salt6) >> 27) + b[7] |= 1 << ((x * salt7) >> 27) +} + +func (b *Block) Check(x uint32) bool { + return ((b[0] & (1 << ((x * salt0) >> 27))) != 0) && + ((b[1] & (1 << ((x * salt1) >> 27))) != 0) && + ((b[2] & (1 << ((x * salt2) >> 27))) != 0) && + ((b[3] & (1 << ((x * salt3) >> 27))) != 0) && + ((b[4] & (1 << ((x * salt4) >> 27))) != 0) && + ((b[5] & (1 << ((x * salt5) >> 27))) != 0) && + ((b[6] & (1 << ((x * salt6) >> 27))) != 0) && + ((b[7] & (1 << ((x * salt7) >> 27))) != 0) +} + +func (f SplitBlockFilter) insertBulk(x []uint64) { + for i := range x { + f.Insert(x[i]) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/bloom.go b/vendor/github.com/parquet-go/parquet-go/bloom/bloom.go new file mode 100644 index 0000000000..337fe0d7d8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/bloom.go @@ -0,0 +1,13 @@ +// Package bloom implements parquet bloom filters. +package bloom + +func fasthash1x64(value uint64, scale int32) uint64 { + return ((value >> 32) * uint64(scale)) >> 32 +} + +func fasthash4x64(dst, src *[4]uint64, scale int32) { + dst[0] = ((src[0] >> 32) * uint64(scale)) >> 32 + dst[1] = ((src[1] >> 32) * uint64(scale)) >> 32 + dst[2] = ((src[2] >> 32) * uint64(scale)) >> 32 + dst[3] = ((src[3] >> 32) * uint64(scale)) >> 32 +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/filter.go b/vendor/github.com/parquet-go/parquet-go/bloom/filter.go new file mode 100644 index 0000000000..11cc255a1c --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/filter.go @@ -0,0 +1,97 @@ +package bloom + +import ( + "io" + "sync" + + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// Filter is an interface representing read-only bloom filters where programs +// can probe for the possible presence of a hash key. +type Filter interface { + Check(uint64) bool +} + +// SplitBlockFilter is an in-memory implementation of the parquet bloom filters. +// +// This type is useful to construct bloom filters that are later serialized +// to a storage medium. +type SplitBlockFilter []Block + +// MakeSplitBlockFilter constructs a SplitBlockFilter value from the data byte +// slice. +func MakeSplitBlockFilter(data []byte) SplitBlockFilter { + return unsafecast.Slice[Block](data) +} + +// NumSplitBlocksOf returns the number of blocks in a filter intended to hold +// the given number of values and bits of filter per value. +// +// This function is useful to determine the number of blocks when creating bloom +// filters in memory, for example: +// +// f := make(bloom.SplitBlockFilter, bloom.NumSplitBlocksOf(n, 10)) +func NumSplitBlocksOf(numValues int64, bitsPerValue uint) int { + numBytes := ((uint(numValues) * bitsPerValue) + 7) / 8 + numBlocks := (numBytes + (BlockSize - 1)) / BlockSize + return int(numBlocks) +} + +// Reset clears the content of the filter f. +func (f SplitBlockFilter) Reset() { + for i := range f { + f[i] = Block{} + } +} + +// Block returns a pointer to the block that the given value hashes to in the +// bloom filter. +func (f SplitBlockFilter) Block(x uint64) *Block { return &f[fasthash1x64(x, int32(len(f)))] } + +// InsertBulk adds all values from x into f. +func (f SplitBlockFilter) InsertBulk(x []uint64) { filterInsertBulk(f, x) } + +// Insert adds x to f. +func (f SplitBlockFilter) Insert(x uint64) { filterInsert(f, x) } + +// Check tests whether x is in f. +func (f SplitBlockFilter) Check(x uint64) bool { return filterCheck(f, x) } + +// Bytes converts f to a byte slice. +// +// The returned slice shares the memory of f. The method is intended to be used +// to serialize the bloom filter to a storage medium. +func (f SplitBlockFilter) Bytes() []byte { + return unsafecast.Slice[byte](f) +} + +// CheckSplitBlock is similar to bloom.SplitBlockFilter.Check but reads the +// bloom filter of n bytes from r. +// +// The size n of the bloom filter is assumed to be a multiple of the block size. +func CheckSplitBlock(r io.ReaderAt, n int64, x uint64) (bool, error) { + block := acquireBlock() + defer releaseBlock(block) + offset := BlockSize * fasthash1x64(x, int32(n/BlockSize)) + _, err := r.ReadAt(block.Bytes(), int64(offset)) + return block.Check(uint32(x)), err +} + +var ( + blockPool sync.Pool +) + +func acquireBlock() *Block { + b, _ := blockPool.Get().(*Block) + if b == nil { + b = new(Block) + } + return b +} + +func releaseBlock(b *Block) { + if b != nil { + blockPool.Put(b) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.go b/vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.go new file mode 100644 index 0000000000..3649da51ca --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.go @@ -0,0 +1,33 @@ +//go:build !purego + +package bloom + +// This file contains the signatures for bloom filter algorithms implemented in +// filter_amd64.s. +// +// The assembly code provides significant speedups on filter inserts and checks, +// with the greatest gains seen on the bulk insert operation where the use of +// vectorized code yields great results. +// +// The following sections record the kind of performance improvements we were +// able to measure, comparing with performing the filter block lookups in Go +// and calling to the block insert and check routines: +// +// name old time/op new time/op delta +// FilterInsertBulk 45.1ns ยฑ 2% 17.8ns ยฑ 3% -60.41% (p=0.000 n=10+10) +// FilterInsert 3.48ns ยฑ 2% 2.55ns ยฑ 1% -26.86% (p=0.000 n=10+8) +// FilterCheck 3.64ns ยฑ 3% 2.66ns ยฑ 2% -26.82% (p=0.000 n=10+9) +// +// name old speed new speed delta +// FilterInsertBulk 11.4GB/s ยฑ 2% 28.7GB/s ยฑ 3% +152.61% (p=0.000 n=10+10) +// FilterInsert 9.19GB/s ยฑ 2% 12.56GB/s ยฑ 1% +36.71% (p=0.000 n=10+8) +// FilterCheck 8.80GB/s ยฑ 3% 12.03GB/s ยฑ 2% +36.61% (p=0.000 n=10+9) + +//go:noescape +func filterInsertBulk(f []Block, x []uint64) + +//go:noescape +func filterInsert(f []Block, x uint64) + +//go:noescape +func filterCheck(f []Block, x uint64) bool diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.s b/vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.s new file mode 100644 index 0000000000..cfa75e69fb --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/filter_amd64.s @@ -0,0 +1,214 @@ +//go:build !purego + +#include "textflag.h" + +#define salt0 0x47b6137b +#define salt1 0x44974d91 +#define salt2 0x8824ad5b +#define salt3 0xa2b7289d +#define salt4 0x705495c7 +#define salt5 0x2df1424b +#define salt6 0x9efc4947 +#define salt7 0x5c6bfb31 + +// See block_amd64.s for a description of this algorithm. +#define generateMask(src, dst) \ + VMOVDQA ones(SB), dst \ + VPMULLD salt(SB), src, src \ + VPSRLD $27, src, src \ + VPSLLVD src, dst, dst + +#define applyMask(src, dst) \ + VPOR dst, src, src \ + VMOVDQU src, dst + +#define fasthash1x64(scale, value) \ + SHRQ $32, value \ + IMULQ scale, value \ + SHRQ $32, value \ + SHLQ $5, value + +#define fasthash4x64(scale, value) \ + VPSRLQ $32, value, value \ + VPMULUDQ scale, value, value \ + VPSRLQ $32, value, value \ + VPSLLQ $5, value, value + +#define extract4x64(srcYMM, srcXMM, tmpXMM, r0, r1, r2, r3) \ + VEXTRACTI128 $1, srcYMM, tmpXMM \ + MOVQ srcXMM, r0 \ + VPEXTRQ $1, srcXMM, r1 \ + MOVQ tmpXMM, r2 \ + VPEXTRQ $1, tmpXMM, r3 + +#define insert(salt, src, dst) \ + MOVL src, CX \ + IMULL salt, CX \ + SHRL $27, CX \ + MOVL $1, DX \ + SHLL CX, DX \ + ORL DX, dst + +#define check(salt, b, x) \ + MOVL b, CX \ + MOVL x, DX \ + IMULL salt, DX \ + SHRL $27, DX \ + BTL DX, CX \ + JAE notfound + +// func filterInsertBulk(f []Block, x []uint64) +TEXT ยทfilterInsertBulk(SB), NOSPLIT, $0-48 + MOVQ f_base+0(FP), AX + MOVQ f_len+8(FP), CX + MOVQ x_base+24(FP), BX + MOVQ x_len+32(FP), DX + CMPB ยทhasAVX2(SB), $0 + JE fallback +avx2: + VPBROADCASTQ f_base+8(FP), Y0 + // Loop initialization, SI holds the current index in `x`, DI is the number + // of elements in `x` rounded down to the nearest multiple of 4. + XORQ SI, SI + MOVQ DX, DI + SHRQ $2, DI + SHLQ $2, DI +avx2loop4x64: + CMPQ SI, DI + JAE avx2loop1x64 + + // The masks and indexes for 4 input hashes are computed in each loop + // iteration. The hashes are loaded in Y1 so we can use vector instructions + // to compute all 4 indexes in parallel. The lower 32 bits of the hashes are + // also broadcasted in 4 YMM registers to compute the 4 masks that will then + // be applied to the filter. + VMOVDQU (BX)(SI*8), Y1 + VPBROADCASTD 0(BX)(SI*8), Y2 + VPBROADCASTD 8(BX)(SI*8), Y3 + VPBROADCASTD 16(BX)(SI*8), Y4 + VPBROADCASTD 24(BX)(SI*8), Y5 + + fasthash4x64(Y0, Y1) + generateMask(Y2, Y6) + generateMask(Y3, Y7) + generateMask(Y4, Y8) + generateMask(Y5, Y9) + + // The next block of instructions move indexes from the vector to general + // purpose registers in order to use them as offsets when applying the mask + // to the filter. + extract4x64(Y1, X1, X10, R8, R9, R10, R11) + + // Apply masks to the filter; this operation is sensitive to aliasing, when + // blocks overlap the, CPU has to serialize the reads and writes, which has + // a measurable impact on throughput. This would be frequent for small bloom + // filters which may have only a few blocks, the probability of seeing + // overlapping blocks on large filters should be small enough to make this + // a non-issue though. + applyMask(Y6, (AX)(R8*1)) + applyMask(Y7, (AX)(R9*1)) + applyMask(Y8, (AX)(R10*1)) + applyMask(Y9, (AX)(R11*1)) + + ADDQ $4, SI + JMP avx2loop4x64 +avx2loop1x64: + // Compute trailing elements in `x` if the length was not a multiple of 4. + // This is the same algorithm as the one in the loop4x64 section, working + // on a single mask/block pair at a time. + CMPQ SI, DX + JE avx2done + MOVQ (BX)(SI*8), R8 + VPBROADCASTD (BX)(SI*8), Y0 + fasthash1x64(CX, R8) + generateMask(Y0, Y1) + applyMask(Y1, (AX)(R8*1)) + INCQ SI + JMP avx2loop1x64 +avx2done: + VZEROUPPER + JMP done +fallback: + XORQ SI, SI + MOVQ DX, DI + MOVQ CX, R10 +loop: + CMPQ SI, DI + JE done + MOVLQZX (BX)(SI*8), R8 + MOVQ (BX)(SI*8), R9 + fasthash1x64(R10, R9) + insert($salt0, R8, 0(AX)(R9*1)) + insert($salt1, R8, 4(AX)(R9*1)) + insert($salt2, R8, 8(AX)(R9*1)) + insert($salt3, R8, 12(AX)(R9*1)) + insert($salt4, R8, 16(AX)(R9*1)) + insert($salt5, R8, 20(AX)(R9*1)) + insert($salt6, R8, 24(AX)(R9*1)) + insert($salt7, R8, 28(AX)(R9*1)) + INCQ SI + JMP loop +done: + RET + +// func filterInsert(f []Block, x uint64) +TEXT ยทfilterInsert(SB), NOSPLIT, $0-32 + MOVQ f_base+0(FP), AX + MOVQ f_len+8(FP), BX + MOVQ x+24(FP), CX + fasthash1x64(BX, CX) + CMPB ยทhasAVX2(SB), $0 + JE fallback +avx2: + VPBROADCASTD x+24(FP), Y1 + generateMask(Y1, Y0) + applyMask(Y0, (AX)(CX*1)) + VZEROUPPER + RET +fallback: + ADDQ CX, AX + MOVL x+24(FP), BX + insert($salt0, BX, 0(AX)) + insert($salt1, BX, 4(AX)) + insert($salt2, BX, 8(AX)) + insert($salt3, BX, 12(AX)) + insert($salt4, BX, 16(AX)) + insert($salt5, BX, 20(AX)) + insert($salt6, BX, 24(AX)) + insert($salt7, BX, 28(AX)) + RET + +// func filterCheck(f []Block, x uint64) bool +TEXT ยทfilterCheck(SB), NOSPLIT, $0-33 + MOVQ f_base+0(FP), AX + MOVQ f_len+8(FP), BX + MOVQ x+24(FP), CX + fasthash1x64(BX, CX) + CMPB ยทhasAVX2(SB), $0 + JE fallback +avx2: + VPBROADCASTD x+24(FP), Y1 + generateMask(Y1, Y0) + VPAND (AX)(CX*1), Y0, Y1 + VPTEST Y0, Y1 + SETCS ret+32(FP) + VZEROUPPER + RET +fallback: + ADDQ CX, AX + MOVL x+24(FP), BX + check($salt0, 0(AX), BX) + check($salt1, 4(AX), BX) + check($salt2, 8(AX), BX) + check($salt3, 12(AX), BX) + check($salt4, 16(AX), BX) + check($salt5, 20(AX), BX) + check($salt6, 24(AX), BX) + check($salt7, 28(AX), BX) + MOVB $1, CX + JMP done +notfound: + XORB CX, CX +done: + MOVB CX, ret+32(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/filter_default.go b/vendor/github.com/parquet-go/parquet-go/bloom/filter_default.go new file mode 100644 index 0000000000..38fadd5be8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/filter_default.go @@ -0,0 +1,17 @@ +//go:build purego || !amd64 + +package bloom + +func filterInsertBulk(f []Block, x []uint64) { + for i := range x { + filterInsert(f, x[i]) + } +} + +func filterInsert(f []Block, x uint64) { + f[fasthash1x64(x, int32(len(f)))].Insert(uint32(x)) +} + +func filterCheck(f []Block, x uint64) bool { + return f[fasthash1x64(x, int32(len(f)))].Check(uint32(x)) +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/hash.go b/vendor/github.com/parquet-go/parquet-go/bloom/hash.go new file mode 100644 index 0000000000..9a3b61a300 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/hash.go @@ -0,0 +1,77 @@ +package bloom + +import "github.com/parquet-go/parquet-go/bloom/xxhash" + +// Hash is an interface abstracting the hashing algorithm used in bloom filters. +// +// Hash instances must be safe to use concurrently from multiple goroutines. +type Hash interface { + // Returns the 64 bit hash of the value passed as argument. + Sum64(value []byte) uint64 + + // Compute hashes of individual values of primitive types. + Sum64Uint8(value uint8) uint64 + Sum64Uint16(value uint16) uint64 + Sum64Uint32(value uint32) uint64 + Sum64Uint64(value uint64) uint64 + Sum64Uint128(value [16]byte) uint64 + + // Compute hashes of the array of fixed size values passed as arguments, + // returning the number of hashes written to the destination buffer. + MultiSum64Uint8(dst []uint64, src []uint8) int + MultiSum64Uint16(dst []uint64, src []uint16) int + MultiSum64Uint32(dst []uint64, src []uint32) int + MultiSum64Uint64(dst []uint64, src []uint64) int + MultiSum64Uint128(dst []uint64, src [][16]byte) int +} + +// XXH64 is an implementation of the Hash interface using the XXH64 algorithm. +type XXH64 struct{} + +func (XXH64) Sum64(b []byte) uint64 { + return xxhash.Sum64(b) +} + +func (XXH64) Sum64Uint8(v uint8) uint64 { + return xxhash.Sum64Uint8(v) +} + +func (XXH64) Sum64Uint16(v uint16) uint64 { + return xxhash.Sum64Uint16(v) +} + +func (XXH64) Sum64Uint32(v uint32) uint64 { + return xxhash.Sum64Uint32(v) +} + +func (XXH64) Sum64Uint64(v uint64) uint64 { + return xxhash.Sum64Uint64(v) +} + +func (XXH64) Sum64Uint128(v [16]byte) uint64 { + return xxhash.Sum64Uint128(v) +} + +func (XXH64) MultiSum64Uint8(h []uint64, v []uint8) int { + return xxhash.MultiSum64Uint8(h, v) +} + +func (XXH64) MultiSum64Uint16(h []uint64, v []uint16) int { + return xxhash.MultiSum64Uint16(h, v) +} + +func (XXH64) MultiSum64Uint32(h []uint64, v []uint32) int { + return xxhash.MultiSum64Uint32(h, v) +} + +func (XXH64) MultiSum64Uint64(h []uint64, v []uint64) int { + return xxhash.MultiSum64Uint64(h, v) +} + +func (XXH64) MultiSum64Uint128(h []uint64, v [][16]byte) int { + return xxhash.MultiSum64Uint128(h, v) +} + +var ( + _ Hash = XXH64{} +) diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/LICENSE b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/LICENSE new file mode 100644 index 0000000000..80bef2ebc4 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/LICENSE @@ -0,0 +1,27 @@ +The following files in this directory were derived from the open-source +project at https://github.com/cespare/xxhash. A copy of the original +license is provided below. +------------------------------------------------------------------------ + +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint.go b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint.go new file mode 100644 index 0000000000..84423d0efb --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint.go @@ -0,0 +1,37 @@ +package xxhash + +func Sum64Uint8(v uint8) uint64 { + h := prime5 + 1 + h ^= uint64(v) * prime5 + return avalanche(rol11(h) * prime1) +} + +func Sum64Uint16(v uint16) uint64 { + h := prime5 + 2 + h ^= uint64(v&0xFF) * prime5 + h = rol11(h) * prime1 + h ^= uint64(v>>8) * prime5 + h = rol11(h) * prime1 + return avalanche(h) +} + +func Sum64Uint32(v uint32) uint64 { + h := prime5 + 4 + h ^= uint64(v) * prime1 + return avalanche(rol23(h)*prime2 + prime3) +} + +func Sum64Uint64(v uint64) uint64 { + h := prime5 + 8 + h ^= round(0, v) + return avalanche(rol27(h)*prime1 + prime4) +} + +func Sum64Uint128(v [16]byte) uint64 { + h := prime5 + 16 + h ^= round(0, u64(v[:8])) + h = rol27(h)*prime1 + prime4 + h ^= round(0, u64(v[8:])) + h = rol27(h)*prime1 + prime4 + return avalanche(h) +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.go b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.go new file mode 100644 index 0000000000..03a4d31b15 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.go @@ -0,0 +1,49 @@ +//go:build !purego + +package xxhash + +import "golang.org/x/sys/cpu" + +// This file contains the declaration of signatures for the multi hashing +// functions implemented in sum64uint_amd64.s, which provides vectorized +// versions of the algorithms written in sum64uint_purego.go. +// +// The use of SIMD optimization yields measurable throughput increases when +// computing multiple hash values in parallel compared to hashing values +// individually in loops: +// +// name old speed new speed delta +// MultiSum64Uint8/4KB 4.94GB/s ยฑ 2% 6.82GB/s ยฑ 5% +38.00% (p=0.000 n=10+10) +// MultiSum64Uint16/4KB 3.44GB/s ยฑ 2% 4.63GB/s ยฑ 4% +34.56% (p=0.000 n=10+10) +// MultiSum64Uint32/4KB 4.84GB/s ยฑ 2% 6.39GB/s ยฑ 4% +31.94% (p=0.000 n=10+10) +// MultiSum64Uint64/4KB 3.77GB/s ยฑ 2% 4.95GB/s ยฑ 2% +31.14% (p=0.000 n=9+10) +// MultiSum64Uint128/4KB 1.84GB/s ยฑ 2% 3.11GB/s ยฑ 4% +68.70% (p=0.000 n=9+10) +// +// name old hash/s new hash/s delta +// MultiSum64Uint8/4KB 617M ยฑ 2% 852M ยฑ 5% +38.00% (p=0.000 n=10+10) +// MultiSum64Uint16/4KB 431M ยฑ 2% 579M ยฑ 4% +34.56% (p=0.000 n=10+10) +// MultiSum64Uint32/4KB 605M ยฑ 2% 799M ยฑ 4% +31.94% (p=0.000 n=10+10) +// MultiSum64Uint64/4KB 471M ยฑ 2% 618M ยฑ 2% +31.14% (p=0.000 n=9+10) +// MultiSum64Uint128/4KB 231M ยฑ 2% 389M ยฑ 4% +68.70% (p=0.000 n=9+10) +// +// The benchmarks measure the throughput of hashes produced, as a rate of values +// and bytes. + +var hasAVX512 = cpu.X86.HasAVX512 && + cpu.X86.HasAVX512F && + cpu.X86.HasAVX512CD + +//go:noescape +func MultiSum64Uint8(h []uint64, v []uint8) int + +//go:noescape +func MultiSum64Uint16(h []uint64, v []uint16) int + +//go:noescape +func MultiSum64Uint32(h []uint64, v []uint32) int + +//go:noescape +func MultiSum64Uint64(h []uint64, v []uint64) int + +//go:noescape +func MultiSum64Uint128(h []uint64, v [][16]byte) int diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.s b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.s new file mode 100644 index 0000000000..9d420a9d91 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_amd64.s @@ -0,0 +1,755 @@ +//go:build !purego + +#include "textflag.h" + +/* +The algorithms in this file are assembly versions of the Go functions in the +sum64uint_default.go file. + +The implementations are mostly direct translations of the Go code to assembly, +leveraging SIMD instructions to process chunks of the input variables in +parallel at each loop iteration. To maximize utilization of the CPU capacity, +some of the functions unroll two steps of the vectorized loop per iteration, +which yields further throughput because the CPU is able to process some of the +instruction from the two steps in parallel due to having no data dependencies +between the inputs and outputs. + +The use of AVX-512 yields a significant increase in throughput on all the +algorithms, in most part thanks to the VPMULLQ instructions which compute +8 x 64 bits multiplication. There were no equivalent instruction in AVX2, which +required emulating vector multiplication with a combination of 32 bits multiply, +additions, shifts, and masks: the amount of instructions and data dependencies +resulted in the AVX2 code yielding equivalent performance characteristics for a +much higher complexity. + +The benchmark results below showcase the improvements that the AVX-512 code +yields on the XXH64 algorithms: + +name old speed new speed delta +MultiSum64Uint8/4KB 4.97GB/s ยฑ 0% 14.59GB/s ยฑ 1% +193.73% (p=0.000 n=10+10) +MultiSum64Uint16/4KB 3.55GB/s ยฑ 0% 9.46GB/s ยฑ 0% +166.20% (p=0.000 n=10+9) +MultiSum64Uint32/4KB 4.48GB/s ยฑ 0% 13.93GB/s ยฑ 1% +210.93% (p=0.000 n=10+10) +MultiSum64Uint64/4KB 3.57GB/s ยฑ 0% 11.12GB/s ยฑ 1% +211.73% (p=0.000 n=9+10) +MultiSum64Uint128/4KB 2.54GB/s ยฑ 0% 6.49GB/s ยฑ 1% +155.69% (p=0.000 n=10+10) + +name old hash/s new hash/s delta +MultiSum64Uint8/4KB 621M ยฑ 0% 1823M ยฑ 1% +193.73% (p=0.000 n=10+10) +MultiSum64Uint16/4KB 444M ยฑ 0% 1182M ยฑ 0% +166.20% (p=0.000 n=10+9) +MultiSum64Uint32/4KB 560M ยฑ 0% 1742M ยฑ 1% +210.93% (p=0.000 n=10+10) +MultiSum64Uint64/4KB 446M ยฑ 0% 1391M ยฑ 1% +211.73% (p=0.000 n=9+10) +MultiSum64Uint128/4KB 317M ยฑ 0% 811M ยฑ 1% +155.69% (p=0.000 n=10+10) + +The functions perform runtime detection of AVX-512 support by testing the value +of the xxhash.hasAVX512 variable declared and initialized in sum64uint_amd64.go. +Branch mispredictions on those tests are very unlikely since the value is never +modified by the application. The cost of the comparisons are also amortized by +the bulk APIs of the MultiSum64* functions (a single test is required per call). + +If a bug is suspected in the vectorized code, compiling the program or running +the tests with -tags=purego can help verify whether the behavior changes when +the program does not use the assembly versions. + +Maintenance of these functions can be complex; however, the XXH64 algorithm is +unlikely to evolve, and the implementations unlikely to change. The tests in +sum64uint_test.go compare the outputs of MultiSum64* functions with the +reference xxhash.Sum64 function, future maintainers can rely on those tests +passing as a guarantee that they have not introduced regressions. +*/ + +#define PRIME1 0x9E3779B185EBCA87 +#define PRIME2 0xC2B2AE3D27D4EB4F +#define PRIME3 0x165667B19E3779F9 +#define PRIME4 0x85EBCA77C2B2AE63 +#define PRIME5 0x27D4EB2F165667C5 + +#define prime1 R12 +#define prime2 R13 +#define prime3 R14 +#define prime4 R11 +#define prime5 R11 // same as prime4 because they are not used together + +#define prime1ZMM Z12 +#define prime2ZMM Z13 +#define prime3ZMM Z14 +#define prime4ZMM Z15 +#define prime5ZMM Z15 + +DATA prime1vec<>+0(SB)/8, $PRIME1 +DATA prime1vec<>+8(SB)/8, $PRIME1 +DATA prime1vec<>+16(SB)/8, $PRIME1 +DATA prime1vec<>+24(SB)/8, $PRIME1 +DATA prime1vec<>+32(SB)/8, $PRIME1 +DATA prime1vec<>+40(SB)/8, $PRIME1 +DATA prime1vec<>+48(SB)/8, $PRIME1 +DATA prime1vec<>+56(SB)/8, $PRIME1 +GLOBL prime1vec<>(SB), RODATA|NOPTR, $64 + +DATA prime2vec<>+0(SB)/8, $PRIME2 +DATA prime2vec<>+8(SB)/8, $PRIME2 +DATA prime2vec<>+16(SB)/8, $PRIME2 +DATA prime2vec<>+24(SB)/8, $PRIME2 +DATA prime2vec<>+32(SB)/8, $PRIME2 +DATA prime2vec<>+40(SB)/8, $PRIME2 +DATA prime2vec<>+48(SB)/8, $PRIME2 +DATA prime2vec<>+56(SB)/8, $PRIME2 +GLOBL prime2vec<>(SB), RODATA|NOPTR, $64 + +DATA prime3vec<>+0(SB)/8, $PRIME3 +DATA prime3vec<>+8(SB)/8, $PRIME3 +DATA prime3vec<>+16(SB)/8, $PRIME3 +DATA prime3vec<>+24(SB)/8, $PRIME3 +DATA prime3vec<>+32(SB)/8, $PRIME3 +DATA prime3vec<>+40(SB)/8, $PRIME3 +DATA prime3vec<>+48(SB)/8, $PRIME3 +DATA prime3vec<>+56(SB)/8, $PRIME3 +GLOBL prime3vec<>(SB), RODATA|NOPTR, $64 + +DATA prime4vec<>+0(SB)/8, $PRIME4 +DATA prime4vec<>+8(SB)/8, $PRIME4 +DATA prime4vec<>+16(SB)/8, $PRIME4 +DATA prime4vec<>+24(SB)/8, $PRIME4 +DATA prime4vec<>+32(SB)/8, $PRIME4 +DATA prime4vec<>+40(SB)/8, $PRIME4 +DATA prime4vec<>+48(SB)/8, $PRIME4 +DATA prime4vec<>+56(SB)/8, $PRIME4 +GLOBL prime4vec<>(SB), RODATA|NOPTR, $64 + +DATA prime5vec<>+0(SB)/8, $PRIME5 +DATA prime5vec<>+8(SB)/8, $PRIME5 +DATA prime5vec<>+16(SB)/8, $PRIME5 +DATA prime5vec<>+24(SB)/8, $PRIME5 +DATA prime5vec<>+32(SB)/8, $PRIME5 +DATA prime5vec<>+40(SB)/8, $PRIME5 +DATA prime5vec<>+48(SB)/8, $PRIME5 +DATA prime5vec<>+56(SB)/8, $PRIME5 +GLOBL prime5vec<>(SB), RODATA|NOPTR, $64 + +DATA prime5vec1<>+0(SB)/8, $PRIME5+1 +DATA prime5vec1<>+8(SB)/8, $PRIME5+1 +DATA prime5vec1<>+16(SB)/8, $PRIME5+1 +DATA prime5vec1<>+24(SB)/8, $PRIME5+1 +DATA prime5vec1<>+32(SB)/8, $PRIME5+1 +DATA prime5vec1<>+40(SB)/8, $PRIME5+1 +DATA prime5vec1<>+48(SB)/8, $PRIME5+1 +DATA prime5vec1<>+56(SB)/8, $PRIME5+1 +GLOBL prime5vec1<>(SB), RODATA|NOPTR, $64 + +DATA prime5vec2<>+0(SB)/8, $PRIME5+2 +DATA prime5vec2<>+8(SB)/8, $PRIME5+2 +DATA prime5vec2<>+16(SB)/8, $PRIME5+2 +DATA prime5vec2<>+24(SB)/8, $PRIME5+2 +DATA prime5vec2<>+32(SB)/8, $PRIME5+2 +DATA prime5vec2<>+40(SB)/8, $PRIME5+2 +DATA prime5vec2<>+48(SB)/8, $PRIME5+2 +DATA prime5vec2<>+56(SB)/8, $PRIME5+2 +GLOBL prime5vec2<>(SB), RODATA|NOPTR, $64 + +DATA prime5vec4<>+0(SB)/8, $PRIME5+4 +DATA prime5vec4<>+8(SB)/8, $PRIME5+4 +DATA prime5vec4<>+16(SB)/8, $PRIME5+4 +DATA prime5vec4<>+24(SB)/8, $PRIME5+4 +DATA prime5vec4<>+32(SB)/8, $PRIME5+4 +DATA prime5vec4<>+40(SB)/8, $PRIME5+4 +DATA prime5vec4<>+48(SB)/8, $PRIME5+4 +DATA prime5vec4<>+56(SB)/8, $PRIME5+4 +GLOBL prime5vec4<>(SB), RODATA|NOPTR, $64 + +DATA prime5vec8<>+0(SB)/8, $PRIME5+8 +DATA prime5vec8<>+8(SB)/8, $PRIME5+8 +DATA prime5vec8<>+16(SB)/8, $PRIME5+8 +DATA prime5vec8<>+24(SB)/8, $PRIME5+8 +DATA prime5vec8<>+32(SB)/8, $PRIME5+8 +DATA prime5vec8<>+40(SB)/8, $PRIME5+8 +DATA prime5vec8<>+48(SB)/8, $PRIME5+8 +DATA prime5vec8<>+56(SB)/8, $PRIME5+8 +GLOBL prime5vec8<>(SB), RODATA|NOPTR, $64 + +DATA prime5vec16<>+0(SB)/8, $PRIME5+16 +DATA prime5vec16<>+8(SB)/8, $PRIME5+16 +DATA prime5vec16<>+16(SB)/8, $PRIME5+16 +DATA prime5vec16<>+24(SB)/8, $PRIME5+16 +DATA prime5vec16<>+32(SB)/8, $PRIME5+16 +DATA prime5vec16<>+40(SB)/8, $PRIME5+16 +DATA prime5vec16<>+48(SB)/8, $PRIME5+16 +DATA prime5vec16<>+56(SB)/8, $PRIME5+16 +GLOBL prime5vec16<>(SB), RODATA|NOPTR, $64 + +DATA lowbytemask<>+0(SB)/8, $0xFF +DATA lowbytemask<>+8(SB)/8, $0xFF +DATA lowbytemask<>+16(SB)/8, $0xFF +DATA lowbytemask<>+24(SB)/8, $0xFF +DATA lowbytemask<>+32(SB)/8, $0xFF +DATA lowbytemask<>+40(SB)/8, $0xFF +DATA lowbytemask<>+48(SB)/8, $0xFF +DATA lowbytemask<>+56(SB)/8, $0xFF +GLOBL lowbytemask<>(SB), RODATA|NOPTR, $64 + +DATA vpermi2qeven<>+0(SB)/8, $0 +DATA vpermi2qeven<>+8(SB)/8, $2 +DATA vpermi2qeven<>+16(SB)/8, $4 +DATA vpermi2qeven<>+24(SB)/8, $6 +DATA vpermi2qeven<>+32(SB)/8, $(1<<3)|0 +DATA vpermi2qeven<>+40(SB)/8, $(1<<3)|2 +DATA vpermi2qeven<>+48(SB)/8, $(1<<3)|4 +DATA vpermi2qeven<>+56(SB)/8, $(1<<3)|6 +GLOBL vpermi2qeven<>(SB), RODATA|NOPTR, $64 + +DATA vpermi2qodd<>+0(SB)/8, $1 +DATA vpermi2qodd<>+8(SB)/8, $3 +DATA vpermi2qodd<>+16(SB)/8, $5 +DATA vpermi2qodd<>+24(SB)/8, $7 +DATA vpermi2qodd<>+32(SB)/8, $(1<<3)|1 +DATA vpermi2qodd<>+40(SB)/8, $(1<<3)|3 +DATA vpermi2qodd<>+48(SB)/8, $(1<<3)|5 +DATA vpermi2qodd<>+56(SB)/8, $(1<<3)|7 +GLOBL vpermi2qodd<>(SB), RODATA|NOPTR, $64 + +#define round(input, acc) \ + IMULQ prime2, input \ + ADDQ input, acc \ + ROLQ $31, acc \ + IMULQ prime1, acc + +#define avalanche(tmp, acc) \ + MOVQ acc, tmp \ + SHRQ $33, tmp \ + XORQ tmp, acc \ + IMULQ prime2, acc \ + MOVQ acc, tmp \ + SHRQ $29, tmp \ + XORQ tmp, acc \ + IMULQ prime3, acc \ + MOVQ acc, tmp \ + SHRQ $32, tmp \ + XORQ tmp, acc + +#define round8x64(input, acc) \ + VPMULLQ prime2ZMM, input, input \ + VPADDQ input, acc, acc \ + VPROLQ $31, acc, acc \ + VPMULLQ prime1ZMM, acc, acc + +#define avalanche8x64(tmp, acc) \ + VPSRLQ $33, acc, tmp \ + VPXORQ tmp, acc, acc \ + VPMULLQ prime2ZMM, acc, acc \ + VPSRLQ $29, acc, tmp \ + VPXORQ tmp, acc, acc \ + VPMULLQ prime3ZMM, acc, acc \ + VPSRLQ $32, acc, tmp \ + VPXORQ tmp, acc, acc + +// func MultiSum64Uint8(h []uint64, v []uint8) int +TEXT ยทMultiSum64Uint8(SB), NOSPLIT, $0-54 + MOVQ $PRIME1, prime1 + MOVQ $PRIME2, prime2 + MOVQ $PRIME3, prime3 + MOVQ $PRIME5, prime5 + + MOVQ h_base+0(FP), AX + MOVQ h_len+8(FP), CX + MOVQ v_base+24(FP), BX + MOVQ v_len+32(FP), DX + + CMPQ CX, DX + CMOVQGT DX, CX + MOVQ CX, ret+48(FP) + + XORQ SI, SI + CMPQ CX, $32 + JB loop + CMPB ยทhasAVX512(SB), $0 + JE loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + + VMOVDQU64 prime1vec<>(SB), prime1ZMM + VMOVDQU64 prime2vec<>(SB), prime2ZMM + VMOVDQU64 prime3vec<>(SB), prime3ZMM + VMOVDQU64 prime5vec<>(SB), prime5ZMM + VMOVDQU64 prime5vec1<>(SB), Z6 +loop32x64: + VMOVDQA64 Z6, Z0 + VMOVDQA64 Z6, Z3 + VMOVDQA64 Z6, Z20 + VMOVDQA64 Z6, Z23 + VPMOVZXBQ (BX)(SI*1), Z1 + VPMOVZXBQ 8(BX)(SI*1), Z4 + VPMOVZXBQ 16(BX)(SI*1), Z21 + VPMOVZXBQ 24(BX)(SI*1), Z24 + + VPMULLQ prime5ZMM, Z1, Z1 + VPMULLQ prime5ZMM, Z4, Z4 + VPMULLQ prime5ZMM, Z21, Z21 + VPMULLQ prime5ZMM, Z24, Z24 + VPXORQ Z1, Z0, Z0 + VPXORQ Z4, Z3, Z3 + VPXORQ Z21, Z20, Z20 + VPXORQ Z24, Z23, Z23 + VPROLQ $11, Z0, Z0 + VPROLQ $11, Z3, Z3 + VPROLQ $11, Z20, Z20 + VPROLQ $11, Z23, Z23 + VPMULLQ prime1ZMM, Z0, Z0 + VPMULLQ prime1ZMM, Z3, Z3 + VPMULLQ prime1ZMM, Z20, Z20 + VPMULLQ prime1ZMM, Z23, Z23 + + avalanche8x64(Z1, Z0) + avalanche8x64(Z4, Z3) + avalanche8x64(Z21, Z20) + avalanche8x64(Z24, Z23) + + VMOVDQU64 Z0, (AX)(SI*8) + VMOVDQU64 Z3, 64(AX)(SI*8) + VMOVDQU64 Z20, 128(AX)(SI*8) + VMOVDQU64 Z23, 192(AX)(SI*8) + ADDQ $32, SI + CMPQ SI, DI + JB loop32x64 + VZEROUPPER +loop: + CMPQ SI, CX + JE done + MOVQ $PRIME5+1, R8 + MOVBQZX (BX)(SI*1), R9 + + IMULQ prime5, R9 + XORQ R9, R8 + ROLQ $11, R8 + IMULQ prime1, R8 + avalanche(R9, R8) + + MOVQ R8, (AX)(SI*8) + INCQ SI + JMP loop +done: + RET + +// func MultiSum64Uint16(h []uint64, v []uint16) int +TEXT ยทMultiSum64Uint16(SB), NOSPLIT, $0-54 + MOVQ $PRIME1, prime1 + MOVQ $PRIME2, prime2 + MOVQ $PRIME3, prime3 + MOVQ $PRIME5, prime5 + + MOVQ h_base+0(FP), AX + MOVQ h_len+8(FP), CX + MOVQ v_base+24(FP), BX + MOVQ v_len+32(FP), DX + + CMPQ CX, DX + CMOVQGT DX, CX + MOVQ CX, ret+48(FP) + + XORQ SI, SI + CMPQ CX, $16 + JB loop + CMPB ยทhasAVX512(SB), $0 + JE loop + + MOVQ CX, DI + SHRQ $4, DI + SHLQ $4, DI + + VMOVDQU64 prime1vec<>(SB), prime1ZMM + VMOVDQU64 prime2vec<>(SB), prime2ZMM + VMOVDQU64 prime3vec<>(SB), prime3ZMM + VMOVDQU64 prime5vec<>(SB), prime5ZMM + VMOVDQU64 prime5vec2<>(SB), Z6 + VMOVDQU64 lowbytemask<>(SB), Z7 +loop16x64: + VMOVDQA64 Z6, Z0 + VMOVDQA64 Z6, Z3 + VPMOVZXWQ (BX)(SI*2), Z1 + VPMOVZXWQ 16(BX)(SI*2), Z4 + + VMOVDQA64 Z1, Z8 + VMOVDQA64 Z4, Z9 + VPSRLQ $8, Z8, Z8 + VPSRLQ $8, Z9, Z9 + VPANDQ Z7, Z1, Z1 + VPANDQ Z7, Z4, Z4 + + VPMULLQ prime5ZMM, Z1, Z1 + VPMULLQ prime5ZMM, Z4, Z4 + VPXORQ Z1, Z0, Z0 + VPXORQ Z4, Z3, Z3 + VPROLQ $11, Z0, Z0 + VPROLQ $11, Z3, Z3 + VPMULLQ prime1ZMM, Z0, Z0 + VPMULLQ prime1ZMM, Z3, Z3 + + VPMULLQ prime5ZMM, Z8, Z8 + VPMULLQ prime5ZMM, Z9, Z9 + VPXORQ Z8, Z0, Z0 + VPXORQ Z9, Z3, Z3 + VPROLQ $11, Z0, Z0 + VPROLQ $11, Z3, Z3 + VPMULLQ prime1ZMM, Z0, Z0 + VPMULLQ prime1ZMM, Z3, Z3 + + avalanche8x64(Z1, Z0) + avalanche8x64(Z4, Z3) + + VMOVDQU64 Z0, (AX)(SI*8) + VMOVDQU64 Z3, 64(AX)(SI*8) + ADDQ $16, SI + CMPQ SI, DI + JB loop16x64 + VZEROUPPER +loop: + CMPQ SI, CX + JE done + MOVQ $PRIME5+2, R8 + MOVWQZX (BX)(SI*2), R9 + + MOVQ R9, R10 + SHRQ $8, R10 + ANDQ $0xFF, R9 + + IMULQ prime5, R9 + XORQ R9, R8 + ROLQ $11, R8 + IMULQ prime1, R8 + + IMULQ prime5, R10 + XORQ R10, R8 + ROLQ $11, R8 + IMULQ prime1, R8 + + avalanche(R9, R8) + + MOVQ R8, (AX)(SI*8) + INCQ SI + JMP loop +done: + RET + +// func MultiSum64Uint32(h []uint64, v []uint32) int +TEXT ยทMultiSum64Uint32(SB), NOSPLIT, $0-54 + MOVQ $PRIME1, prime1 + MOVQ $PRIME2, prime2 + MOVQ $PRIME3, prime3 + + MOVQ h_base+0(FP), AX + MOVQ h_len+8(FP), CX + MOVQ v_base+24(FP), BX + MOVQ v_len+32(FP), DX + + CMPQ CX, DX + CMOVQGT DX, CX + MOVQ CX, ret+48(FP) + + XORQ SI, SI + CMPQ CX, $32 + JB loop + CMPB ยทhasAVX512(SB), $0 + JE loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + + VMOVDQU64 prime1vec<>(SB), prime1ZMM + VMOVDQU64 prime2vec<>(SB), prime2ZMM + VMOVDQU64 prime3vec<>(SB), prime3ZMM + VMOVDQU64 prime5vec4<>(SB), Z6 +loop32x64: + VMOVDQA64 Z6, Z0 + VMOVDQA64 Z6, Z3 + VMOVDQA64 Z6, Z20 + VMOVDQA64 Z6, Z23 + VPMOVZXDQ (BX)(SI*4), Z1 + VPMOVZXDQ 32(BX)(SI*4), Z4 + VPMOVZXDQ 64(BX)(SI*4), Z21 + VPMOVZXDQ 96(BX)(SI*4), Z24 + + VPMULLQ prime1ZMM, Z1, Z1 + VPMULLQ prime1ZMM, Z4, Z4 + VPMULLQ prime1ZMM, Z21, Z21 + VPMULLQ prime1ZMM, Z24, Z24 + VPXORQ Z1, Z0, Z0 + VPXORQ Z4, Z3, Z3 + VPXORQ Z21, Z20, Z20 + VPXORQ Z24, Z23, Z23 + VPROLQ $23, Z0, Z0 + VPROLQ $23, Z3, Z3 + VPROLQ $23, Z20, Z20 + VPROLQ $23, Z23, Z23 + VPMULLQ prime2ZMM, Z0, Z0 + VPMULLQ prime2ZMM, Z3, Z3 + VPMULLQ prime2ZMM, Z20, Z20 + VPMULLQ prime2ZMM, Z23, Z23 + VPADDQ prime3ZMM, Z0, Z0 + VPADDQ prime3ZMM, Z3, Z3 + VPADDQ prime3ZMM, Z20, Z20 + VPADDQ prime3ZMM, Z23, Z23 + + avalanche8x64(Z1, Z0) + avalanche8x64(Z4, Z3) + avalanche8x64(Z21, Z20) + avalanche8x64(Z24, Z23) + + VMOVDQU64 Z0, (AX)(SI*8) + VMOVDQU64 Z3, 64(AX)(SI*8) + VMOVDQU64 Z20, 128(AX)(SI*8) + VMOVDQU64 Z23, 192(AX)(SI*8) + ADDQ $32, SI + CMPQ SI, DI + JB loop32x64 + VZEROUPPER +loop: + CMPQ SI, CX + JE done + MOVQ $PRIME5+4, R8 + MOVLQZX (BX)(SI*4), R9 + + IMULQ prime1, R9 + XORQ R9, R8 + ROLQ $23, R8 + IMULQ prime2, R8 + ADDQ prime3, R8 + avalanche(R9, R8) + + MOVQ R8, (AX)(SI*8) + INCQ SI + JMP loop +done: + RET + +// func MultiSum64Uint64(h []uint64, v []uint64) int +TEXT ยทMultiSum64Uint64(SB), NOSPLIT, $0-54 + MOVQ $PRIME1, prime1 + MOVQ $PRIME2, prime2 + MOVQ $PRIME3, prime3 + MOVQ $PRIME4, prime4 + + MOVQ h_base+0(FP), AX + MOVQ h_len+8(FP), CX + MOVQ v_base+24(FP), BX + MOVQ v_len+32(FP), DX + + CMPQ CX, DX + CMOVQGT DX, CX + MOVQ CX, ret+48(FP) + + XORQ SI, SI + CMPQ CX, $32 + JB loop + CMPB ยทhasAVX512(SB), $0 + JE loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + + VMOVDQU64 prime1vec<>(SB), prime1ZMM + VMOVDQU64 prime2vec<>(SB), prime2ZMM + VMOVDQU64 prime3vec<>(SB), prime3ZMM + VMOVDQU64 prime4vec<>(SB), prime4ZMM + VMOVDQU64 prime5vec8<>(SB), Z6 +loop32x64: + VMOVDQA64 Z6, Z0 + VMOVDQA64 Z6, Z3 + VMOVDQA64 Z6, Z20 + VMOVDQA64 Z6, Z23 + VMOVDQU64 (BX)(SI*8), Z1 + VMOVDQU64 64(BX)(SI*8), Z4 + VMOVDQU64 128(BX)(SI*8), Z21 + VMOVDQU64 192(BX)(SI*8), Z24 + + VPXORQ Z2, Z2, Z2 + VPXORQ Z5, Z5, Z5 + VPXORQ Z22, Z22, Z22 + VPXORQ Z25, Z25, Z25 + round8x64(Z1, Z2) + round8x64(Z4, Z5) + round8x64(Z21, Z22) + round8x64(Z24, Z25) + + VPXORQ Z2, Z0, Z0 + VPXORQ Z5, Z3, Z3 + VPXORQ Z22, Z20, Z20 + VPXORQ Z25, Z23, Z23 + VPROLQ $27, Z0, Z0 + VPROLQ $27, Z3, Z3 + VPROLQ $27, Z20, Z20 + VPROLQ $27, Z23, Z23 + VPMULLQ prime1ZMM, Z0, Z0 + VPMULLQ prime1ZMM, Z3, Z3 + VPMULLQ prime1ZMM, Z20, Z20 + VPMULLQ prime1ZMM, Z23, Z23 + VPADDQ prime4ZMM, Z0, Z0 + VPADDQ prime4ZMM, Z3, Z3 + VPADDQ prime4ZMM, Z20, Z20 + VPADDQ prime4ZMM, Z23, Z23 + + avalanche8x64(Z1, Z0) + avalanche8x64(Z4, Z3) + avalanche8x64(Z21, Z20) + avalanche8x64(Z24, Z23) + + VMOVDQU64 Z0, (AX)(SI*8) + VMOVDQU64 Z3, 64(AX)(SI*8) + VMOVDQU64 Z20, 128(AX)(SI*8) + VMOVDQU64 Z23, 192(AX)(SI*8) + ADDQ $32, SI + CMPQ SI, DI + JB loop32x64 + VZEROUPPER +loop: + CMPQ SI, CX + JE done + MOVQ $PRIME5+8, R8 + MOVQ (BX)(SI*8), R9 + + XORQ R10, R10 + round(R9, R10) + XORQ R10, R8 + ROLQ $27, R8 + IMULQ prime1, R8 + ADDQ prime4, R8 + avalanche(R9, R8) + + MOVQ R8, (AX)(SI*8) + INCQ SI + JMP loop +done: + RET + +// func MultiSum64Uint128(h []uint64, v [][16]byte) int +TEXT ยทMultiSum64Uint128(SB), NOSPLIT, $0-54 + MOVQ $PRIME1, prime1 + MOVQ $PRIME2, prime2 + MOVQ $PRIME3, prime3 + MOVQ $PRIME4, prime4 + + MOVQ h_base+0(FP), AX + MOVQ h_len+8(FP), CX + MOVQ v_base+24(FP), BX + MOVQ v_len+32(FP), DX + + CMPQ CX, DX + CMOVQGT DX, CX + MOVQ CX, ret+48(FP) + + XORQ SI, SI + CMPQ CX, $16 + JB loop + CMPB ยทhasAVX512(SB), $0 + JE loop + + MOVQ CX, DI + SHRQ $4, DI + SHLQ $4, DI + + VMOVDQU64 prime1vec<>(SB), prime1ZMM + VMOVDQU64 prime2vec<>(SB), prime2ZMM + VMOVDQU64 prime3vec<>(SB), prime3ZMM + VMOVDQU64 prime4vec<>(SB), prime4ZMM + VMOVDQU64 prime5vec16<>(SB), Z6 + VMOVDQU64 vpermi2qeven<>(SB), Z7 + VMOVDQU64 vpermi2qodd<>(SB), Z8 +loop16x64: + // This algorithm is slightly different from the other ones, because it is + // the only case where the input values are larger than the output (128 bits + // vs 64 bits). + // + // Computing the XXH64 of 128 bits values requires doing two passes over the + // lower and upper 64 bits. The lower and upper quad/ words are split in + // separate vectors, the first pass is applied on the vector holding the + // lower bits of 8 input values, then the second pass is applied with the + // vector holding the upper bits. + // + // Following the model used in the other functions, we unroll the work of + // two consecutive groups of 8 values per loop iteration in order to + // maximize utilization of CPU resources. + CMPQ SI, DI + JE loop + VMOVDQA64 Z6, Z0 + VMOVDQA64 Z6, Z20 + VMOVDQU64 (BX), Z1 + VMOVDQU64 64(BX), Z9 + VMOVDQU64 128(BX), Z21 + VMOVDQU64 192(BX), Z29 + + VMOVDQA64 Z7, Z2 + VMOVDQA64 Z8, Z3 + VMOVDQA64 Z7, Z22 + VMOVDQA64 Z8, Z23 + + VPERMI2Q Z9, Z1, Z2 + VPERMI2Q Z9, Z1, Z3 + VPERMI2Q Z29, Z21, Z22 + VPERMI2Q Z29, Z21, Z23 + + // Compute the rounds on inputs. + VPXORQ Z4, Z4, Z4 + VPXORQ Z5, Z5, Z5 + VPXORQ Z24, Z24, Z24 + VPXORQ Z25, Z25, Z25 + round8x64(Z2, Z4) + round8x64(Z3, Z5) + round8x64(Z22, Z24) + round8x64(Z23, Z25) + + // Lower 64 bits. + VPXORQ Z4, Z0, Z0 + VPXORQ Z24, Z20, Z20 + VPROLQ $27, Z0, Z0 + VPROLQ $27, Z20, Z20 + VPMULLQ prime1ZMM, Z0, Z0 + VPMULLQ prime1ZMM, Z20, Z20 + VPADDQ prime4ZMM, Z0, Z0 + VPADDQ prime4ZMM, Z20, Z20 + + // Upper 64 bits. + VPXORQ Z5, Z0, Z0 + VPXORQ Z25, Z20, Z20 + VPROLQ $27, Z0, Z0 + VPROLQ $27, Z20, Z20 + VPMULLQ prime1ZMM, Z0, Z0 + VPMULLQ prime1ZMM, Z20, Z20 + VPADDQ prime4ZMM, Z0, Z0 + VPADDQ prime4ZMM, Z20, Z20 + + avalanche8x64(Z1, Z0) + avalanche8x64(Z21, Z20) + VMOVDQU64 Z0, (AX)(SI*8) + VMOVDQU64 Z20, 64(AX)(SI*8) + ADDQ $256, BX + ADDQ $16, SI + JMP loop16x64 + VZEROUPPER +loop: + CMPQ SI, CX + JE done + MOVQ $PRIME5+16, R8 + MOVQ (BX), DX + MOVQ 8(BX), DI + + XORQ R9, R9 + XORQ R10, R10 + round(DX, R9) + round(DI, R10) + + XORQ R9, R8 + ROLQ $27, R8 + IMULQ prime1, R8 + ADDQ prime4, R8 + + XORQ R10, R8 + ROLQ $27, R8 + IMULQ prime1, R8 + ADDQ prime4, R8 + + avalanche(R9, R8) + MOVQ R8, (AX)(SI*8) + ADDQ $16, BX + INCQ SI + JMP loop +done: + RET diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_purego.go b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_purego.go new file mode 100644 index 0000000000..4b420dc4f7 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/sum64uint_purego.go @@ -0,0 +1,53 @@ +//go:build purego || !amd64 + +package xxhash + +func MultiSum64Uint8(h []uint64, v []uint8) int { + n := min(len(h), len(v)) + h = h[:n] + v = v[:n] + for i := range v { + h[i] = Sum64Uint8(v[i]) + } + return n +} + +func MultiSum64Uint16(h []uint64, v []uint16) int { + n := min(len(h), len(v)) + h = h[:n] + v = v[:n] + for i := range v { + h[i] = Sum64Uint16(v[i]) + } + return n +} + +func MultiSum64Uint32(h []uint64, v []uint32) int { + n := min(len(h), len(v)) + h = h[:n] + v = v[:n] + for i := range v { + h[i] = Sum64Uint32(v[i]) + } + return n +} + +func MultiSum64Uint64(h []uint64, v []uint64) int { + n := min(len(h), len(v)) + h = h[:n] + v = v[:n] + for i := range v { + h[i] = Sum64Uint64(v[i]) + } + return n +} + +func MultiSum64Uint128(h []uint64, v [][16]byte) int { + n := min(len(h), len(v)) + h = h[:n] + v = v[:n] + for i := range v { + h[i] = Sum64Uint128(v[i]) + } + return n +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash.go b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash.go new file mode 100644 index 0000000000..0d4b44b2dc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash.go @@ -0,0 +1,55 @@ +// Package xxhash is an extension of github.com/cespare/xxhash which adds +// routines optimized to hash arrays of fixed size elements. +package xxhash + +import ( + "encoding/binary" + "math/bits" +) + +const ( + prime1 uint64 = 0x9E3779B185EBCA87 + prime2 uint64 = 0xC2B2AE3D27D4EB4F + prime3 uint64 = 0x165667B19E3779F9 + prime4 uint64 = 0x85EBCA77C2B2AE63 + prime5 uint64 = 0x27D4EB2F165667C5 + // Pre-computed operations because the compiler otherwise complains that the + // results overflow 64 bit integers. + prime1plus2 uint64 = 0x60EA27EEADC0B5D6 // prime1 + prime2 + negprime1 uint64 = 0x61C8864E7A143579 // -prime1 +) + +func avalanche(h uint64) uint64 { + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + return h +} + +func round(acc, input uint64) uint64 { + acc += input * prime2 + acc = rol31(acc) + acc *= prime1 + return acc +} + +func mergeRound(acc, val uint64) uint64 { + val = round(0, val) + acc ^= val + acc = acc*prime1 + prime4 + return acc +} + +func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } +func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } + +func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } +func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } +func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } +func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } +func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } +func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } +func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } +func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.go b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.go new file mode 100644 index 0000000000..4b32923def --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.go @@ -0,0 +1,6 @@ +//go:build !purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +func Sum64(b []byte) uint64 diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.s b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.s new file mode 100644 index 0000000000..8f88dc1d6a --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_amd64.s @@ -0,0 +1,180 @@ +//go:build !purego + +#include "textflag.h" + +#define PRIME1 0x9E3779B185EBCA87 +#define PRIME2 0xC2B2AE3D27D4EB4F +#define PRIME3 0x165667B19E3779F9 +#define PRIME4 0x85EBCA77C2B2AE63 +#define PRIME5 0x27D4EB2F165667C5 + +DATA prime3<>+0(SB)/8, $PRIME3 +GLOBL prime3<>(SB), RODATA|NOPTR, $8 + +DATA prime5<>+0(SB)/8, $PRIME5 +GLOBL prime5<>(SB), RODATA|NOPTR, $8 + +// Register allocation: +// AX h +// SI pointer to advance through b +// DX n +// BX loop end +// R8 v1, k1 +// R9 v2 +// R10 v3 +// R11 v4 +// R12 tmp +// R13 PRIME1 +// R14 PRIME2 +// DI PRIME4 + +// round reads from and advances the buffer pointer in SI. +// It assumes that R13 has PRIME1 and R14 has PRIME2. +#define round(r) \ + MOVQ (SI), R12 \ + ADDQ $8, SI \ + IMULQ R14, R12 \ + ADDQ R12, r \ + ROLQ $31, r \ + IMULQ R13, r + +// mergeRound applies a merge round on the two registers acc and val. +// It assumes that R13 has PRIME1, R14 has PRIME2, and DI has PRIME4. +#define mergeRound(acc, val) \ + IMULQ R14, val \ + ROLQ $31, val \ + IMULQ R13, val \ + XORQ val, acc \ + IMULQ R13, acc \ + ADDQ DI, acc + +// func Sum64(b []byte) uint64 +TEXT ยทSum64(SB), NOSPLIT, $0-32 + // Load fixed primes. + MOVQ $PRIME1, R13 + MOVQ $PRIME2, R14 + MOVQ $PRIME4, DI + + // Load slice. + MOVQ b_base+0(FP), SI + MOVQ b_len+8(FP), DX + LEAQ (SI)(DX*1), BX + + // The first loop limit will be len(b)-32. + SUBQ $32, BX + + // Check whether we have at least one block. + CMPQ DX, $32 + JLT noBlocks + + // Set up initial state (v1, v2, v3, v4). + MOVQ R13, R8 + ADDQ R14, R8 + MOVQ R14, R9 + XORQ R10, R10 + XORQ R11, R11 + SUBQ R13, R11 + + // Loop until SI > BX. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ SI, BX + JLE blockLoop + + MOVQ R8, AX + ROLQ $1, AX + MOVQ R9, R12 + ROLQ $7, R12 + ADDQ R12, AX + MOVQ R10, R12 + ROLQ $12, R12 + ADDQ R12, AX + MOVQ R11, R12 + ROLQ $18, R12 + ADDQ R12, AX + + mergeRound(AX, R8) + mergeRound(AX, R9) + mergeRound(AX, R10) + mergeRound(AX, R11) + + JMP afterBlocks + +noBlocks: + MOVQ $PRIME5, AX + +afterBlocks: + ADDQ DX, AX + + // Right now BX has len(b)-32, and we want to loop until SI > len(b)-8. + ADDQ $24, BX + + CMPQ SI, BX + JG fourByte + +wordLoop: + // Calculate k1. + MOVQ (SI), R8 + ADDQ $8, SI + IMULQ R14, R8 + ROLQ $31, R8 + IMULQ R13, R8 + + XORQ R8, AX + ROLQ $27, AX + IMULQ R13, AX + ADDQ DI, AX + + CMPQ SI, BX + JLE wordLoop + +fourByte: + ADDQ $4, BX + CMPQ SI, BX + JG singles + + MOVL (SI), R8 + ADDQ $4, SI + IMULQ R13, R8 + XORQ R8, AX + + ROLQ $23, AX + IMULQ R14, AX + ADDQ prime3<>(SB), AX + +singles: + ADDQ $4, BX + CMPQ SI, BX + JGE finalize + +singlesLoop: + MOVBQZX (SI), R12 + ADDQ $1, SI + IMULQ prime5<>(SB), R12 + XORQ R12, AX + + ROLQ $11, AX + IMULQ R13, AX + + CMPQ SI, BX + JL singlesLoop + +finalize: + MOVQ AX, R12 + SHRQ $33, R12 + XORQ R12, AX + IMULQ R14, AX + MOVQ AX, R12 + SHRQ $29, R12 + XORQ R12, AX + IMULQ prime3<>(SB), AX + MOVQ AX, R12 + SHRQ $32, R12 + XORQ R12, AX + + MOVQ AX, ret+24(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_purego.go b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_purego.go new file mode 100644 index 0000000000..896f817a5f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom/xxhash/xxhash_purego.go @@ -0,0 +1,50 @@ +//go:build purego || !amd64 + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +func Sum64(b []byte) uint64 { + var n = len(b) + var h uint64 + + if n >= 32 { + v1 := prime1plus2 + v2 := prime2 + v3 := uint64(0) + v4 := negprime1 + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = prime5 + } + + h += uint64(n) + + i, end := 0, len(b) + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(b[i:i+8:len(b)])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for ; i < end; i++ { + h ^= uint64(b[i]) * prime5 + h = rol11(h) * prime1 + } + + return avalanche(h) +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom_be.go b/vendor/github.com/parquet-go/parquet-go/bloom_be.go new file mode 100644 index 0000000000..f7800301a6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom_be.go @@ -0,0 +1,19 @@ +//go:build s390x + +package parquet + +import ( + "encoding/binary" + + "github.com/parquet-go/parquet-go/deprecated" +) + +func unsafecastInt96ToBytes(src []deprecated.Int96) []byte { + out := make([]byte, len(src)*12) + for i := range src { + binary.LittleEndian.PutUint32(out[(i*12):4+(i*12)], uint32(src[i][0])) + binary.LittleEndian.PutUint32(out[4+(i*12):8+(i*12)], uint32(src[i][1])) + binary.LittleEndian.PutUint32(out[8+(i*12):12+(i*12)], uint32(src[i][2])) + } + return out +} diff --git a/vendor/github.com/parquet-go/parquet-go/bloom_le.go b/vendor/github.com/parquet-go/parquet-go/bloom_le.go new file mode 100644 index 0000000000..5b93bf0717 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/bloom_le.go @@ -0,0 +1,12 @@ +//go:build !s390x + +package parquet + +import ( + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +func unsafecastInt96ToBytes(src []deprecated.Int96) []byte { + return unsafecast.Slice[byte](src) +} diff --git a/vendor/github.com/parquet-go/parquet-go/buffer.go b/vendor/github.com/parquet-go/parquet-go/buffer.go new file mode 100644 index 0000000000..add907a249 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/buffer.go @@ -0,0 +1,716 @@ +package parquet + +import ( + "log" + "reflect" + "runtime" + "sort" + "sync" + "sync/atomic" + + "github.com/parquet-go/parquet-go/internal/debug" +) + +// GenericBuffer is similar to a Buffer but uses a type parameter to define the +// Go type representing the schema of rows in the buffer. +// +// See GenericWriter for details about the benefits over the classic Buffer API. +type GenericBuffer[T any] struct { + base Buffer + write bufferFunc[T] +} + +// NewGenericBuffer is like NewBuffer but returns a GenericBuffer[T] suited to write +// rows of Go type T. +// +// The type parameter T should be a map, struct, or any. Any other types will +// cause a panic at runtime. Type checking is a lot more effective when the +// generic parameter is a struct type, using map and interface types is somewhat +// similar to using a Writer. If using an interface type for the type parameter, +// then providing a schema at instantiation is required. +// +// If the option list may explicitly declare a schema, it must be compatible +// with the schema generated from T. +func NewGenericBuffer[T any](options ...RowGroupOption) *GenericBuffer[T] { + config, err := NewRowGroupConfig(options...) + if err != nil { + panic(err) + } + + t := typeOf[T]() + if config.Schema == nil && t != nil { + config.Schema = schemaOf(dereference(t)) + } + + if config.Schema == nil { + panic("generic buffer must be instantiated with schema or concrete type.") + } + + buf := &GenericBuffer[T]{ + base: Buffer{config: config}, + } + buf.base.configure(config.Schema) + buf.write = bufferFuncOf[T](t, config.Schema) + return buf +} + +func typeOf[T any]() reflect.Type { + var v T + return reflect.TypeOf(v) +} + +type bufferFunc[T any] func(*GenericBuffer[T], []T) (int, error) + +func bufferFuncOf[T any](t reflect.Type, schema *Schema) bufferFunc[T] { + if t == nil { + return (*GenericBuffer[T]).writeRows + } + switch t.Kind() { + case reflect.Interface, reflect.Map: + return (*GenericBuffer[T]).writeRows + + case reflect.Struct: + return makeBufferFunc[T](t, schema) + + case reflect.Pointer: + if e := t.Elem(); e.Kind() == reflect.Struct { + return makeBufferFunc[T](t, schema) + } + } + panic("cannot create buffer for values of type " + t.String()) +} + +func makeBufferFunc[T any](t reflect.Type, schema *Schema) bufferFunc[T] { + writeRows := writeRowsFuncOf(t, schema, nil) + return func(buf *GenericBuffer[T], rows []T) (n int, err error) { + err = writeRows(buf.base.columns, makeArrayOf(rows), columnLevels{}) + if err == nil { + n = len(rows) + } + return n, err + } +} + +func (buf *GenericBuffer[T]) Size() int64 { + return buf.base.Size() +} + +func (buf *GenericBuffer[T]) NumRows() int64 { + return buf.base.NumRows() +} + +func (buf *GenericBuffer[T]) ColumnChunks() []ColumnChunk { + return buf.base.ColumnChunks() +} + +func (buf *GenericBuffer[T]) ColumnBuffers() []ColumnBuffer { + return buf.base.ColumnBuffers() +} + +func (buf *GenericBuffer[T]) SortingColumns() []SortingColumn { + return buf.base.SortingColumns() +} + +func (buf *GenericBuffer[T]) Len() int { + return buf.base.Len() +} + +func (buf *GenericBuffer[T]) Less(i, j int) bool { + return buf.base.Less(i, j) +} + +func (buf *GenericBuffer[T]) Swap(i, j int) { + buf.base.Swap(i, j) +} + +func (buf *GenericBuffer[T]) Reset() { + buf.base.Reset() +} + +func (buf *GenericBuffer[T]) Write(rows []T) (int, error) { + if len(rows) == 0 { + return 0, nil + } + return buf.write(buf, rows) +} + +func (buf *GenericBuffer[T]) WriteRows(rows []Row) (int, error) { + return buf.base.WriteRows(rows) +} + +func (buf *GenericBuffer[T]) WriteRowGroup(rowGroup RowGroup) (int64, error) { + return buf.base.WriteRowGroup(rowGroup) +} + +func (buf *GenericBuffer[T]) Rows() Rows { + return buf.base.Rows() +} + +func (buf *GenericBuffer[T]) Schema() *Schema { + return buf.base.Schema() +} + +func (buf *GenericBuffer[T]) writeRows(rows []T) (int, error) { + if cap(buf.base.rowbuf) < len(rows) { + buf.base.rowbuf = make([]Row, len(rows)) + } else { + buf.base.rowbuf = buf.base.rowbuf[:len(rows)] + } + defer clearRows(buf.base.rowbuf) + + schema := buf.base.Schema() + for i := range rows { + buf.base.rowbuf[i] = schema.Deconstruct(buf.base.rowbuf[i], &rows[i]) + } + + return buf.base.WriteRows(buf.base.rowbuf) +} + +var ( + _ RowGroup = (*GenericBuffer[any])(nil) + _ RowGroupWriter = (*GenericBuffer[any])(nil) + _ sort.Interface = (*GenericBuffer[any])(nil) + + _ RowGroup = (*GenericBuffer[struct{}])(nil) + _ RowGroupWriter = (*GenericBuffer[struct{}])(nil) + _ sort.Interface = (*GenericBuffer[struct{}])(nil) + + _ RowGroup = (*GenericBuffer[map[struct{}]struct{}])(nil) + _ RowGroupWriter = (*GenericBuffer[map[struct{}]struct{}])(nil) + _ sort.Interface = (*GenericBuffer[map[struct{}]struct{}])(nil) +) + +// Buffer represents an in-memory group of parquet rows. +// +// The main purpose of the Buffer type is to provide a way to sort rows before +// writing them to a parquet file. Buffer implements sort.Interface as a way +// to support reordering the rows that have been written to it. +type Buffer struct { + config *RowGroupConfig + schema *Schema + rowbuf []Row + colbuf [][]Value + chunks []ColumnChunk + columns []ColumnBuffer + sorted []ColumnBuffer +} + +// NewBuffer constructs a new buffer, using the given list of buffer options +// to configure the buffer returned by the function. +// +// The function panics if the buffer configuration is invalid. Programs that +// cannot guarantee the validity of the options passed to NewBuffer should +// construct the buffer configuration independently prior to calling this +// function: +// +// config, err := parquet.NewRowGroupConfig(options...) +// if err != nil { +// // handle the configuration error +// ... +// } else { +// // this call to create a buffer is guaranteed not to panic +// buffer := parquet.NewBuffer(config) +// ... +// } +func NewBuffer(options ...RowGroupOption) *Buffer { + config, err := NewRowGroupConfig(options...) + if err != nil { + panic(err) + } + buf := &Buffer{ + config: config, + } + if config.Schema != nil { + buf.configure(config.Schema) + } + return buf +} + +func (buf *Buffer) configure(schema *Schema) { + if schema == nil { + return + } + sortingColumns := buf.config.Sorting.SortingColumns + buf.sorted = make([]ColumnBuffer, len(sortingColumns)) + + forEachLeafColumnOf(schema, func(leaf leafColumn) { + nullOrdering := nullsGoLast + columnIndex := int(leaf.columnIndex) + columnType := leaf.node.Type() + bufferCap := buf.config.ColumnBufferCapacity + dictionary := (Dictionary)(nil) + encoding := encodingOf(leaf.node) + + if isDictionaryEncoding(encoding) { + estimatedDictBufferSize := columnType.EstimateSize(bufferCap) + dictBuffer := columnType.NewValues( + make([]byte, 0, estimatedDictBufferSize), + nil, + ) + dictionary = columnType.NewDictionary(columnIndex, 0, dictBuffer) + columnType = dictionary.Type() + } + + sortingIndex := searchSortingColumn(sortingColumns, leaf.path) + if sortingIndex < len(sortingColumns) && sortingColumns[sortingIndex].NullsFirst() { + nullOrdering = nullsGoFirst + } + + column := columnType.NewColumnBuffer(columnIndex, bufferCap) + switch { + case leaf.maxRepetitionLevel > 0: + column = newRepeatedColumnBuffer(column, leaf.maxRepetitionLevel, leaf.maxDefinitionLevel, nullOrdering) + case leaf.maxDefinitionLevel > 0: + column = newOptionalColumnBuffer(column, leaf.maxDefinitionLevel, nullOrdering) + } + buf.columns = append(buf.columns, column) + + if sortingIndex < len(sortingColumns) { + if sortingColumns[sortingIndex].Descending() { + column = &reversedColumnBuffer{column} + } + buf.sorted[sortingIndex] = column + } + }) + + buf.schema = schema + buf.rowbuf = make([]Row, 0, 1) + buf.colbuf = make([][]Value, len(buf.columns)) + buf.chunks = make([]ColumnChunk, len(buf.columns)) + + for i, column := range buf.columns { + buf.chunks[i] = column + } +} + +// Size returns the estimated size of the buffer in memory (in bytes). +func (buf *Buffer) Size() int64 { + size := int64(0) + for _, col := range buf.columns { + size += col.Size() + } + return size +} + +// NumRows returns the number of rows written to the buffer. +func (buf *Buffer) NumRows() int64 { return int64(buf.Len()) } + +// ColumnChunks returns the buffer columns. +func (buf *Buffer) ColumnChunks() []ColumnChunk { return buf.chunks } + +// ColumnBuffers returns the buffer columns. +// +// This method is similar to ColumnChunks, but returns a list of ColumnBuffer +// instead of a list of ColumnChunk (the latter being read-only); calling +// ColumnBuffers or ColumnChunks with the same index returns the same underlying +// objects, but with different types, which removes the need for making a type +// assertion if the program needed to write directly to the column buffers. +// The presence of the ColumnChunks method is still required to satisfy the +// RowGroup interface. +func (buf *Buffer) ColumnBuffers() []ColumnBuffer { return buf.columns } + +// Schema returns the schema of the buffer. +// +// The schema is either configured by passing a Schema in the option list when +// constructing the buffer, or lazily discovered when the first row is written. +func (buf *Buffer) Schema() *Schema { return buf.schema } + +// SortingColumns returns the list of columns by which the buffer will be +// sorted. +// +// The sorting order is configured by passing a SortingColumns option when +// constructing the buffer. +func (buf *Buffer) SortingColumns() []SortingColumn { return buf.config.Sorting.SortingColumns } + +// Len returns the number of rows written to the buffer. +func (buf *Buffer) Len() int { + if len(buf.columns) == 0 { + return 0 + } else { + // All columns have the same number of rows. + return buf.columns[0].Len() + } +} + +// Less returns true if row[i] < row[j] in the buffer. +func (buf *Buffer) Less(i, j int) bool { + for _, col := range buf.sorted { + switch { + case col.Less(i, j): + return true + case col.Less(j, i): + return false + } + } + return false +} + +// Swap exchanges the rows at indexes i and j. +func (buf *Buffer) Swap(i, j int) { + for _, col := range buf.columns { + col.Swap(i, j) + } +} + +// Reset clears the content of the buffer, allowing it to be reused. +func (buf *Buffer) Reset() { + for _, col := range buf.columns { + col.Reset() + } +} + +// Write writes a row held in a Go value to the buffer. +func (buf *Buffer) Write(row any) error { + if buf.schema == nil { + buf.configure(SchemaOf(row)) + } + + buf.rowbuf = buf.rowbuf[:1] + defer clearRows(buf.rowbuf) + + buf.rowbuf[0] = buf.schema.Deconstruct(buf.rowbuf[0], row) + _, err := buf.WriteRows(buf.rowbuf) + return err +} + +// WriteRows writes parquet rows to the buffer. +func (buf *Buffer) WriteRows(rows []Row) (int, error) { + defer func() { + for i, colbuf := range buf.colbuf { + clearValues(colbuf) + buf.colbuf[i] = colbuf[:0] + } + }() + + if buf.schema == nil { + return 0, ErrRowGroupSchemaMissing + } + + for _, row := range rows { + for _, value := range row { + columnIndex := value.Column() + buf.colbuf[columnIndex] = append(buf.colbuf[columnIndex], value) + } + } + + for columnIndex, values := range buf.colbuf { + if _, err := buf.columns[columnIndex].WriteValues(values); err != nil { + // TODO: an error at this stage will leave the buffer in an invalid + // state since the row was partially written. Applications are not + // expected to continue using the buffer after getting an error, + // maybe we can enforce it? + return 0, err + } + } + + return len(rows), nil +} + +// WriteRowGroup satisfies the RowGroupWriter interface. +func (buf *Buffer) WriteRowGroup(rowGroup RowGroup) (int64, error) { + rowGroupSchema := rowGroup.Schema() + switch { + case rowGroupSchema == nil: + return 0, ErrRowGroupSchemaMissing + case buf.schema == nil: + buf.configure(rowGroupSchema) + case !EqualNodes(buf.schema, rowGroupSchema): + return 0, ErrRowGroupSchemaMismatch + } + if !sortingColumnsHavePrefix(rowGroup.SortingColumns(), buf.SortingColumns()) { + return 0, ErrRowGroupSortingColumnsMismatch + } + n := buf.NumRows() + r := rowGroup.Rows() + defer r.Close() + _, err := CopyRows(bufferWriter{buf}, r) + return buf.NumRows() - n, err +} + +// Rows returns a reader exposing the current content of the buffer. +// +// The buffer and the returned reader share memory. Mutating the buffer +// concurrently to reading rows may result in non-deterministic behavior. +func (buf *Buffer) Rows() Rows { return NewRowGroupRowReader(buf) } + +// bufferWriter is an adapter for Buffer which implements both RowWriter and +// PageWriter to enable optimizations in CopyRows for types that support writing +// rows by copying whole pages instead of calling WriteRow repeatedly. +type bufferWriter struct{ buf *Buffer } + +func (w bufferWriter) WriteRows(rows []Row) (int, error) { + return w.buf.WriteRows(rows) +} + +func (w bufferWriter) WriteValues(values []Value) (int, error) { + return w.buf.columns[values[0].Column()].WriteValues(values) +} + +func (w bufferWriter) WritePage(page Page) (int64, error) { + return CopyValues(w.buf.columns[page.Column()], page.Values()) +} + +var ( + _ RowGroup = (*Buffer)(nil) + _ RowGroupWriter = (*Buffer)(nil) + _ sort.Interface = (*Buffer)(nil) + + _ RowWriter = (*bufferWriter)(nil) + _ PageWriter = (*bufferWriter)(nil) + _ ValueWriter = (*bufferWriter)(nil) +) + +type buffer struct { + data []byte + refc uintptr + pool *bufferPool + stack []byte +} + +func (b *buffer) refCount() int { + return int(atomic.LoadUintptr(&b.refc)) +} + +func (b *buffer) ref() { + atomic.AddUintptr(&b.refc, +1) +} + +func (b *buffer) unref() { + if atomic.AddUintptr(&b.refc, ^uintptr(0)) == 0 { + if b.pool != nil { + b.pool.put(b) + } + } +} + +func monitorBufferRelease(b *buffer) { + if rc := b.refCount(); rc != 0 { + log.Printf("PARQUETGODEBUG: buffer garbage collected with non-zero reference count\n%s", string(b.stack)) + } +} + +type bufferPool struct { + // Buckets are split in two groups for short and large buffers. In the short + // buffer group (below 256KB), the growth rate between each bucket is 2. The + // growth rate changes to 1.5 in the larger buffer group. + // + // Short buffer buckets: + // --------------------- + // 4K, 8K, 16K, 32K, 64K, 128K, 256K + // + // Large buffer buckets: + // --------------------- + // 364K, 546K, 819K ... + // + buckets [bufferPoolBucketCount]sync.Pool +} + +func (p *bufferPool) newBuffer(bufferSize, bucketSize int) *buffer { + b := &buffer{ + data: make([]byte, bufferSize, bucketSize), + refc: 1, + pool: p, + } + if debug.TRACEBUF > 0 { + b.stack = make([]byte, 4096) + runtime.SetFinalizer(b, monitorBufferRelease) + } + return b +} + +// get returns a buffer from the levelled buffer pool. size is used to choose +// the appropriate pool. +func (p *bufferPool) get(bufferSize int) *buffer { + bucketIndex, bucketSize := bufferPoolBucketIndexAndSizeOfGet(bufferSize) + + b := (*buffer)(nil) + if bucketIndex >= 0 { + b, _ = p.buckets[bucketIndex].Get().(*buffer) + } + + if b == nil { + b = p.newBuffer(bufferSize, bucketSize) + } else { + b.data = b.data[:bufferSize] + b.ref() + } + + if debug.TRACEBUF > 0 { + b.stack = b.stack[:runtime.Stack(b.stack[:cap(b.stack)], false)] + } + return b +} + +func (p *bufferPool) put(b *buffer) { + if b.pool != p { + panic("BUG: buffer returned to a different pool than the one it was allocated from") + } + if b.refCount() != 0 { + panic("BUG: buffer returned to pool with a non-zero reference count") + } + if bucketIndex, _ := bufferPoolBucketIndexAndSizeOfPut(cap(b.data)); bucketIndex >= 0 { + p.buckets[bucketIndex].Put(b) + } +} + +const ( + bufferPoolBucketCount = 32 + bufferPoolMinSize = 4096 + bufferPoolLastShortBucketSize = 262144 +) + +func bufferPoolNextSize(size int) int { + if size < bufferPoolLastShortBucketSize { + return size * 2 + } else { + return size + (size / 2) + } +} + +func bufferPoolBucketIndexAndSizeOfGet(size int) (int, int) { + limit := bufferPoolMinSize + + for i := range bufferPoolBucketCount { + if size <= limit { + return i, limit + } + limit = bufferPoolNextSize(limit) + } + + return -1, size +} + +func bufferPoolBucketIndexAndSizeOfPut(size int) (int, int) { + // When releasing buffers, some may have a capacity that is not one of the + // bucket sizes (due to the use of append for example). In this case, we + // have to put the buffer is the highest bucket with a size less or equal + // to the buffer capacity. + if limit := bufferPoolMinSize; size >= limit { + for i := range bufferPoolBucketCount { + n := bufferPoolNextSize(limit) + if size < n { + return i, limit + } + limit = n + } + } + return -1, size +} + +var ( + buffers bufferPool +) + +type bufferedPage struct { + Page + values *buffer + offsets *buffer + repetitionLevels *buffer + definitionLevels *buffer +} + +func newBufferedPage(page Page, values, offsets, definitionLevels, repetitionLevels *buffer) *bufferedPage { + p := &bufferedPage{ + Page: page, + values: values, + offsets: offsets, + definitionLevels: definitionLevels, + repetitionLevels: repetitionLevels, + } + bufferRef(values) + bufferRef(offsets) + bufferRef(definitionLevels) + bufferRef(repetitionLevels) + return p +} + +func (p *bufferedPage) Slice(i, j int64) Page { + return newBufferedPage( + p.Page.Slice(i, j), + p.values, + p.offsets, + p.definitionLevels, + p.repetitionLevels, + ) +} + +func (p *bufferedPage) Retain() { + bufferRef(p.values) + bufferRef(p.offsets) + bufferRef(p.definitionLevels) + bufferRef(p.repetitionLevels) +} + +func (p *bufferedPage) Release() { + bufferUnref(p.values) + bufferUnref(p.offsets) + bufferUnref(p.definitionLevels) + bufferUnref(p.repetitionLevels) +} + +func bufferRef(buf *buffer) { + if buf != nil { + buf.ref() + } +} + +func bufferUnref(buf *buffer) { + if buf != nil { + buf.unref() + } +} + +// Retain is a helper function to increment the reference counter of pages +// backed by memory which can be granularly managed by the application. +// +// Usage of this function is optional and with Release, is intended to allow +// finer grain memory management in the application. Most programs should be +// able to rely on automated memory management provided by the Go garbage +// collector instead. +// +// The function should be called when a page lifetime is about to be shared +// between multiple goroutines or layers of an application, and the program +// wants to express "sharing ownership" of the page. +// +// Calling this function on pages that do not embed a reference counter does +// nothing. +func Retain(page Page) { + if p, _ := page.(retainable); p != nil { + p.Retain() + } +} + +// Release is a helper function to decrement the reference counter of pages +// backed by memory which can be granularly managed by the application. +// +// Usage of this is optional and with Retain, is intended to allow finer grained +// memory management in the application, at the expense of potentially causing +// panics if the page is used after its reference count has reached zero. Most +// programs should be able to rely on automated memory management provided by +// the Go garbage collector instead. +// +// The function should be called to return a page to the internal buffer pool, +// when a goroutine "releases ownership" it acquired either by being the single +// owner (e.g. capturing the return value from a ReadPage call) or having gotten +// shared ownership by calling Retain. +// +// Calling this function on pages that do not embed a reference counter does +// nothing. +func Release(page Page) { + if p, _ := page.(releasable); p != nil { + p.Release() + } +} + +type retainable interface { + Retain() +} + +type releasable interface { + Release() +} + +var ( + _ retainable = (*bufferedPage)(nil) + _ releasable = (*bufferedPage)(nil) +) diff --git a/vendor/github.com/parquet-go/parquet-go/buffer_pool.go b/vendor/github.com/parquet-go/parquet-go/buffer_pool.go new file mode 100644 index 0000000000..2dfc158be8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/buffer_pool.go @@ -0,0 +1,377 @@ +package parquet + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// BufferPool is an interface abstracting the underlying implementation of +// page buffer pools. +// +// The parquet-go package provides two implementations of this interface, one +// backed by in-memory buffers (on the Go heap), and the other using temporary +// files on disk. +// +// Applications which need finer grain control over the allocation and retention +// of page buffers may choose to provide their own implementation and install it +// via the parquet.ColumnPageBuffers writer option. +// +// BufferPool implementations must be safe to use concurrently from multiple +// goroutines. +type BufferPool interface { + // GetBuffer is called when a parquet writer needs to acquire a new + // page buffer from the pool. + GetBuffer() io.ReadWriteSeeker + + // PutBuffer is called when a parquet writer releases a page buffer to + // the pool. + // + // The parquet.Writer type guarantees that the buffers it calls this method + // with were previously acquired by a call to GetBuffer on the same + // pool, and that it will not use them anymore after the call. + PutBuffer(io.ReadWriteSeeker) +} + +// NewBufferPool creates a new in-memory page buffer pool. +// +// The implementation is backed by sync.Pool and allocates memory buffers on the +// Go heap. +func NewBufferPool() BufferPool { return new(memoryBufferPool) } + +type memoryBuffer struct { + data []byte + off int +} + +func (p *memoryBuffer) Reset() { + p.data, p.off = p.data[:0], 0 +} + +func (p *memoryBuffer) Read(b []byte) (n int, err error) { + n = copy(b, p.data[p.off:]) + p.off += n + if p.off == len(p.data) { + err = io.EOF + } + return n, err +} + +func (p *memoryBuffer) Write(b []byte) (int, error) { + n := copy(p.data[p.off:cap(p.data)], b) + p.data = p.data[:p.off+n] + + if n < len(b) { + p.data = append(p.data, b[n:]...) + } + + p.off += len(b) + return len(b), nil +} + +func (p *memoryBuffer) WriteTo(w io.Writer) (int64, error) { + n, err := w.Write(p.data[p.off:]) + p.off += n + return int64(n), err +} + +func (p *memoryBuffer) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekCurrent: + offset += int64(p.off) + case io.SeekEnd: + offset += int64(len(p.data)) + } + if offset < 0 { + return 0, fmt.Errorf("seek: negative offset: %d<0", offset) + } + if offset > int64(len(p.data)) { + offset = int64(len(p.data)) + } + p.off = int(offset) + return offset, nil +} + +type memoryBufferPool struct{ sync.Pool } + +func (pool *memoryBufferPool) GetBuffer() io.ReadWriteSeeker { + b, _ := pool.Get().(*memoryBuffer) + if b == nil { + b = new(memoryBuffer) + } else { + b.Reset() + } + return b +} + +func (pool *memoryBufferPool) PutBuffer(buf io.ReadWriteSeeker) { + if b, _ := buf.(*memoryBuffer); b != nil { + pool.Put(b) + } +} + +// NewChunkBufferPool creates a new in-memory page buffer pool. +// +// The implementation is backed by sync.Pool and allocates memory buffers on the +// Go heap in fixed-size chunks. +func NewChunkBufferPool(chunkSize int) BufferPool { + return newChunkMemoryBufferPool(chunkSize) +} + +func newChunkMemoryBufferPool(chunkSize int) *chunkMemoryBufferPool { + pool := &chunkMemoryBufferPool{} + pool.bytesPool.New = func() any { + return make([]byte, chunkSize) + } + return pool +} + +// chunkMemoryBuffer implements an io.ReadWriteSeeker by storing a slice of fixed-size +// buffers into which it copies data. (It uses a sync.Pool to reuse buffers across +// instances.) +type chunkMemoryBuffer struct { + bytesPool *sync.Pool + + data [][]byte + idx int + off int +} + +func (c *chunkMemoryBuffer) Reset() { + for i := range c.data { + c.bytesPool.Put(c.data[i]) + } + for i := range c.data { + c.data[i] = nil + } + c.data, c.idx, c.off = c.data[:0], 0, 0 +} + +func (c *chunkMemoryBuffer) Read(b []byte) (n int, err error) { + if len(b) == 0 { + return 0, nil + } + + if c.idx >= len(c.data) { + return 0, io.EOF + } + + curData := c.data[c.idx] + + if c.idx == len(c.data)-1 && c.off == len(curData) { + return 0, io.EOF + } + + n = copy(b, curData[c.off:]) + c.off += n + + if c.off == cap(curData) { + c.idx++ + c.off = 0 + } + + return n, err +} + +func (c *chunkMemoryBuffer) Write(b []byte) (int, error) { + lenB := len(b) + + if lenB == 0 { + return 0, nil + } + + for len(b) > 0 { + if c.idx == len(c.data) { + c.data = append(c.data, c.bytesPool.Get().([]byte)[:0]) + } + curData := c.data[c.idx] + n := copy(curData[c.off:cap(curData)], b) + c.data[c.idx] = curData[:c.off+n] + c.off += n + b = b[n:] + if c.off >= cap(curData) { + c.idx++ + c.off = 0 + } + } + + return lenB, nil +} + +func (c *chunkMemoryBuffer) WriteTo(w io.Writer) (int64, error) { + var numWritten int64 + var err error + for err == nil { + curData := c.data[c.idx] + n, e := w.Write(curData[c.off:]) + numWritten += int64(n) + err = e + if c.idx == len(c.data)-1 { + c.off = int(numWritten) + break + } + c.idx++ + c.off = 0 + } + return numWritten, err +} + +func (c *chunkMemoryBuffer) Seek(offset int64, whence int) (int64, error) { + // Because this is the common case, we check it first to avoid computing endOff. + if offset == 0 && whence == io.SeekStart { + c.idx = 0 + c.off = 0 + return offset, nil + } + endOff := c.endOff() + switch whence { + case io.SeekCurrent: + offset += c.currentOff() + case io.SeekEnd: + offset += endOff + } + if offset < 0 { + return 0, fmt.Errorf("seek: negative offset: %d<0", offset) + } + if offset > endOff { + offset = endOff + } + // Repeat this case now that we know the absolute offset. This is a bit faster, but + // mainly protects us from an out-of-bounds if c.data is empty. (If the buffer is + // empty and the absolute offset isn't zero, we'd have errored (if negative) or + // clamped to zero (if positive) above. + if offset == 0 { + c.idx = 0 + c.off = 0 + } else { + stride := cap(c.data[0]) + c.idx = int(offset) / stride + c.off = int(offset) % stride + } + return offset, nil +} + +func (c *chunkMemoryBuffer) currentOff() int64 { + if c.idx == 0 { + return int64(c.off) + } + return int64((c.idx-1)*cap(c.data[0]) + c.off) +} + +func (c *chunkMemoryBuffer) endOff() int64 { + if len(c.data) == 0 { + return 0 + } + l := len(c.data) + last := c.data[l-1] + return int64(cap(last)*(l-1) + len(last)) +} + +type chunkMemoryBufferPool struct { + sync.Pool + bytesPool sync.Pool +} + +func (pool *chunkMemoryBufferPool) GetBuffer() io.ReadWriteSeeker { + b, _ := pool.Get().(*chunkMemoryBuffer) + if b == nil { + b = &chunkMemoryBuffer{bytesPool: &pool.bytesPool} + } else { + b.Reset() + } + return b +} + +func (pool *chunkMemoryBufferPool) PutBuffer(buf io.ReadWriteSeeker) { + if b, _ := buf.(*chunkMemoryBuffer); b != nil { + for _, bytes := range b.data { + b.bytesPool.Put(bytes) + } + for i := range b.data { + b.data[i] = nil + } + b.data = b.data[:0] + pool.Put(b) + } +} + +type fileBufferPool struct { + err error + tempdir string + pattern string +} + +// NewFileBufferPool creates a new on-disk page buffer pool. +func NewFileBufferPool(tempdir, pattern string) BufferPool { + pool := &fileBufferPool{ + tempdir: tempdir, + pattern: pattern, + } + pool.tempdir, pool.err = filepath.Abs(pool.tempdir) + return pool +} + +func (pool *fileBufferPool) GetBuffer() io.ReadWriteSeeker { + if pool.err != nil { + return &errorBuffer{err: pool.err} + } + f, err := os.CreateTemp(pool.tempdir, pool.pattern) + if err != nil { + return &errorBuffer{err: err} + } + return f +} + +func (pool *fileBufferPool) PutBuffer(buf io.ReadWriteSeeker) { + if f, _ := buf.(*os.File); f != nil { + defer f.Close() + os.Remove(f.Name()) + } +} + +type errorBuffer struct{ err error } + +func (buf *errorBuffer) Read([]byte) (int, error) { return 0, buf.err } +func (buf *errorBuffer) Write([]byte) (int, error) { return 0, buf.err } +func (buf *errorBuffer) ReadFrom(io.Reader) (int64, error) { return 0, buf.err } +func (buf *errorBuffer) WriteTo(io.Writer) (int64, error) { return 0, buf.err } +func (buf *errorBuffer) Seek(int64, int) (int64, error) { return 0, buf.err } + +var ( + defaultColumnBufferPool = *newChunkMemoryBufferPool(256 * 1024) + defaultSortingBufferPool memoryBufferPool + + _ io.ReaderFrom = (*errorBuffer)(nil) + _ io.WriterTo = (*errorBuffer)(nil) + _ io.ReadWriteSeeker = (*memoryBuffer)(nil) + _ io.WriterTo = (*memoryBuffer)(nil) + _ io.ReadWriteSeeker = (*chunkMemoryBuffer)(nil) + _ io.WriterTo = (*chunkMemoryBuffer)(nil) +) + +type readerAt struct { + reader io.ReadSeeker + offset int64 +} + +func (r *readerAt) ReadAt(b []byte, off int64) (int, error) { + if r.offset < 0 || off != r.offset { + off, err := r.reader.Seek(off, io.SeekStart) + if err != nil { + return 0, err + } + r.offset = off + } + n, err := r.reader.Read(b) + r.offset += int64(n) + return n, err +} + +func newReaderAt(r io.ReadSeeker) io.ReaderAt { + if rr, ok := r.(io.ReaderAt); ok { + return rr + } + return &readerAt{reader: r, offset: -1} +} diff --git a/vendor/github.com/parquet-go/parquet-go/column.go b/vendor/github.com/parquet-go/parquet-go/column.go new file mode 100644 index 0000000000..52790c43b2 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column.go @@ -0,0 +1,818 @@ +package parquet + +import ( + "encoding/binary" + "fmt" + "io" + "reflect" + + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// Column represents a column in a parquet file. +// +// Methods of Column values are safe to call concurrently from multiple +// goroutines. +// +// Column instances satisfy the Node interface. +type Column struct { + typ Type + file *File + schema *format.SchemaElement + order *format.ColumnOrder + path columnPath + fields []Field + columns []*Column + chunks []*format.ColumnChunk + columnIndex []*format.ColumnIndex + offsetIndex []*format.OffsetIndex + encoding encoding.Encoding + compression compress.Codec + + depth int8 + maxRepetitionLevel byte + maxDefinitionLevel byte + index int16 +} + +// Type returns the type of the column. +// +// The returned value is unspecified if c is not a leaf column. +func (c *Column) Type() Type { return c.typ } + +// Optional returns true if the column is optional. +func (c *Column) Optional() bool { return schemaRepetitionTypeOf(c.schema) == format.Optional } + +// Repeated returns true if the column may repeat. +func (c *Column) Repeated() bool { return schemaRepetitionTypeOf(c.schema) == format.Repeated } + +// Required returns true if the column is required. +func (c *Column) Required() bool { return schemaRepetitionTypeOf(c.schema) == format.Required } + +// Leaf returns true if c is a leaf column. +func (c *Column) Leaf() bool { return c.index >= 0 } + +// Fields returns the list of fields on the column. +func (c *Column) Fields() []Field { return c.fields } + +// Encoding returns the encodings used by this column. +func (c *Column) Encoding() encoding.Encoding { return c.encoding } + +// Compression returns the compression codecs used by this column. +func (c *Column) Compression() compress.Codec { return c.compression } + +// Path of the column in the parquet schema. +func (c *Column) Path() []string { return c.path[1:] } + +// Name returns the column name. +func (c *Column) Name() string { return c.schema.Name } + +// ID returns column field id +func (c *Column) ID() int { return int(c.schema.FieldID) } + +// Columns returns the list of child columns. +// +// The method returns the same slice across multiple calls, the program must +// treat it as a read-only value. +func (c *Column) Columns() []*Column { return c.columns } + +// Column returns the child column matching the given name. +func (c *Column) Column(name string) *Column { + for _, child := range c.columns { + if child.Name() == name { + return child + } + } + return nil +} + +// Pages returns a reader exposing all pages in this column, across row groups. +func (c *Column) Pages() Pages { + if c.file == nil { + return emptyPages{} + } + return c.PagesFrom(c.file.reader) +} + +func (c *Column) PagesFrom(reader io.ReaderAt) Pages { + if c.index < 0 || c.file == nil { + return emptyPages{} + } + r := &columnPages{ + pages: make([]FilePages, len(c.file.rowGroups)), + } + for i := range r.pages { + r.pages[i].init(c.file.rowGroups[i].(*FileRowGroup).columns[c.index].(*FileColumnChunk), c.file.config.ReadBufferSize, reader) + } + return r +} + +type columnPages struct { + pages []FilePages + index int +} + +func (c *columnPages) ReadPage() (Page, error) { + for { + if c.index >= len(c.pages) { + return nil, io.EOF + } + p, err := c.pages[c.index].ReadPage() + if err == nil || err != io.EOF { + return p, err + } + c.index++ + } +} + +func (c *columnPages) SeekToRow(rowIndex int64) error { + c.index = 0 + + for c.index < len(c.pages) && c.pages[c.index].chunk.rowGroup.NumRows < rowIndex { + rowIndex -= c.pages[c.index].chunk.rowGroup.NumRows + c.index++ + } + + if c.index < len(c.pages) { + if err := c.pages[c.index].SeekToRow(rowIndex); err != nil { + return err + } + for i := c.index + 1; i < len(c.pages); i++ { + p := &c.pages[i] + if err := p.SeekToRow(0); err != nil { + return err + } + } + } + return nil +} + +func (c *columnPages) Close() error { + var lastErr error + + for i := range c.pages { + if err := c.pages[i].Close(); err != nil { + lastErr = err + } + } + + c.pages = nil + c.index = 0 + return lastErr +} + +// Depth returns the position of the column relative to the root. +func (c *Column) Depth() int { return int(c.depth) } + +// MaxRepetitionLevel returns the maximum value of repetition levels on this +// column. +func (c *Column) MaxRepetitionLevel() int { return int(c.maxRepetitionLevel) } + +// MaxDefinitionLevel returns the maximum value of definition levels on this +// column. +func (c *Column) MaxDefinitionLevel() int { return int(c.maxDefinitionLevel) } + +// Index returns the position of the column in a row. Only leaf columns have a +// column index, the method returns -1 when called on non-leaf columns. +func (c *Column) Index() int { return int(c.index) } + +// GoType returns the Go type that best represents the parquet column. +func (c *Column) GoType() reflect.Type { return goTypeOf(c) } + +// Value returns the sub-value in base for the child column at the given +// index. +func (c *Column) Value(base reflect.Value) reflect.Value { + return base.MapIndex(reflect.ValueOf(&c.schema.Name).Elem()) +} + +// String returns a human-readable string representation of the column. +func (c *Column) String() string { return c.path.String() + ": " + sprint(c.Name(), c) } + +func (c *Column) forEachLeaf(do func(*Column)) { + if len(c.columns) == 0 { + do(c) + } else { + for _, child := range c.columns { + child.forEachLeaf(do) + } + } +} + +func openColumns(file *File, metadata *format.FileMetaData, columnIndexes []format.ColumnIndex, offsetIndexes []format.OffsetIndex) (*Column, error) { + cl := columnLoader{} + + c, err := cl.open(file, metadata, columnIndexes, offsetIndexes, nil) + if err != nil { + return nil, err + } + + // Validate that there aren't extra entries in the row group columns, + // which would otherwise indicate that there are dangling data pages + // in the file. + for index, rowGroup := range metadata.RowGroups { + if cl.rowGroupColumnIndex != len(rowGroup.Columns) { + return nil, fmt.Errorf("row group at index %d contains %d columns but %d were referenced by the column schemas", + index, len(rowGroup.Columns), cl.rowGroupColumnIndex) + } + } + + _, err = c.setLevels(0, 0, 0, 0) + return c, err +} + +func (c *Column) setLevels(depth, repetition, definition, index int) (int, error) { + if depth > MaxColumnDepth { + return -1, fmt.Errorf("cannot represent parquet columns with more than %d nested levels: %s", MaxColumnDepth, c.path) + } + if index > MaxColumnIndex { + return -1, fmt.Errorf("cannot represent parquet rows with more than %d columns: %s", MaxColumnIndex, c.path) + } + if repetition > MaxRepetitionLevel { + return -1, fmt.Errorf("cannot represent parquet columns with more than %d repetition levels: %s", MaxRepetitionLevel, c.path) + } + if definition > MaxDefinitionLevel { + return -1, fmt.Errorf("cannot represent parquet columns with more than %d definition levels: %s", MaxDefinitionLevel, c.path) + } + + switch schemaRepetitionTypeOf(c.schema) { + case format.Optional: + definition++ + case format.Repeated: + repetition++ + definition++ + } + + c.depth = int8(depth) + c.maxRepetitionLevel = byte(repetition) + c.maxDefinitionLevel = byte(definition) + depth++ + + if len(c.columns) > 0 { + c.index = -1 + } else { + c.index = int16(index) + index++ + } + + var err error + for _, child := range c.columns { + if index, err = child.setLevels(depth, repetition, definition, index); err != nil { + return -1, err + } + } + return index, nil +} + +type columnLoader struct { + schemaIndex int + columnOrderIndex int + rowGroupColumnIndex int +} + +func (cl *columnLoader) open(file *File, metadata *format.FileMetaData, columnIndexes []format.ColumnIndex, offsetIndexes []format.OffsetIndex, path []string) (*Column, error) { + c := &Column{ + file: file, + schema: &metadata.Schema[cl.schemaIndex], + } + c.path = columnPath(path).append(c.schema.Name) + + cl.schemaIndex++ + numChildren := int(c.schema.NumChildren) + + if numChildren == 0 { + c.typ = schemaElementTypeOf(c.schema) + + if cl.columnOrderIndex < len(metadata.ColumnOrders) { + c.order = &metadata.ColumnOrders[cl.columnOrderIndex] + cl.columnOrderIndex++ + } + + rowGroups := metadata.RowGroups + rowGroupColumnIndex := cl.rowGroupColumnIndex + cl.rowGroupColumnIndex++ + + c.chunks = make([]*format.ColumnChunk, 0, len(rowGroups)) + c.columnIndex = make([]*format.ColumnIndex, 0, len(rowGroups)) + c.offsetIndex = make([]*format.OffsetIndex, 0, len(rowGroups)) + + for i, rowGroup := range rowGroups { + if rowGroupColumnIndex >= len(rowGroup.Columns) { + return nil, fmt.Errorf("row group at index %d does not have enough columns", i) + } + c.chunks = append(c.chunks, &rowGroup.Columns[rowGroupColumnIndex]) + } + + if len(columnIndexes) > 0 { + for i := range rowGroups { + if rowGroupColumnIndex >= len(columnIndexes) { + return nil, fmt.Errorf("row group at index %d does not have enough column index pages", i) + } + c.columnIndex = append(c.columnIndex, &columnIndexes[rowGroupColumnIndex]) + } + } + + if len(offsetIndexes) > 0 { + for i := range rowGroups { + if rowGroupColumnIndex >= len(offsetIndexes) { + return nil, fmt.Errorf("row group at index %d does not have enough offset index pages", i) + } + c.offsetIndex = append(c.offsetIndex, &offsetIndexes[rowGroupColumnIndex]) + } + } + + if len(c.chunks) > 0 { + // Pick the encoding and compression codec of the first chunk. + // + // Technically each column chunk may use a different compression + // codec, and each page of the column chunk might have a different + // encoding. Exposing these details does not provide a lot of value + // to the end user. + // + // Programs that wish to determine the encoding and compression of + // each page of the column should iterate through the pages and read + // the page headers to determine which compression and encodings are + // applied. + for _, encoding := range c.chunks[0].MetaData.Encoding { + if c.encoding == nil { + c.encoding = LookupEncoding(encoding) + } + if encoding != format.Plain && encoding != format.RLE { + c.encoding = LookupEncoding(encoding) + break + } + } + c.compression = LookupCompressionCodec(c.chunks[0].MetaData.Codec) + } + + return c, nil + } + + c.typ = &groupType{} + if lt := c.schema.LogicalType; lt != nil && lt.Map != nil { + c.typ = &mapType{} + } else if lt != nil && lt.List != nil { + c.typ = &listType{} + } else if lt != nil && lt.Variant != nil { + c.typ = &variantType{} + } + c.columns = make([]*Column, numChildren) + + for i := range c.columns { + if cl.schemaIndex >= len(metadata.Schema) { + return nil, fmt.Errorf("column %q has more children than there are schemas in the file: %d > %d", + c.schema.Name, cl.schemaIndex+1, len(metadata.Schema)) + } + + var err error + c.columns[i], err = cl.open(file, metadata, columnIndexes, offsetIndexes, c.path) + if err != nil { + return nil, fmt.Errorf("%s: %w", c.schema.Name, err) + } + } + + c.fields = make([]Field, len(c.columns)) + for i, column := range c.columns { + c.fields[i] = column + } + return c, nil +} + +func schemaElementTypeOf(s *format.SchemaElement) Type { + if lt := s.LogicalType; lt != nil { + // A logical type exists, the Type interface implementations in this + // package are all based on the logical parquet types declared in the + // format sub-package so we can return them directly via a pointer type + // conversion. + switch { + case lt.UTF8 != nil: + return (*stringType)(lt.UTF8) + case lt.Map != nil: + return (*mapType)(lt.Map) + case lt.List != nil: + return (*listType)(lt.List) + case lt.Enum != nil: + return (*enumType)(lt.Enum) + case lt.Decimal != nil: + // A parquet decimal can be one of several different physical types. + if t := s.Type; t != nil { + var typ Type + switch kind := Kind(*s.Type); kind { + case Int32: + typ = Int32Type + case Int64: + typ = Int64Type + case FixedLenByteArray: + if s.TypeLength == nil { + panic("DECIMAL using FIXED_LEN_BYTE_ARRAY must specify a length") + } + typ = FixedLenByteArrayType(int(*s.TypeLength)) + default: + panic("DECIMAL must be of type INT32, INT64, or FIXED_LEN_BYTE_ARRAY but got " + kind.String()) + } + return &decimalType{ + decimal: *lt.Decimal, + Type: typ, + } + } + case lt.Date != nil: + return (*dateType)(lt.Date) + case lt.Time != nil: + return (*timeType)(lt.Time) + case lt.Timestamp != nil: + return (*timestampType)(lt.Timestamp) + case lt.Integer != nil: + return (*intType)(lt.Integer) + case lt.Unknown != nil: + return (*nullType)(lt.Unknown) + case lt.Json != nil: + return (*jsonType)(lt.Json) + case lt.Bson != nil: + return (*bsonType)(lt.Bson) + case lt.UUID != nil: + return (*uuidType)(lt.UUID) + } + } + + if ct := s.ConvertedType; ct != nil { + // This column contains no logical type but has a converted type, it + // was likely created by an older parquet writer. Convert the legacy + // type representation to the equivalent logical parquet type. + switch *ct { + case deprecated.UTF8: + return &stringType{} + case deprecated.Map: + return &mapType{} + case deprecated.MapKeyValue: + return &groupType{} + case deprecated.List: + return &listType{} + case deprecated.Enum: + return &enumType{} + case deprecated.Decimal: + if s.Scale != nil && s.Precision != nil { + // A parquet decimal can be one of several different physical types. + if t := s.Type; t != nil { + var typ Type + switch kind := Kind(*s.Type); kind { + case Int32: + typ = Int32Type + case Int64: + typ = Int64Type + case FixedLenByteArray: + if s.TypeLength == nil { + panic("DECIMAL using FIXED_LEN_BYTE_ARRAY must specify a length") + } + typ = FixedLenByteArrayType(int(*s.TypeLength)) + case ByteArray: + typ = ByteArrayType + default: + panic("DECIMAL must be of type INT32, INT64, BYTE_ARRAY or FIXED_LEN_BYTE_ARRAY but got " + kind.String()) + } + return &decimalType{ + decimal: format.DecimalType{ + Scale: *s.Scale, + Precision: *s.Precision, + }, + Type: typ, + } + } + } + case deprecated.Date: + return &dateType{} + case deprecated.TimeMillis: + return &timeType{IsAdjustedToUTC: true, Unit: Millisecond.TimeUnit()} + case deprecated.TimeMicros: + return &timeType{IsAdjustedToUTC: true, Unit: Microsecond.TimeUnit()} + case deprecated.TimestampMillis: + return ×tampType{IsAdjustedToUTC: true, Unit: Millisecond.TimeUnit()} + case deprecated.TimestampMicros: + return ×tampType{IsAdjustedToUTC: true, Unit: Microsecond.TimeUnit()} + case deprecated.Uint8: + return &unsignedIntTypes[0] + case deprecated.Uint16: + return &unsignedIntTypes[1] + case deprecated.Uint32: + return &unsignedIntTypes[2] + case deprecated.Uint64: + return &unsignedIntTypes[3] + case deprecated.Int8: + return &signedIntTypes[0] + case deprecated.Int16: + return &signedIntTypes[1] + case deprecated.Int32: + return &signedIntTypes[2] + case deprecated.Int64: + return &signedIntTypes[3] + case deprecated.Json: + return &jsonType{} + case deprecated.Bson: + return &bsonType{} + case deprecated.Interval: + // TODO + } + } + + if t := s.Type; t != nil { + // The column only has a physical type, convert it to one of the + // primitive types supported by this package. + switch kind := Kind(*t); kind { + case Boolean: + return BooleanType + case Int32: + return Int32Type + case Int64: + return Int64Type + case Int96: + return Int96Type + case Float: + return FloatType + case Double: + return DoubleType + case ByteArray: + return ByteArrayType + case FixedLenByteArray: + if s.TypeLength != nil { + return FixedLenByteArrayType(int(*s.TypeLength)) + } + } + } + + // If we reach this point, we are likely reading a parquet column that was + // written with a non-standard type or is in a newer version of the format + // than this package supports. + return &nullType{} +} + +func schemaRepetitionTypeOf(s *format.SchemaElement) format.FieldRepetitionType { + if s.RepetitionType != nil { + return *s.RepetitionType + } + return format.Required +} + +func (c *Column) decompress(compressedPageData []byte, uncompressedPageSize int32) (page *buffer, err error) { + page = buffers.get(int(uncompressedPageSize)) + page.data, err = c.compression.Decode(page.data, compressedPageData) + if err != nil { + page.unref() + page = nil + } + return page, err +} + +// DecodeDataPageV1 decodes a data page from the header, compressed data, and +// optional dictionary passed as arguments. +func (c *Column) DecodeDataPageV1(header DataPageHeaderV1, page []byte, dict Dictionary) (Page, error) { + return c.decodeDataPageV1(header, &buffer{data: page}, dict, -1) +} + +func (c *Column) decodeDataPageV1(header DataPageHeaderV1, page *buffer, dict Dictionary, size int32) (Page, error) { + var pageData = page.data + var err error + + if isCompressed(c.compression) { + if page, err = c.decompress(pageData, size); err != nil { + return nil, fmt.Errorf("decompressing data page v1: %w", err) + } + defer page.unref() + pageData = page.data + } + + var numValues = int(header.NumValues()) + var repetitionLevels *buffer + var definitionLevels *buffer + + if c.maxRepetitionLevel > 0 { + encoding := lookupLevelEncoding(header.RepetitionLevelEncoding(), c.maxRepetitionLevel) + repetitionLevels, pageData, err = decodeLevelsV1(encoding, numValues, pageData) + if err != nil { + return nil, fmt.Errorf("decoding repetition levels of data page v1: %w", err) + } + defer repetitionLevels.unref() + } + + if c.maxDefinitionLevel > 0 { + encoding := lookupLevelEncoding(header.DefinitionLevelEncoding(), c.maxDefinitionLevel) + definitionLevels, pageData, err = decodeLevelsV1(encoding, numValues, pageData) + if err != nil { + return nil, fmt.Errorf("decoding definition levels of data page v1: %w", err) + } + defer definitionLevels.unref() + + // Data pages v1 did not embed the number of null values, + // so we have to compute it from the definition levels. + numValues -= countLevelsNotEqual(definitionLevels.data, c.maxDefinitionLevel) + } + + return c.decodeDataPage(header, numValues, repetitionLevels, definitionLevels, page, pageData, dict) +} + +// DecodeDataPageV2 decodes a data page from the header, compressed data, and +// optional dictionary passed as arguments. +func (c *Column) DecodeDataPageV2(header DataPageHeaderV2, page []byte, dict Dictionary) (Page, error) { + return c.decodeDataPageV2(header, &buffer{data: page}, dict, -1) +} + +func (c *Column) decodeDataPageV2(header DataPageHeaderV2, page *buffer, dict Dictionary, size int32) (Page, error) { + var numValues = int(header.NumValues()) + var pageData = page.data + var err error + var repetitionLevels *buffer + var definitionLevels *buffer + + if length := header.RepetitionLevelsByteLength(); length > 0 { + if c.maxRepetitionLevel == 0 { + // In some cases we've observed files which have a non-zero + // repetition level despite the column not being repeated + // (nor nested within a repeated column). + // + // See https://github.com/apache/parquet-testing/pull/24 + pageData, err = skipLevelsV2(pageData, length) + } else { + encoding := lookupLevelEncoding(header.RepetitionLevelEncoding(), c.maxRepetitionLevel) + repetitionLevels, pageData, err = decodeLevelsV2(encoding, numValues, pageData, length) + } + if err != nil { + return nil, fmt.Errorf("decoding repetition levels of data page v2: %w", io.ErrUnexpectedEOF) + } + if repetitionLevels != nil { + defer repetitionLevels.unref() + + if len(repetitionLevels.data) != 0 && repetitionLevels.data[0] != 0 { + return nil, fmt.Errorf("%w: first repetition level for column %d (%s) is %d instead of zero, indicating that the page contains trailing values from the previous page (this is forbidden for data pages v2)", + ErrMalformedRepetitionLevel, c.Index(), c.Name(), repetitionLevels.data[0]) + } + } + } + + if length := header.DefinitionLevelsByteLength(); length > 0 { + if c.maxDefinitionLevel == 0 { + pageData, err = skipLevelsV2(pageData, length) + } else { + encoding := lookupLevelEncoding(header.DefinitionLevelEncoding(), c.maxDefinitionLevel) + definitionLevels, pageData, err = decodeLevelsV2(encoding, numValues, pageData, length) + } + if err != nil { + return nil, fmt.Errorf("decoding definition levels of data page v2: %w", io.ErrUnexpectedEOF) + } + if definitionLevels != nil { + defer definitionLevels.unref() + } + } + + if isCompressed(c.compression) && header.IsCompressed() { + if page, err = c.decompress(pageData, size); err != nil { + return nil, fmt.Errorf("decompressing data page v2: %w", err) + } + defer page.unref() + pageData = page.data + } + + numValues -= int(header.NumNulls()) + return c.decodeDataPage(header, numValues, repetitionLevels, definitionLevels, page, pageData, dict) +} + +func (c *Column) decodeDataPage(header DataPageHeader, numValues int, repetitionLevels, definitionLevels, page *buffer, data []byte, dict Dictionary) (Page, error) { + pageEncoding := LookupEncoding(header.Encoding()) + pageType := c.Type() + + if isDictionaryEncoding(pageEncoding) { + // In some legacy configurations, the PLAIN_DICTIONARY encoding is used + // on data page headers to indicate that the page contains indexes into + // the dictionary page, but the page is still encoded using the RLE + // encoding in this case, so we convert it to RLE_DICTIONARY. + pageEncoding = &RLEDictionary + pageType = indexedPageType{newIndexedType(pageType, dict)} + } + + var vbuf, obuf *buffer + var pageValues []byte + var pageOffsets []uint32 + + if pageEncoding.CanDecodeInPlace() { + vbuf = page + pageValues = data + } else { + vbuf = buffers.get(pageType.EstimateDecodeSize(numValues, data, pageEncoding)) + defer vbuf.unref() + pageValues = vbuf.data + } + + // Page offsets not needed when dictionary-encoded + if pageType.Kind() == ByteArray && !isDictionaryEncoding(pageEncoding) { + obuf = buffers.get(4 * (numValues + 1)) + defer obuf.unref() + pageOffsets = unsafecast.Slice[uint32](obuf.data) + } + + values := pageType.NewValues(pageValues, pageOffsets) + values, err := pageType.Decode(values, data, pageEncoding) + if err != nil { + return nil, err + } + + newPage := pageType.NewPage(c.Index(), numValues, values) + switch { + case c.maxRepetitionLevel > 0: + newPage = newRepeatedPage( + newPage, + c.maxRepetitionLevel, + c.maxDefinitionLevel, + repetitionLevels.data, + definitionLevels.data, + ) + case c.maxDefinitionLevel > 0: + newPage = newOptionalPage( + newPage, + c.maxDefinitionLevel, + definitionLevels.data, + ) + } + + return newBufferedPage(newPage, vbuf, obuf, repetitionLevels, definitionLevels), nil +} + +func decodeLevelsV1(enc encoding.Encoding, numValues int, data []byte) (*buffer, []byte, error) { + if len(data) < 4 { + return nil, data, io.ErrUnexpectedEOF + } + i := 4 + j := 4 + int(binary.LittleEndian.Uint32(data)) + if j > len(data) { + return nil, data, io.ErrUnexpectedEOF + } + levels, err := decodeLevels(enc, numValues, data[i:j]) + return levels, data[j:], err +} + +func decodeLevelsV2(enc encoding.Encoding, numValues int, data []byte, length int64) (*buffer, []byte, error) { + levels, err := decodeLevels(enc, numValues, data[:length]) + return levels, data[length:], err +} + +func decodeLevels(enc encoding.Encoding, numValues int, data []byte) (levels *buffer, err error) { + levels = buffers.get(numValues) + levels.data, err = enc.DecodeLevels(levels.data, data) + if err != nil { + levels.unref() + levels = nil + } else { + switch { + case len(levels.data) < numValues: + err = fmt.Errorf("decoding level expected %d values but got only %d", numValues, len(levels.data)) + case len(levels.data) > numValues: + levels.data = levels.data[:numValues] + } + } + return levels, err +} + +func skipLevelsV2(data []byte, length int64) ([]byte, error) { + if length >= int64(len(data)) { + return data, io.ErrUnexpectedEOF + } + return data[length:], nil +} + +// DecodeDictionary decodes a data page from the header and compressed data +// passed as arguments. +func (c *Column) DecodeDictionary(header DictionaryPageHeader, page []byte) (Dictionary, error) { + return c.decodeDictionary(header, &buffer{data: page}, -1) +} + +func (c *Column) decodeDictionary(header DictionaryPageHeader, page *buffer, size int32) (Dictionary, error) { + pageData := page.data + + if isCompressed(c.compression) { + var err error + if page, err = c.decompress(pageData, size); err != nil { + return nil, fmt.Errorf("decompressing dictionary page: %w", err) + } + defer page.unref() + pageData = page.data + } + + pageType := c.Type() + pageEncoding := header.Encoding() + if pageEncoding == format.PlainDictionary { + pageEncoding = format.Plain + } + + // Dictionaries always have PLAIN encoding, so we need to allocate offsets for the decoded page. + numValues := int(header.NumValues()) + dictBufferSize := pageType.EstimateDecodeSize(numValues, pageData, LookupEncoding(pageEncoding)) + values := pageType.NewValues(make([]byte, 0, dictBufferSize), make([]uint32, 0, numValues)) + values, err := pageType.Decode(values, pageData, LookupEncoding(pageEncoding)) + if err != nil { + return nil, err + } + return pageType.NewDictionary(int(c.index), numValues, values), nil +} + +var ( + _ Node = (*Column)(nil) +) diff --git a/vendor/github.com/parquet-go/parquet-go/column_buffer.go b/vendor/github.com/parquet-go/parquet-go/column_buffer.go new file mode 100644 index 0000000000..6a54dcf0fc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_buffer.go @@ -0,0 +1,2472 @@ +package parquet + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math/bits" + "reflect" + "sort" + "time" + "unsafe" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding/plain" + "github.com/parquet-go/parquet-go/internal/bitpack" + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" + "slices" +) + +// ColumnBuffer is an interface representing columns of a row group. +// +// ColumnBuffer implements sort.Interface as a way to support reordering the +// rows that have been written to it. +// +// The current implementation has a limitation which prevents applications from +// providing custom versions of this interface because it contains unexported +// methods. The only way to create ColumnBuffer values is to call the +// NewColumnBuffer of Type instances. This limitation may be lifted in future +// releases. +type ColumnBuffer interface { + // Exposes a read-only view of the column buffer. + ColumnChunk + + // The column implements ValueReaderAt as a mechanism to read values at + // specific locations within the buffer. + ValueReaderAt + + // The column implements ValueWriter as a mechanism to optimize the copy + // of values into the buffer in contexts where the row information is + // provided by the values because the repetition and definition levels + // are set. + ValueWriter + + // For indexed columns, returns the underlying dictionary holding the column + // values. If the column is not indexed, nil is returned. + Dictionary() Dictionary + + // Returns a copy of the column. The returned copy shares no memory with + // the original, mutations of either column will not modify the other. + Clone() ColumnBuffer + + // Returns the column as a Page. + Page() Page + + // Clears all rows written to the column. + Reset() + + // Returns the current capacity of the column (rows). + Cap() int + + // Returns the number of rows currently written to the column. + Len() int + + // Compares rows at index i and j and reports whether i < j. + Less(i, j int) bool + + // Swaps rows at index i and j. + Swap(i, j int) + + // Returns the size of the column buffer in bytes. + Size() int64 + + // This method is employed to write rows from arrays of Go values into the + // column buffer. The method is currently unexported because it uses unsafe + // APIs which would be difficult for applications to leverage, increasing + // the risk of introducing bugs in the code. As a consequence, applications + // cannot use custom implementations of the ColumnBuffer interface since + // they cannot declare an unexported method that would match this signature. + // It means that in order to create a ColumnBuffer value, programs need to + // go through a call to NewColumnBuffer on a Type instance. We make this + // trade off for now as it is preferrable to optimize for safety over + // extensibility in the public APIs, we might revisit in the future if we + // learn about valid use cases for custom column buffer types. + writeValues(rows sparse.Array, levels columnLevels) +} + +type columnLevels struct { + repetitionDepth byte + repetitionLevel byte + definitionLevel byte +} + +func columnIndexOfNullable(base ColumnBuffer, maxDefinitionLevel byte, definitionLevels []byte) (ColumnIndex, error) { + index, err := base.ColumnIndex() + if err != nil { + return nil, err + } + return &nullableColumnIndex{ + ColumnIndex: index, + maxDefinitionLevel: maxDefinitionLevel, + definitionLevels: definitionLevels, + }, nil +} + +type nullableColumnIndex struct { + ColumnIndex + maxDefinitionLevel byte + definitionLevels []byte +} + +func (index *nullableColumnIndex) NullPage(i int) bool { + return index.NullCount(i) == int64(len(index.definitionLevels)) +} + +func (index *nullableColumnIndex) NullCount(i int) int64 { + return int64(countLevelsNotEqual(index.definitionLevels, index.maxDefinitionLevel)) +} + +type nullOrdering func(column ColumnBuffer, i, j int, maxDefinitionLevel, definitionLevel1, definitionLevel2 byte) bool + +func nullsGoFirst(column ColumnBuffer, i, j int, maxDefinitionLevel, definitionLevel1, definitionLevel2 byte) bool { + if definitionLevel1 != maxDefinitionLevel { + return definitionLevel2 == maxDefinitionLevel + } else { + return definitionLevel2 == maxDefinitionLevel && column.Less(i, j) + } +} + +func nullsGoLast(column ColumnBuffer, i, j int, maxDefinitionLevel, definitionLevel1, definitionLevel2 byte) bool { + return definitionLevel1 == maxDefinitionLevel && (definitionLevel2 != maxDefinitionLevel || column.Less(i, j)) +} + +// reversedColumnBuffer is an adapter of ColumnBuffer which inverses the order +// in which rows are ordered when the column gets sorted. +// +// This type is used when buffers are constructed with sorting columns ordering +// values in descending order. +type reversedColumnBuffer struct{ ColumnBuffer } + +func (col *reversedColumnBuffer) Less(i, j int) bool { return col.ColumnBuffer.Less(j, i) } + +// optionalColumnBuffer is an implementation of the ColumnBuffer interface used +// as a wrapper to an underlying ColumnBuffer to manage the creation of +// definition levels. +// +// Null values are not written to the underlying column; instead, the buffer +// tracks offsets of row values in the column, null row values are represented +// by the value -1 and a definition level less than the max. +// +// This column buffer type is used for all leaf columns that have a non-zero +// max definition level and a zero repetition level, which may be because the +// column or one of its parent(s) are marked optional. +type optionalColumnBuffer struct { + base ColumnBuffer + reordered bool + maxDefinitionLevel byte + rows []int32 + sortIndex []int32 + definitionLevels []byte + nullOrdering nullOrdering +} + +func newOptionalColumnBuffer(base ColumnBuffer, maxDefinitionLevel byte, nullOrdering nullOrdering) *optionalColumnBuffer { + n := base.Cap() + return &optionalColumnBuffer{ + base: base, + maxDefinitionLevel: maxDefinitionLevel, + rows: make([]int32, 0, n), + definitionLevels: make([]byte, 0, n), + nullOrdering: nullOrdering, + } +} + +func (col *optionalColumnBuffer) Clone() ColumnBuffer { + return &optionalColumnBuffer{ + base: col.base.Clone(), + reordered: col.reordered, + maxDefinitionLevel: col.maxDefinitionLevel, + rows: slices.Clone(col.rows), + definitionLevels: slices.Clone(col.definitionLevels), + nullOrdering: col.nullOrdering, + } +} + +func (col *optionalColumnBuffer) Type() Type { + return col.base.Type() +} + +func (col *optionalColumnBuffer) NumValues() int64 { + return int64(len(col.definitionLevels)) +} + +func (col *optionalColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return columnIndexOfNullable(col.base, col.maxDefinitionLevel, col.definitionLevels) +} + +func (col *optionalColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return col.base.OffsetIndex() +} + +func (col *optionalColumnBuffer) BloomFilter() BloomFilter { + return col.base.BloomFilter() +} + +func (col *optionalColumnBuffer) Dictionary() Dictionary { + return col.base.Dictionary() +} + +func (col *optionalColumnBuffer) Column() int { + return col.base.Column() +} + +func (col *optionalColumnBuffer) Pages() Pages { + return onePage(col.Page()) +} + +func (col *optionalColumnBuffer) Page() Page { + // No need for any cyclic sorting if the rows have not been reordered. + // This case is also important because the cyclic sorting modifies the + // buffer which makes it unsafe to read the buffer concurrently. + if col.reordered { + numNulls := countLevelsNotEqual(col.definitionLevels, col.maxDefinitionLevel) + numValues := len(col.rows) - numNulls + + if numValues > 0 { + if cap(col.sortIndex) < numValues { + col.sortIndex = make([]int32, numValues) + } + sortIndex := col.sortIndex[:numValues] + i := 0 + for _, j := range col.rows { + if j >= 0 { + sortIndex[j] = int32(i) + i++ + } + } + + // Cyclic sort: O(N) + for i := range sortIndex { + for j := int(sortIndex[i]); i != j; j = int(sortIndex[i]) { + col.base.Swap(i, j) + sortIndex[i], sortIndex[j] = sortIndex[j], sortIndex[i] + } + } + } + + i := 0 + for _, r := range col.rows { + if r >= 0 { + col.rows[i] = int32(i) + i++ + } + } + + col.reordered = false + } + + return newOptionalPage(col.base.Page(), col.maxDefinitionLevel, col.definitionLevels) +} + +func (col *optionalColumnBuffer) Reset() { + col.base.Reset() + col.rows = col.rows[:0] + col.definitionLevels = col.definitionLevels[:0] +} + +func (col *optionalColumnBuffer) Size() int64 { + return int64(4*len(col.rows)+4*len(col.sortIndex)+len(col.definitionLevels)) + col.base.Size() +} + +func (col *optionalColumnBuffer) Cap() int { return cap(col.rows) } + +func (col *optionalColumnBuffer) Len() int { return len(col.rows) } + +func (col *optionalColumnBuffer) Less(i, j int) bool { + return col.nullOrdering( + col.base, + int(col.rows[i]), + int(col.rows[j]), + col.maxDefinitionLevel, + col.definitionLevels[i], + col.definitionLevels[j], + ) +} + +func (col *optionalColumnBuffer) Swap(i, j int) { + // Because the underlying column does not contain null values, we cannot + // swap its values at indexes i and j. We swap the row indexes only, then + // reorder the underlying buffer using a cyclic sort when the buffer is + // materialized into a page view. + col.reordered = true + col.rows[i], col.rows[j] = col.rows[j], col.rows[i] + col.definitionLevels[i], col.definitionLevels[j] = col.definitionLevels[j], col.definitionLevels[i] +} + +func (col *optionalColumnBuffer) WriteValues(values []Value) (n int, err error) { + rowIndex := int32(col.base.Len()) + + for n < len(values) { + // Collect index range of contiguous null values, from i to n. If this + // for loop exhausts the values, all remaining if statements and for + // loops will be no-ops and the loop will terminate. + i := n + for n < len(values) && values[n].definitionLevel != col.maxDefinitionLevel { + n++ + } + + // Write the contiguous null values up until the first non-null value + // obtained in the for loop above. + for _, v := range values[i:n] { + col.rows = append(col.rows, -1) + col.definitionLevels = append(col.definitionLevels, v.definitionLevel) + } + + // Collect index range of contiguous non-null values, from i to n. + i = n + for n < len(values) && values[n].definitionLevel == col.maxDefinitionLevel { + n++ + } + + // As long as i < n we have non-null values still to write. It is + // possible that we just exhausted the input values in which case i == n + // and the outer for loop will terminate. + if i < n { + count, err := col.base.WriteValues(values[i:n]) + col.definitionLevels = appendLevel(col.definitionLevels, col.maxDefinitionLevel, count) + + for count > 0 { + col.rows = append(col.rows, rowIndex) + rowIndex++ + count-- + } + + if err != nil { + return n, err + } + } + } + return n, nil +} + +func (col *optionalColumnBuffer) writeValues(rows sparse.Array, levels columnLevels) { + // The row count is zero when writing an null optional value, in which case + // we still need to output a row to the buffer to record the definition + // level. + if rows.Len() == 0 { + col.definitionLevels = append(col.definitionLevels, levels.definitionLevel) + col.rows = append(col.rows, -1) + return + } + + col.definitionLevels = appendLevel(col.definitionLevels, levels.definitionLevel, rows.Len()) + + i := len(col.rows) + j := len(col.rows) + rows.Len() + + if j <= cap(col.rows) { + col.rows = col.rows[:j] + } else { + tmp := make([]int32, j, 2*j) + copy(tmp, col.rows) + col.rows = tmp + } + + if levels.definitionLevel != col.maxDefinitionLevel { + broadcastValueInt32(col.rows[i:], -1) + } else { + broadcastRangeInt32(col.rows[i:], int32(col.base.Len())) + col.base.writeValues(rows, levels) + } +} + +func (col *optionalColumnBuffer) ReadValuesAt(values []Value, offset int64) (int, error) { + length := int64(len(col.definitionLevels)) + if offset < 0 { + return 0, errRowIndexOutOfBounds(offset, length) + } + if offset >= length { + return 0, io.EOF + } + if length -= offset; length < int64(len(values)) { + values = values[:length] + } + + numNulls1 := int64(countLevelsNotEqual(col.definitionLevels[:offset], col.maxDefinitionLevel)) + numNulls2 := int64(countLevelsNotEqual(col.definitionLevels[offset:offset+length], col.maxDefinitionLevel)) + + if numNulls2 < length { + n, err := col.base.ReadValuesAt(values[:length-numNulls2], offset-numNulls1) + if err != nil { + return n, err + } + } + + if numNulls2 > 0 { + columnIndex := ^int16(col.Column()) + i := numNulls2 - 1 + j := length - 1 + definitionLevels := col.definitionLevels[offset : offset+length] + maxDefinitionLevel := col.maxDefinitionLevel + + for n := len(definitionLevels) - 1; n >= 0 && j > i; n-- { + if definitionLevels[n] != maxDefinitionLevel { + values[j] = Value{definitionLevel: definitionLevels[n], columnIndex: columnIndex} + } else { + values[j] = values[i] + i-- + } + j-- + } + } + + return int(length), nil +} + +// repeatedColumnBuffer is an implementation of the ColumnBuffer interface used +// as a wrapper to an underlying ColumnBuffer to manage the creation of +// repetition levels, definition levels, and map rows to the region of the +// underlying buffer that contains their sequence of values. +// +// Null values are not written to the underlying column; instead, the buffer +// tracks offsets of row values in the column, null row values are represented +// by the value -1 and a definition level less than the max. +// +// This column buffer type is used for all leaf columns that have a non-zero +// max repetition level, which may be because the column or one of its parent(s) +// are marked repeated. +type repeatedColumnBuffer struct { + base ColumnBuffer + reordered bool + maxRepetitionLevel byte + maxDefinitionLevel byte + rows []offsetMapping + repetitionLevels []byte + definitionLevels []byte + buffer []Value + reordering *repeatedColumnBuffer + nullOrdering nullOrdering +} + +// The offsetMapping type maps the logical offset of rows within the repetition +// and definition levels, to the base offsets in the underlying column buffers +// where the non-null values have been written. +type offsetMapping struct { + offset uint32 + baseOffset uint32 +} + +func newRepeatedColumnBuffer(base ColumnBuffer, maxRepetitionLevel, maxDefinitionLevel byte, nullOrdering nullOrdering) *repeatedColumnBuffer { + n := base.Cap() + return &repeatedColumnBuffer{ + base: base, + maxRepetitionLevel: maxRepetitionLevel, + maxDefinitionLevel: maxDefinitionLevel, + rows: make([]offsetMapping, 0, n/8), + repetitionLevels: make([]byte, 0, n), + definitionLevels: make([]byte, 0, n), + nullOrdering: nullOrdering, + } +} + +func (col *repeatedColumnBuffer) Clone() ColumnBuffer { + return &repeatedColumnBuffer{ + base: col.base.Clone(), + reordered: col.reordered, + maxRepetitionLevel: col.maxRepetitionLevel, + maxDefinitionLevel: col.maxDefinitionLevel, + rows: slices.Clone(col.rows), + repetitionLevels: slices.Clone(col.repetitionLevels), + definitionLevels: slices.Clone(col.definitionLevels), + nullOrdering: col.nullOrdering, + } +} + +func (col *repeatedColumnBuffer) Type() Type { + return col.base.Type() +} + +func (col *repeatedColumnBuffer) NumValues() int64 { + return int64(len(col.definitionLevels)) +} + +func (col *repeatedColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return columnIndexOfNullable(col.base, col.maxDefinitionLevel, col.definitionLevels) +} + +func (col *repeatedColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return col.base.OffsetIndex() +} + +func (col *repeatedColumnBuffer) BloomFilter() BloomFilter { + return col.base.BloomFilter() +} + +func (col *repeatedColumnBuffer) Dictionary() Dictionary { + return col.base.Dictionary() +} + +func (col *repeatedColumnBuffer) Column() int { + return col.base.Column() +} + +func (col *repeatedColumnBuffer) Pages() Pages { + return onePage(col.Page()) +} + +func (col *repeatedColumnBuffer) Page() Page { + if col.reordered { + if col.reordering == nil { + col.reordering = col.Clone().(*repeatedColumnBuffer) + } + + column := col.reordering + column.Reset() + maxNumValues := 0 + defer func() { + clearValues(col.buffer[:maxNumValues]) + }() + + baseOffset := 0 + + for _, row := range col.rows { + rowOffset := int(row.offset) + rowLength := repeatedRowLength(col.repetitionLevels[rowOffset:]) + numNulls := countLevelsNotEqual(col.definitionLevels[rowOffset:rowOffset+rowLength], col.maxDefinitionLevel) + numValues := rowLength - numNulls + + if numValues > 0 { + if numValues > cap(col.buffer) { + col.buffer = make([]Value, numValues) + } else { + col.buffer = col.buffer[:numValues] + } + n, err := col.base.ReadValuesAt(col.buffer, int64(row.baseOffset)) + if err != nil && n < numValues { + return newErrorPage(col.Type(), col.Column(), "reordering rows of repeated column: %w", err) + } + if _, err := column.base.WriteValues(col.buffer); err != nil { + return newErrorPage(col.Type(), col.Column(), "reordering rows of repeated column: %w", err) + } + if numValues > maxNumValues { + maxNumValues = numValues + } + } + + column.rows = append(column.rows, offsetMapping{ + offset: uint32(len(column.repetitionLevels)), + baseOffset: uint32(baseOffset), + }) + + column.repetitionLevels = append(column.repetitionLevels, col.repetitionLevels[rowOffset:rowOffset+rowLength]...) + column.definitionLevels = append(column.definitionLevels, col.definitionLevels[rowOffset:rowOffset+rowLength]...) + baseOffset += numValues + } + + col.swapReorderingBuffer(column) + col.reordered = false + } + + return newRepeatedPage( + col.base.Page(), + col.maxRepetitionLevel, + col.maxDefinitionLevel, + col.repetitionLevels, + col.definitionLevels, + ) +} + +func (col *repeatedColumnBuffer) swapReorderingBuffer(buf *repeatedColumnBuffer) { + col.base, buf.base = buf.base, col.base + col.rows, buf.rows = buf.rows, col.rows + col.repetitionLevels, buf.repetitionLevels = buf.repetitionLevels, col.repetitionLevels + col.definitionLevels, buf.definitionLevels = buf.definitionLevels, col.definitionLevels +} + +func (col *repeatedColumnBuffer) Reset() { + col.base.Reset() + col.rows = col.rows[:0] + col.repetitionLevels = col.repetitionLevels[:0] + col.definitionLevels = col.definitionLevels[:0] +} + +func (col *repeatedColumnBuffer) Size() int64 { + return int64(8*len(col.rows)+len(col.repetitionLevels)+len(col.definitionLevels)) + col.base.Size() +} + +func (col *repeatedColumnBuffer) Cap() int { return cap(col.rows) } + +func (col *repeatedColumnBuffer) Len() int { return len(col.rows) } + +func (col *repeatedColumnBuffer) Less(i, j int) bool { + row1 := col.rows[i] + row2 := col.rows[j] + less := col.nullOrdering + row1Length := repeatedRowLength(col.repetitionLevels[row1.offset:]) + row2Length := repeatedRowLength(col.repetitionLevels[row2.offset:]) + + for k := 0; k < row1Length && k < row2Length; k++ { + x := int(row1.baseOffset) + y := int(row2.baseOffset) + definitionLevel1 := col.definitionLevels[int(row1.offset)+k] + definitionLevel2 := col.definitionLevels[int(row2.offset)+k] + switch { + case less(col.base, x, y, col.maxDefinitionLevel, definitionLevel1, definitionLevel2): + return true + case less(col.base, y, x, col.maxDefinitionLevel, definitionLevel2, definitionLevel1): + return false + } + } + + return row1Length < row2Length +} + +func (col *repeatedColumnBuffer) Swap(i, j int) { + // Because the underlying column does not contain null values, and may hold + // an arbitrary number of values per row, we cannot swap its values at + // indexes i and j. We swap the row indexes only, then reorder the base + // column buffer when its view is materialized into a page by creating a + // copy and writing rows back to it following the order of rows in the + // repeated column buffer. + col.reordered = true + col.rows[i], col.rows[j] = col.rows[j], col.rows[i] +} + +func (col *repeatedColumnBuffer) WriteValues(values []Value) (numValues int, err error) { + maxRowLen := 0 + defer func() { + clearValues(col.buffer[:maxRowLen]) + }() + + for i := 0; i < len(values); { + j := i + + if values[j].repetitionLevel == 0 { + j++ + } + + for j < len(values) && values[j].repetitionLevel != 0 { + j++ + } + + if err := col.writeRow(values[i:j]); err != nil { + return numValues, err + } + + if len(col.buffer) > maxRowLen { + maxRowLen = len(col.buffer) + } + + numValues += j - i + i = j + } + + return numValues, nil +} + +func (col *repeatedColumnBuffer) writeRow(row []Value) error { + col.buffer = col.buffer[:0] + + for _, v := range row { + if v.definitionLevel == col.maxDefinitionLevel { + col.buffer = append(col.buffer, v) + } + } + + baseOffset := col.base.NumValues() + if len(col.buffer) > 0 { + if _, err := col.base.WriteValues(col.buffer); err != nil { + return err + } + } + + if row[0].repetitionLevel == 0 { + col.rows = append(col.rows, offsetMapping{ + offset: uint32(len(col.repetitionLevels)), + baseOffset: uint32(baseOffset), + }) + } + + for _, v := range row { + col.repetitionLevels = append(col.repetitionLevels, v.repetitionLevel) + col.definitionLevels = append(col.definitionLevels, v.definitionLevel) + } + + return nil +} + +func (col *repeatedColumnBuffer) writeValues(row sparse.Array, levels columnLevels) { + if levels.repetitionLevel == 0 { + col.rows = append(col.rows, offsetMapping{ + offset: uint32(len(col.repetitionLevels)), + baseOffset: uint32(col.base.NumValues()), + }) + } + + if row.Len() == 0 { + col.repetitionLevels = append(col.repetitionLevels, levels.repetitionLevel) + col.definitionLevels = append(col.definitionLevels, levels.definitionLevel) + return + } + + col.repetitionLevels = appendLevel(col.repetitionLevels, levels.repetitionLevel, row.Len()) + col.definitionLevels = appendLevel(col.definitionLevels, levels.definitionLevel, row.Len()) + + if levels.definitionLevel == col.maxDefinitionLevel { + col.base.writeValues(row, levels) + } +} + +func (col *repeatedColumnBuffer) ReadValuesAt(values []Value, offset int64) (int, error) { + // TODO: + panic("NOT IMPLEMENTED") +} + +// repeatedRowLength gives the length of the repeated row starting at the +// beginning of the repetitionLevels slice. +func repeatedRowLength(repetitionLevels []byte) int { + // If a repetition level exists, at least one value is required to represent + // the column. + if len(repetitionLevels) > 0 { + // The subsequent levels will represent the start of a new record when + // they go back to zero. + if i := bytes.IndexByte(repetitionLevels[1:], 0); i >= 0 { + return i + 1 + } + } + return len(repetitionLevels) +} + +// ============================================================================= +// The types below are in-memory implementations of the ColumnBuffer interface +// for each parquet type. +// +// These column buffers are created by calling NewColumnBuffer on parquet.Type +// instances; each parquet type manages to construct column buffers of the +// appropriate type, which ensures that we are packing as many values as we +// can in memory. +// +// See Type.NewColumnBuffer for details about how these types get created. +// ============================================================================= + +type booleanColumnBuffer struct{ booleanPage } + +func newBooleanColumnBuffer(typ Type, columnIndex int16, numValues int32) *booleanColumnBuffer { + // Boolean values are bit-packed, we can fit up to 8 values per byte. + bufferSize := (numValues + 7) / 8 + return &booleanColumnBuffer{ + booleanPage: booleanPage{ + typ: typ, + bits: make([]byte, 0, bufferSize), + columnIndex: ^columnIndex, + }, + } +} + +func (col *booleanColumnBuffer) Clone() ColumnBuffer { + return &booleanColumnBuffer{ + booleanPage: booleanPage{ + typ: col.typ, + bits: slices.Clone(col.bits), + offset: col.offset, + numValues: col.numValues, + columnIndex: col.columnIndex, + }, + } +} + +func (col *booleanColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return booleanColumnIndex{&col.booleanPage}, nil +} + +func (col *booleanColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return booleanOffsetIndex{&col.booleanPage}, nil +} + +func (col *booleanColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *booleanColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *booleanColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *booleanColumnBuffer) Page() Page { return &col.booleanPage } + +func (col *booleanColumnBuffer) Reset() { + col.bits = col.bits[:0] + col.offset = 0 + col.numValues = 0 +} + +func (col *booleanColumnBuffer) Cap() int { return 8 * cap(col.bits) } + +func (col *booleanColumnBuffer) Len() int { return int(col.numValues) } + +func (col *booleanColumnBuffer) Less(i, j int) bool { + a := col.valueAt(i) + b := col.valueAt(j) + return a != b && !a +} + +func (col *booleanColumnBuffer) valueAt(i int) bool { + j := uint32(i) / 8 + k := uint32(i) % 8 + return ((col.bits[j] >> k) & 1) != 0 +} + +func (col *booleanColumnBuffer) setValueAt(i int, v bool) { + // `offset` is always zero in the page of a column buffer + j := uint32(i) / 8 + k := uint32(i) % 8 + x := byte(0) + if v { + x = 1 + } + col.bits[j] = (col.bits[j] & ^(1 << k)) | (x << k) +} + +func (col *booleanColumnBuffer) Swap(i, j int) { + a := col.valueAt(i) + b := col.valueAt(j) + col.setValueAt(i, b) + col.setValueAt(j, a) +} + +func (col *booleanColumnBuffer) WriteBooleans(values []bool) (int, error) { + col.writeValues(sparse.MakeBoolArray(values).UnsafeArray(), columnLevels{}) + return len(values), nil +} + +func (col *booleanColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfBool), columnLevels{}) + return len(values), nil +} + +func (col *booleanColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + numBytes := bitpack.ByteCount(uint(col.numValues) + uint(rows.Len())) + if cap(col.bits) < numBytes { + col.bits = append(make([]byte, 0, max(numBytes, 2*cap(col.bits))), col.bits...) + } + col.bits = col.bits[:numBytes] + i := 0 + r := 8 - (int(col.numValues) % 8) + bytes := rows.Uint8Array() + + if r <= bytes.Len() { + // First we attempt to write enough bits to align the number of values + // in the column buffer on 8 bytes. After this step the next bit should + // be written at the zero'th index of a byte of the buffer. + if r < 8 { + var b byte + for i < r { + v := bytes.Index(i) + b |= (v & 1) << uint(i) + i++ + } + x := uint(col.numValues) / 8 + y := uint(col.numValues) % 8 + col.bits[x] = (b << y) | (col.bits[x] & ^(0xFF << y)) + col.numValues += int32(i) + } + + if n := ((bytes.Len() - i) / 8) * 8; n > 0 { + // At this stage, we know that that we have at least 8 bits to write + // and the bits will be aligned on the address of a byte in the + // output buffer. We can work on 8 values per loop iteration, + // packing them into a single byte and writing it to the output + // buffer. This effectively reduces by 87.5% the number of memory + // stores that the program needs to perform to generate the values. + i += sparse.GatherBits(col.bits[col.numValues/8:], bytes.Slice(i, i+n)) + col.numValues += int32(n) + } + } + + for i < bytes.Len() { + x := uint(col.numValues) / 8 + y := uint(col.numValues) % 8 + b := bytes.Index(i) + col.bits[x] = ((b & 1) << y) | (col.bits[x] & ^(1 << y)) + col.numValues++ + i++ + } + + col.bits = col.bits[:bitpack.ByteCount(uint(col.numValues))] +} + +func (col *booleanColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(col.numValues)) + case i >= int(col.numValues): + return 0, io.EOF + default: + for n < len(values) && i < int(col.numValues) { + values[n] = col.makeValue(col.valueAt(i)) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type int32ColumnBuffer struct{ int32Page } + +func newInt32ColumnBuffer(typ Type, columnIndex int16, numValues int32) *int32ColumnBuffer { + return &int32ColumnBuffer{ + int32Page: int32Page{ + typ: typ, + values: make([]int32, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *int32ColumnBuffer) Clone() ColumnBuffer { + return &int32ColumnBuffer{ + int32Page: int32Page{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *int32ColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return int32ColumnIndex{&col.int32Page}, nil +} + +func (col *int32ColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return int32OffsetIndex{&col.int32Page}, nil +} + +func (col *int32ColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *int32ColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *int32ColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *int32ColumnBuffer) Page() Page { return &col.int32Page } + +func (col *int32ColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *int32ColumnBuffer) Cap() int { return cap(col.values) } + +func (col *int32ColumnBuffer) Len() int { return len(col.values) } + +func (col *int32ColumnBuffer) Less(i, j int) bool { return col.values[i] < col.values[j] } + +func (col *int32ColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *int32ColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 4) != 0 { + return 0, fmt.Errorf("cannot write INT32 values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[int32](b)...) + return len(b), nil +} + +func (col *int32ColumnBuffer) WriteInt32s(values []int32) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *int32ColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfU32), columnLevels{}) + return len(values), nil +} + +func (col *int32ColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([]int32, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherInt32(col.values[n:], rows.Int32Array()) + +} + +func (col *int32ColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type int64ColumnBuffer struct{ int64Page } + +func newInt64ColumnBuffer(typ Type, columnIndex int16, numValues int32) *int64ColumnBuffer { + return &int64ColumnBuffer{ + int64Page: int64Page{ + typ: typ, + values: make([]int64, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *int64ColumnBuffer) Clone() ColumnBuffer { + return &int64ColumnBuffer{ + int64Page: int64Page{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *int64ColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return int64ColumnIndex{&col.int64Page}, nil +} + +func (col *int64ColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return int64OffsetIndex{&col.int64Page}, nil +} + +func (col *int64ColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *int64ColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *int64ColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *int64ColumnBuffer) Page() Page { return &col.int64Page } + +func (col *int64ColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *int64ColumnBuffer) Cap() int { return cap(col.values) } + +func (col *int64ColumnBuffer) Len() int { return len(col.values) } + +func (col *int64ColumnBuffer) Less(i, j int) bool { return col.values[i] < col.values[j] } + +func (col *int64ColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *int64ColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 8) != 0 { + return 0, fmt.Errorf("cannot write INT64 values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[int64](b)...) + return len(b), nil +} + +func (col *int64ColumnBuffer) WriteInt64s(values []int64) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *int64ColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfU64), columnLevels{}) + return len(values), nil +} + +func (col *int64ColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([]int64, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherInt64(col.values[n:], rows.Int64Array()) +} + +func (col *int64ColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type int96ColumnBuffer struct{ int96Page } + +func newInt96ColumnBuffer(typ Type, columnIndex int16, numValues int32) *int96ColumnBuffer { + return &int96ColumnBuffer{ + int96Page: int96Page{ + typ: typ, + values: make([]deprecated.Int96, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *int96ColumnBuffer) Clone() ColumnBuffer { + return &int96ColumnBuffer{ + int96Page: int96Page{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *int96ColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return int96ColumnIndex{&col.int96Page}, nil +} + +func (col *int96ColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return int96OffsetIndex{&col.int96Page}, nil +} + +func (col *int96ColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *int96ColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *int96ColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *int96ColumnBuffer) Page() Page { return &col.int96Page } + +func (col *int96ColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *int96ColumnBuffer) Cap() int { return cap(col.values) } + +func (col *int96ColumnBuffer) Len() int { return len(col.values) } + +func (col *int96ColumnBuffer) Less(i, j int) bool { return col.values[i].Less(col.values[j]) } + +func (col *int96ColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *int96ColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 12) != 0 { + return 0, fmt.Errorf("cannot write INT96 values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[deprecated.Int96](b)...) + return len(b), nil +} + +func (col *int96ColumnBuffer) WriteInt96s(values []deprecated.Int96) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *int96ColumnBuffer) WriteValues(values []Value) (int, error) { + for _, v := range values { + col.values = append(col.values, v.Int96()) + } + return len(values), nil +} + +func (col *int96ColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + for i := range rows.Len() { + p := rows.Index(i) + col.values = append(col.values, *(*deprecated.Int96)(p)) + } +} + +func (col *int96ColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type floatColumnBuffer struct{ floatPage } + +func newFloatColumnBuffer(typ Type, columnIndex int16, numValues int32) *floatColumnBuffer { + return &floatColumnBuffer{ + floatPage: floatPage{ + typ: typ, + values: make([]float32, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *floatColumnBuffer) Clone() ColumnBuffer { + return &floatColumnBuffer{ + floatPage: floatPage{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *floatColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return floatColumnIndex{&col.floatPage}, nil +} + +func (col *floatColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return floatOffsetIndex{&col.floatPage}, nil +} + +func (col *floatColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *floatColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *floatColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *floatColumnBuffer) Page() Page { return &col.floatPage } + +func (col *floatColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *floatColumnBuffer) Cap() int { return cap(col.values) } + +func (col *floatColumnBuffer) Len() int { return len(col.values) } + +func (col *floatColumnBuffer) Less(i, j int) bool { return col.values[i] < col.values[j] } + +func (col *floatColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *floatColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 4) != 0 { + return 0, fmt.Errorf("cannot write FLOAT values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[float32](b)...) + return len(b), nil +} + +func (col *floatColumnBuffer) WriteFloats(values []float32) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *floatColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfU32), columnLevels{}) + return len(values), nil +} + +func (col *floatColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([]float32, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherFloat32(col.values[n:], rows.Float32Array()) +} + +func (col *floatColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type doubleColumnBuffer struct{ doublePage } + +func newDoubleColumnBuffer(typ Type, columnIndex int16, numValues int32) *doubleColumnBuffer { + return &doubleColumnBuffer{ + doublePage: doublePage{ + typ: typ, + values: make([]float64, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *doubleColumnBuffer) Clone() ColumnBuffer { + return &doubleColumnBuffer{ + doublePage: doublePage{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *doubleColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return doubleColumnIndex{&col.doublePage}, nil +} + +func (col *doubleColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return doubleOffsetIndex{&col.doublePage}, nil +} + +func (col *doubleColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *doubleColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *doubleColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *doubleColumnBuffer) Page() Page { return &col.doublePage } + +func (col *doubleColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *doubleColumnBuffer) Cap() int { return cap(col.values) } + +func (col *doubleColumnBuffer) Len() int { return len(col.values) } + +func (col *doubleColumnBuffer) Less(i, j int) bool { return col.values[i] < col.values[j] } + +func (col *doubleColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *doubleColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 8) != 0 { + return 0, fmt.Errorf("cannot write DOUBLE values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[float64](b)...) + return len(b), nil +} + +func (col *doubleColumnBuffer) WriteDoubles(values []float64) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *doubleColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfU64), columnLevels{}) + return len(values), nil +} + +func (col *doubleColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([]float64, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherFloat64(col.values[n:], rows.Float64Array()) +} + +func (col *doubleColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type byteArrayColumnBuffer struct { + byteArrayPage + lengths []uint32 + scratch []byte +} + +func newByteArrayColumnBuffer(typ Type, columnIndex int16, numValues int32) *byteArrayColumnBuffer { + return &byteArrayColumnBuffer{ + byteArrayPage: byteArrayPage{ + typ: typ, + values: make([]byte, 0, typ.EstimateSize(int(numValues))), + offsets: make([]uint32, 0, numValues+1), + columnIndex: ^columnIndex, + }, + lengths: make([]uint32, 0, numValues), + } +} + +func (col *byteArrayColumnBuffer) Clone() ColumnBuffer { + return &byteArrayColumnBuffer{ + byteArrayPage: byteArrayPage{ + typ: col.typ, + values: col.cloneValues(), + offsets: col.cloneOffsets(), + columnIndex: col.columnIndex, + }, + lengths: col.cloneLengths(), + } +} + +func (col *byteArrayColumnBuffer) cloneLengths() []uint32 { + lengths := make([]uint32, len(col.lengths)) + copy(lengths, col.lengths) + return lengths +} + +func (col *byteArrayColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return byteArrayColumnIndex{&col.byteArrayPage}, nil +} + +func (col *byteArrayColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return byteArrayOffsetIndex{&col.byteArrayPage}, nil +} + +func (col *byteArrayColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *byteArrayColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *byteArrayColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *byteArrayColumnBuffer) Page() Page { + if len(col.lengths) > 0 && orderOfUint32(col.offsets) < 1 { // unordered? + if cap(col.scratch) < len(col.values) { + col.scratch = make([]byte, 0, cap(col.values)) + } else { + col.scratch = col.scratch[:0] + } + + for i := range col.lengths { + n := len(col.scratch) + col.scratch = append(col.scratch, col.index(i)...) + col.offsets[i] = uint32(n) + } + + col.values, col.scratch = col.scratch, col.values + } + // The offsets have the total length as the last item. Since we are about to + // expose the column buffer's internal state as a Page value we ensure that + // the last offset is the total length of all values. + col.offsets = append(col.offsets[:len(col.lengths)], uint32(len(col.values))) + return &col.byteArrayPage +} + +func (col *byteArrayColumnBuffer) Reset() { + col.values = col.values[:0] + col.offsets = col.offsets[:0] + col.lengths = col.lengths[:0] +} + +func (col *byteArrayColumnBuffer) NumRows() int64 { return int64(col.Len()) } + +func (col *byteArrayColumnBuffer) NumValues() int64 { return int64(col.Len()) } + +func (col *byteArrayColumnBuffer) Cap() int { return cap(col.lengths) } + +func (col *byteArrayColumnBuffer) Len() int { return len(col.lengths) } + +func (col *byteArrayColumnBuffer) Less(i, j int) bool { + return bytes.Compare(col.index(i), col.index(j)) < 0 +} + +func (col *byteArrayColumnBuffer) Swap(i, j int) { + col.offsets[i], col.offsets[j] = col.offsets[j], col.offsets[i] + col.lengths[i], col.lengths[j] = col.lengths[j], col.lengths[i] +} + +func (col *byteArrayColumnBuffer) Write(b []byte) (int, error) { + _, n, err := col.writeByteArrays(b) + return n, err +} + +func (col *byteArrayColumnBuffer) WriteByteArrays(values []byte) (int, error) { + n, _, err := col.writeByteArrays(values) + return n, err +} + +func (col *byteArrayColumnBuffer) writeByteArrays(values []byte) (count, bytes int, err error) { + baseCount := len(col.lengths) + baseBytes := len(col.values) + (plain.ByteArrayLengthSize * len(col.lengths)) + + err = plain.RangeByteArray(values, func(value []byte) error { + col.append(unsafecast.String(value)) + return nil + }) + + count = len(col.lengths) - baseCount + bytes = (len(col.values) - baseBytes) + (plain.ByteArrayLengthSize * count) + return count, bytes, err +} + +func (col *byteArrayColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfPtr), columnLevels{}) + return len(values), nil +} + +func (col *byteArrayColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + for i := range rows.Len() { + p := rows.Index(i) + col.append(*(*string)(p)) + } +} + +func (col *byteArrayColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.lengths))) + case i >= len(col.lengths): + return 0, io.EOF + default: + for n < len(values) && i < len(col.lengths) { + values[n] = col.makeValueBytes(col.index(i)) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +func (col *byteArrayColumnBuffer) append(value string) { + col.offsets = append(col.offsets, uint32(len(col.values))) + col.lengths = append(col.lengths, uint32(len(value))) + col.values = append(col.values, value...) +} + +func (col *byteArrayColumnBuffer) index(i int) []byte { + offset := col.offsets[i] + length := col.lengths[i] + end := offset + length + return col.values[offset:end:end] +} + +type fixedLenByteArrayColumnBuffer struct { + fixedLenByteArrayPage + tmp []byte +} + +func newFixedLenByteArrayColumnBuffer(typ Type, columnIndex int16, numValues int32) *fixedLenByteArrayColumnBuffer { + size := typ.Length() + return &fixedLenByteArrayColumnBuffer{ + fixedLenByteArrayPage: fixedLenByteArrayPage{ + typ: typ, + size: size, + data: make([]byte, 0, typ.EstimateSize(int(numValues))), + columnIndex: ^columnIndex, + }, + tmp: make([]byte, size), + } +} + +func (col *fixedLenByteArrayColumnBuffer) Clone() ColumnBuffer { + return &fixedLenByteArrayColumnBuffer{ + fixedLenByteArrayPage: fixedLenByteArrayPage{ + typ: col.typ, + size: col.size, + data: slices.Clone(col.data), + columnIndex: col.columnIndex, + }, + tmp: make([]byte, col.size), + } +} + +func (col *fixedLenByteArrayColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return fixedLenByteArrayColumnIndex{&col.fixedLenByteArrayPage}, nil +} + +func (col *fixedLenByteArrayColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return fixedLenByteArrayOffsetIndex{&col.fixedLenByteArrayPage}, nil +} + +func (col *fixedLenByteArrayColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *fixedLenByteArrayColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *fixedLenByteArrayColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *fixedLenByteArrayColumnBuffer) Page() Page { return &col.fixedLenByteArrayPage } + +func (col *fixedLenByteArrayColumnBuffer) Reset() { col.data = col.data[:0] } + +func (col *fixedLenByteArrayColumnBuffer) Cap() int { return cap(col.data) / col.size } + +func (col *fixedLenByteArrayColumnBuffer) Len() int { return len(col.data) / col.size } + +func (col *fixedLenByteArrayColumnBuffer) Less(i, j int) bool { + return bytes.Compare(col.index(i), col.index(j)) < 0 +} + +func (col *fixedLenByteArrayColumnBuffer) Swap(i, j int) { + t, u, v := col.tmp[:col.size], col.index(i), col.index(j) + copy(t, u) + copy(u, v) + copy(v, t) +} + +func (col *fixedLenByteArrayColumnBuffer) index(i int) []byte { + j := (i + 0) * col.size + k := (i + 1) * col.size + return col.data[j:k:k] +} + +func (col *fixedLenByteArrayColumnBuffer) Write(b []byte) (int, error) { + n, err := col.WriteFixedLenByteArrays(b) + return n * col.size, err +} + +func (col *fixedLenByteArrayColumnBuffer) WriteFixedLenByteArrays(values []byte) (int, error) { + if len(values) == 0 { + return 0, nil + } + d, m := len(values)/col.size, len(values)%col.size + if d == 0 || m != 0 { + return 0, fmt.Errorf("cannot write FIXED_LEN_BYTE_ARRAY values of size %d from input of size %d", col.size, len(values)) + } + col.data = append(col.data, values...) + return d, nil +} + +func (col *fixedLenByteArrayColumnBuffer) WriteValues(values []Value) (int, error) { + for i, v := range values { + if n := len(v.byteArray()); n != col.size { + return i, fmt.Errorf("cannot write FIXED_LEN_BYTE_ARRAY values of size %d from input of size %d", col.size, n) + } + col.data = append(col.data, v.byteArray()...) + } + return len(values), nil +} + +func (col *fixedLenByteArrayColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + n := col.size * rows.Len() + i := len(col.data) + j := len(col.data) + n + + if cap(col.data) < j { + col.data = append(make([]byte, 0, max(i+n, 2*cap(col.data))), col.data...) + } + + col.data = col.data[:j] + newData := col.data[i:] + + for i := range rows.Len() { + p := rows.Index(i) + copy(newData[i*col.size:], unsafe.Slice((*byte)(p), col.size)) + } +} + +func (col *fixedLenByteArrayColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) * col.size + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.data)/col.size)) + case i >= len(col.data): + return 0, io.EOF + default: + for n < len(values) && i < len(col.data) { + values[n] = col.makeValueBytes(col.data[i : i+col.size]) + n++ + i += col.size + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type uint32ColumnBuffer struct{ uint32Page } + +func newUint32ColumnBuffer(typ Type, columnIndex int16, numValues int32) *uint32ColumnBuffer { + return &uint32ColumnBuffer{ + uint32Page: uint32Page{ + typ: typ, + values: make([]uint32, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *uint32ColumnBuffer) Clone() ColumnBuffer { + return &uint32ColumnBuffer{ + uint32Page: uint32Page{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *uint32ColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return uint32ColumnIndex{&col.uint32Page}, nil +} + +func (col *uint32ColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return uint32OffsetIndex{&col.uint32Page}, nil +} + +func (col *uint32ColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *uint32ColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *uint32ColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *uint32ColumnBuffer) Page() Page { return &col.uint32Page } + +func (col *uint32ColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *uint32ColumnBuffer) Cap() int { return cap(col.values) } + +func (col *uint32ColumnBuffer) Len() int { return len(col.values) } + +func (col *uint32ColumnBuffer) Less(i, j int) bool { return col.values[i] < col.values[j] } + +func (col *uint32ColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *uint32ColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 4) != 0 { + return 0, fmt.Errorf("cannot write INT32 values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[uint32](b)...) + return len(b), nil +} + +func (col *uint32ColumnBuffer) WriteUint32s(values []uint32) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *uint32ColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfU32), columnLevels{}) + return len(values), nil +} + +func (col *uint32ColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([]uint32, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherUint32(col.values[n:], rows.Uint32Array()) +} + +func (col *uint32ColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type uint64ColumnBuffer struct{ uint64Page } + +func newUint64ColumnBuffer(typ Type, columnIndex int16, numValues int32) *uint64ColumnBuffer { + return &uint64ColumnBuffer{ + uint64Page: uint64Page{ + typ: typ, + values: make([]uint64, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *uint64ColumnBuffer) Clone() ColumnBuffer { + return &uint64ColumnBuffer{ + uint64Page: uint64Page{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *uint64ColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return uint64ColumnIndex{&col.uint64Page}, nil +} + +func (col *uint64ColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return uint64OffsetIndex{&col.uint64Page}, nil +} + +func (col *uint64ColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *uint64ColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *uint64ColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *uint64ColumnBuffer) Page() Page { return &col.uint64Page } + +func (col *uint64ColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *uint64ColumnBuffer) Cap() int { return cap(col.values) } + +func (col *uint64ColumnBuffer) Len() int { return len(col.values) } + +func (col *uint64ColumnBuffer) Less(i, j int) bool { return col.values[i] < col.values[j] } + +func (col *uint64ColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *uint64ColumnBuffer) Write(b []byte) (int, error) { + if (len(b) % 8) != 0 { + return 0, fmt.Errorf("cannot write INT64 values from input of size %d", len(b)) + } + col.values = append(col.values, unsafecast.Slice[uint64](b)...) + return len(b), nil +} + +func (col *uint64ColumnBuffer) WriteUint64s(values []uint64) (int, error) { + col.values = append(col.values, values...) + return len(values), nil +} + +func (col *uint64ColumnBuffer) WriteValues(values []Value) (int, error) { + col.writeValues(makeArrayValue(values, offsetOfU64), columnLevels{}) + return len(values), nil +} + +func (col *uint64ColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([]uint64, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherUint64(col.values[n:], rows.Uint64Array()) +} + +func (col *uint64ColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +type be128ColumnBuffer struct{ be128Page } + +func newBE128ColumnBuffer(typ Type, columnIndex int16, numValues int32) *be128ColumnBuffer { + return &be128ColumnBuffer{ + be128Page: be128Page{ + typ: typ, + values: make([][16]byte, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *be128ColumnBuffer) Clone() ColumnBuffer { + return &be128ColumnBuffer{ + be128Page: be128Page{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *be128ColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return be128ColumnIndex{&col.be128Page}, nil +} + +func (col *be128ColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return be128OffsetIndex{&col.be128Page}, nil +} + +func (col *be128ColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *be128ColumnBuffer) Dictionary() Dictionary { return nil } + +func (col *be128ColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *be128ColumnBuffer) Page() Page { return &col.be128Page } + +func (col *be128ColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *be128ColumnBuffer) Cap() int { return cap(col.values) } + +func (col *be128ColumnBuffer) Len() int { return len(col.values) } + +func (col *be128ColumnBuffer) Less(i, j int) bool { + return lessBE128(&col.values[i], &col.values[j]) +} + +func (col *be128ColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *be128ColumnBuffer) WriteValues(values []Value) (int, error) { + if n := len(col.values) + len(values); n > cap(col.values) { + col.values = append(make([][16]byte, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+len(values)] + newValues := col.values[n:] + for i, v := range values { + copy(newValues[i][:], v.byteArray()) + } + return len(values), nil +} + +func (col *be128ColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + if n := len(col.values) + rows.Len(); n > cap(col.values) { + col.values = append(make([][16]byte, 0, max(n, 2*cap(col.values))), col.values...) + } + n := len(col.values) + col.values = col.values[:n+rows.Len()] + sparse.GatherUint128(col.values[n:], rows.Uint128Array()) +} + +func (col *be128ColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.makeValue(&col.values[i]) + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +var ( + _ sort.Interface = (ColumnBuffer)(nil) + _ io.Writer = (*byteArrayColumnBuffer)(nil) + _ io.Writer = (*fixedLenByteArrayColumnBuffer)(nil) +) + +// writeRowsFunc is the type of functions that apply rows to a set of column +// buffers. +// +// - columns is the array of column buffer where the rows are written. +// +// - rows is the array of Go values to write to the column buffers. +// +// - levels is used to track the column index, repetition and definition levels +// of values when writing optional or repeated columns. +type writeRowsFunc func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error + +// writeRowsFuncOf generates a writeRowsFunc function for the given Go type and +// parquet schema. The column path indicates the column that the function is +// being generated for in the parquet schema. +func writeRowsFuncOf(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + if leaf, exists := schema.Lookup(path...); exists && leaf.Node.Type().LogicalType() != nil && leaf.Node.Type().LogicalType().Json != nil { + return writeRowsFuncOfJSON(t, schema, path) + } + + switch t { + case reflect.TypeOf(deprecated.Int96{}): + return writeRowsFuncOfRequired(t, schema, path) + case reflect.TypeOf(time.Time{}): + return writeRowsFuncOfTime(t, schema, path) + } + + switch t.Kind() { + case reflect.Bool, + reflect.Int, + reflect.Uint, + reflect.Int32, + reflect.Uint32, + reflect.Int64, + reflect.Uint64, + reflect.Float32, + reflect.Float64, + reflect.String: + return writeRowsFuncOfRequired(t, schema, path) + + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return writeRowsFuncOfRequired(t, schema, path) + } else { + return writeRowsFuncOfSlice(t, schema, path) + } + + case reflect.Array: + if t.Elem().Kind() == reflect.Uint8 { + return writeRowsFuncOfArray(t, schema, path) + } + + case reflect.Pointer: + return writeRowsFuncOfPointer(t, schema, path) + + case reflect.Struct: + return writeRowsFuncOfStruct(t, schema, path) + + case reflect.Map: + return writeRowsFuncOfMap(t, schema, path) + } + + panic("cannot convert Go values of type " + typeNameOf(t) + " to parquet value") +} + +func writeRowsFuncOfRequired(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + column := schema.lazyLoadState().mapping.lookup(path) + columnIndex := column.columnIndex + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + columns[columnIndex].writeValues(rows, levels) + return nil + } +} + +func writeRowsFuncOfOptional(t reflect.Type, schema *Schema, path columnPath, writeRows writeRowsFunc) writeRowsFunc { + if t.Kind() == reflect.Slice { // assume nested list + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeRows(columns, rows, levels) + } + levels.definitionLevel++ + return writeRows(columns, rows, levels) + } + } + nullIndex := nullIndexFuncOf(t) + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeRows(columns, rows, levels) + } + + nulls := acquireBitmap(rows.Len()) + defer releaseBitmap(nulls) + nullIndex(nulls.bits, rows) + + nullLevels := levels + levels.definitionLevel++ + // In this function, we are dealing with optional values which are + // neither pointers nor slices; for example, a int32 field marked + // "optional" in its parent struct. + // + // We need to find zero values, which should be represented as nulls + // in the parquet column. In order to minimize the calls to writeRows + // and maximize throughput, we use the nullIndex and nonNullIndex + // functions, which are type-specific implementations of the algorithm. + // + // Sections of the input that are contiguous nulls or non-nulls can be + // sent to a single call to writeRows to be written to the underlying + // buffer since they share the same definition level. + // + // This optimization is defeated by inputs alternating null and non-null + // sequences of single values, we do not expect this condition to be a + // common case. + for i := 0; i < rows.Len(); { + j := 0 + x := i / 64 + y := i % 64 + + if y != 0 { + if b := nulls.bits[x] >> uint(y); b == 0 { + x++ + y = 0 + } else { + y += bits.TrailingZeros64(b) + goto writeNulls + } + } + + for x < len(nulls.bits) && nulls.bits[x] == 0 { + x++ + } + + if x < len(nulls.bits) { + y = bits.TrailingZeros64(nulls.bits[x]) % 64 + } + + writeNulls: + if j = x*64 + y; j > rows.Len() { + j = rows.Len() + } + + if i < j { + if err := writeRows(columns, rows.Slice(i, j), nullLevels); err != nil { + return err + } + i = j + } + + if y != 0 { + if b := nulls.bits[x] >> uint(y); b == (1< rows.Len() { + j = rows.Len() + } + + if i < j { + if err := writeRows(columns, rows.Slice(i, j), levels); err != nil { + return err + } + i = j + } + } + + return nil + } +} + +func writeRowsFuncOfArray(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + column := schema.lazyLoadState().mapping.lookup(path) + arrayLen := t.Len() + columnLen := column.node.Type().Length() + if arrayLen != columnLen { + panic(fmt.Sprintf("cannot convert Go values of type "+typeNameOf(t)+" to FIXED_LEN_BYTE_ARRAY(%d)", columnLen)) + } + return writeRowsFuncOfRequired(t, schema, path) +} + +func writeRowsFuncOfPointer(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + elemType := t.Elem() + elemSize := uintptr(elemType.Size()) + writeRows := writeRowsFuncOf(elemType, schema, path) + + if len(path) == 0 { + // This code path is taken when generating a writeRowsFunc for a pointer + // type. In this case, we do not need to increase the definition level + // since we are not deailng with an optional field but a pointer to the + // row type. + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeRows(columns, rows, levels) + } + + for i := range rows.Len() { + p := *(*unsafe.Pointer)(rows.Index(i)) + a := sparse.Array{} + if p != nil { + a = makeArray(p, 1, elemSize) + } + if err := writeRows(columns, a, levels); err != nil { + return err + } + } + + return nil + } + } + + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeRows(columns, rows, levels) + } + + for i := range rows.Len() { + p := *(*unsafe.Pointer)(rows.Index(i)) + a := sparse.Array{} + elemLevels := levels + if p != nil { + a = makeArray(p, 1, elemSize) + elemLevels.definitionLevel++ + } + if err := writeRows(columns, a, elemLevels); err != nil { + return err + } + } + + return nil + } +} + +func writeRowsFuncOfSlice(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + elemType := t.Elem() + elemSize := uintptr(elemType.Size()) + writeRows := writeRowsFuncOf(elemType, schema, path) + + // When the element is a pointer type, the writeRows function will be an + // instance returned by writeRowsFuncOfPointer, which handles incrementing + // the definition level if the pointer value is not nil. + definitionLevelIncrement := byte(0) + if elemType.Kind() != reflect.Ptr { + definitionLevelIncrement = 1 + } + + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeRows(columns, rows, levels) + } + + levels.repetitionDepth++ + + for i := range rows.Len() { + p := (*sliceHeader)(rows.Index(i)) + a := makeArray(p.base, p.len, elemSize) + b := sparse.Array{} + + elemLevels := levels + if a.Len() > 0 { + b = a.Slice(0, 1) + elemLevels.definitionLevel += definitionLevelIncrement + } + + if err := writeRows(columns, b, elemLevels); err != nil { + return err + } + + if a.Len() > 1 { + elemLevels.repetitionLevel = elemLevels.repetitionDepth + + if err := writeRows(columns, a.Slice(1, a.Len()), elemLevels); err != nil { + return err + } + } + } + + return nil + } +} + +func writeRowsFuncOfStruct(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + type column struct { + offset uintptr + writeRows writeRowsFunc + } + + fields := structFieldsOf(t) + columns := make([]column, len(fields)) + + for i, f := range fields { + list, optional := false, false + columnPath := path.append(f.Name) + forEachStructTagOption(f, func(_ reflect.Type, option, _ string) { + switch option { + case "list": + list = true + columnPath = columnPath.append("list", "element") + case "optional": + optional = true + } + }) + + writeRows := writeRowsFuncOf(f.Type, schema, columnPath) + if optional { + kind := f.Type.Kind() + switch { + case kind == reflect.Pointer: + case kind == reflect.Slice && !list: + default: + writeRows = writeRowsFuncOfOptional(f.Type, schema, columnPath, writeRows) + } + } + + columns[i] = column{ + offset: f.Offset, + writeRows: writeRows, + } + } + + return func(buffers []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + for _, column := range columns { + if err := column.writeRows(buffers, rows, levels); err != nil { + return err + } + } + } else { + for _, column := range columns { + if err := column.writeRows(buffers, rows.Offset(column.offset), levels); err != nil { + return err + } + } + } + return nil + } +} + +func writeRowsFuncOfMap(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + keyPath := path.append("key_value", "key") + keyType := t.Key() + keySize := uintptr(keyType.Size()) + writeKeys := writeRowsFuncOf(keyType, schema, keyPath) + + valuePath := path.append("key_value", "value") + valueType := t.Elem() + valueSize := uintptr(valueType.Size()) + writeValues := writeRowsFuncOf(valueType, schema, valuePath) + + writeKeyValues := func(columns []ColumnBuffer, keys, values sparse.Array, levels columnLevels) error { + if err := writeKeys(columns, keys, levels); err != nil { + return err + } + if err := writeValues(columns, values, levels); err != nil { + return err + } + return nil + } + + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeKeyValues(columns, rows, rows, levels) + } + + levels.repetitionDepth++ + mapKey := reflect.New(keyType).Elem() + mapValue := reflect.New(valueType).Elem() + + for i := range rows.Len() { + m := reflect.NewAt(t, rows.Index(i)).Elem() + + if m.Len() == 0 { + empty := sparse.Array{} + if err := writeKeyValues(columns, empty, empty, levels); err != nil { + return err + } + } else { + elemLevels := levels + elemLevels.definitionLevel++ + + for it := m.MapRange(); it.Next(); { + mapKey.SetIterKey(it) + mapValue.SetIterValue(it) + + k := makeArray(reflectValueData(mapKey), 1, keySize) + v := makeArray(reflectValueData(mapValue), 1, valueSize) + + if err := writeKeyValues(columns, k, v, elemLevels); err != nil { + return err + } + + elemLevels.repetitionLevel = elemLevels.repetitionDepth + } + } + } + + return nil + } +} + +func writeRowsFuncOfJSON(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + // If this is a string or a byte array write directly. + switch t.Kind() { + case reflect.String: + return writeRowsFuncOfRequired(t, schema, path) + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return writeRowsFuncOfRequired(t, schema, path) + } + } + + // Otherwise handle with a json.Marshal + asStrT := reflect.TypeOf(string("")) + writer := writeRowsFuncOfRequired(asStrT, schema, path) + + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writer(columns, rows, levels) + } + for i := range rows.Len() { + val := reflect.NewAt(t, rows.Index(i)) + asI := val.Interface() + + b, err := json.Marshal(asI) + if err != nil { + return err + } + + asStr := string(b) + a := sparse.MakeStringArray([]string{asStr}) + if err := writer(columns, a.UnsafeArray(), levels); err != nil { + return err + } + } + return nil + } +} + +func writeRowsFuncOfTime(_ reflect.Type, schema *Schema, path columnPath) writeRowsFunc { + t := reflect.TypeOf(int64(0)) + elemSize := uintptr(t.Size()) + writeRows := writeRowsFuncOf(t, schema, path) + + col, _ := schema.Lookup(path...) + unit := Nanosecond.TimeUnit() + lt := col.Node.Type().LogicalType() + if lt != nil && lt.Timestamp != nil { + unit = lt.Timestamp.Unit + } + + return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error { + if rows.Len() == 0 { + return writeRows(columns, rows, levels) + } + + times := rows.TimeArray() + for i := range times.Len() { + t := times.Index(i) + var val int64 + switch { + case unit.Millis != nil: + val = t.UnixMilli() + case unit.Micros != nil: + val = t.UnixMicro() + default: + val = t.UnixNano() + } + + a := makeArray(reflectValueData(reflect.ValueOf(val)), 1, elemSize) + if err := writeRows(columns, a, levels); err != nil { + return err + } + } + + return nil + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.go b/vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.go new file mode 100644 index 0000000000..4571726996 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.go @@ -0,0 +1,30 @@ +//go:build !purego + +package parquet + +import ( + "github.com/parquet-go/parquet-go/internal/bytealg" + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" + "golang.org/x/sys/cpu" +) + +func broadcastValueInt32(dst []int32, src int8) { + bytealg.Broadcast(unsafecast.Slice[byte](dst), byte(src)) +} + +//go:noescape +func broadcastRangeInt32AVX2(dst []int32, base int32) + +func broadcastRangeInt32(dst []int32, base int32) { + if len(dst) >= 8 && cpu.X86.HasAVX2 { + broadcastRangeInt32AVX2(dst, base) + } else { + for i := range dst { + dst[i] = base + int32(i) + } + } +} + +//go:noescape +func writePointersBE128(values [][16]byte, rows sparse.Array) diff --git a/vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.s b/vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.s new file mode 100644 index 0000000000..1eef03d1d2 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_buffer_amd64.s @@ -0,0 +1,67 @@ +//go:build !purego + +#include "textflag.h" + +// func broadcastRangeInt32AVX2(dst []int32, base int32) +TEXT ยทbroadcastRangeInt32AVX2(SB), NOSPLIT, $0-28 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVL base+24(FP), CX + XORQ SI, SI + + CMPQ BX, $8 + JB test1x4 + + VMOVDQU ยทrange0n8(SB), Y0 // [0,1,2,3,4,5,6,7] + VPBROADCASTD ยทrange0n8+32(SB), Y1 // [8,8,8,8,8,8,8,8] + VPBROADCASTD base+24(FP), Y2 // [base...] + VPADDD Y2, Y0, Y0 // [base,base+1,...] + + MOVQ BX, DI + SHRQ $3, DI + SHLQ $3, DI + JMP test8x4 +loop8x4: + VMOVDQU Y0, (AX)(SI*4) + VPADDD Y1, Y0, Y0 + ADDQ $8, SI +test8x4: + CMPQ SI, DI + JNE loop8x4 + VZEROUPPER + JMP test1x4 + +loop1x4: + INCQ SI + MOVL CX, DX + IMULL SI, DX + MOVL DX, -4(AX)(SI*4) +test1x4: + CMPQ SI, BX + JNE loop1x4 + RET + +// func writePointersBE128(values [][16]byte, rows sparse.Array) +TEXT ยทwritePointersBE128(SB), NOSPLIT, $0-48 + MOVQ values_base+0(FP), AX + MOVQ rows_array_ptr+24(FP), BX + MOVQ rows_array_len+32(FP), CX + MOVQ rows_array_off+40(FP), DX + + XORQ SI, SI + JMP test +loop: + PXOR X0, X0 + MOVQ (BX), DI // *[16]byte + CMPQ DI, $0 + JE next + MOVOU (DI), X0 +next: + MOVOU X0, (AX) + ADDQ $16, AX + ADDQ DX, BX + INCQ SI +test: + CMPQ SI, CX + JNE loop + RET diff --git a/vendor/github.com/parquet-go/parquet-go/column_buffer_purego.go b/vendor/github.com/parquet-go/parquet-go/column_buffer_purego.go new file mode 100644 index 0000000000..6f9996f1cb --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_buffer_purego.go @@ -0,0 +1,30 @@ +//go:build !amd64 || purego + +package parquet + +import "github.com/parquet-go/parquet-go/sparse" + +func broadcastValueInt32(dst []int32, src int8) { + value := 0x01010101 * int32(src) + for i := range dst { + dst[i] = value + } +} + +func broadcastRangeInt32(dst []int32, base int32) { + for i := range dst { + dst[i] = base + int32(i) + } +} + +func writePointersBE128(values [][16]byte, rows sparse.Array) { + for i := range values { + p := *(**[16]byte)(rows.Index(i)) + + if p != nil { + values[i] = *p + } else { + values[i] = [16]byte{} + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_chunk.go b/vendor/github.com/parquet-go/parquet-go/column_chunk.go new file mode 100644 index 0000000000..56dd356800 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_chunk.go @@ -0,0 +1,339 @@ +package parquet + +import ( + "errors" + "io" +) + +var ( + ErrMissingBloomFilter = errors.New("missing bloom filter") + ErrMissingColumnIndex = errors.New("missing column index") + ErrMissingOffsetIndex = errors.New("missing offset index") +) + +// The ColumnChunk interface represents individual columns of a row group. +type ColumnChunk interface { + // Returns the column type. + Type() Type + + // Returns the index of this column in its parent row group. + Column() int + + // Returns a reader exposing the pages of the column. + Pages() Pages + + // Returns the components of the page index for this column chunk, + // containing details about the content and location of pages within the + // chunk. + // + // Note that the returned value may be the same across calls to these + // methods, programs must treat those as read-only. + // + // If the column chunk does not have a column or offset index, the methods return + // ErrMissingColumnIndex or ErrMissingOffsetIndex respectively. + // + // Prior to v0.20, these methods did not return an error because the page index + // for a file was either fully read when the file was opened, or skipped + // completely using the parquet.SkipPageIndex option. Version v0.20 introduced a + // change that the page index can be read on-demand at any time, even if a file + // was opened with the parquet.SkipPageIndex option. Since reading the page index + // can fail, these methods now return an error. + ColumnIndex() (ColumnIndex, error) + OffsetIndex() (OffsetIndex, error) + BloomFilter() BloomFilter + + // Returns the number of values in the column chunk. + // + // This quantity may differ from the number of rows in the parent row group + // because repeated columns may hold zero or more values per row. + NumValues() int64 +} + +// AsyncColumnChunk returns a ColumnChunk that reads pages asynchronously. +func AsyncColumnChunk(columnChunk ColumnChunk) ColumnChunk { + return &asyncColumnChunk{columnChunk} +} + +type asyncColumnChunk struct { + ColumnChunk +} + +func (c *asyncColumnChunk) Pages() Pages { + return AsyncPages(c.ColumnChunk.Pages()) +} + +// NewColumnChunkRowReader creates a new ColumnChunkRowReader for the given +// column chunks. +func NewColumnChunkRowReader(columns []ColumnChunk) RowReadSeekCloser { + return newRowGroupRows(nil, columns, defaultValueBufferSize) +} + +// ColumnChunkValueReader is an interface for reading values from a column chunk. +type ColumnChunkValueReader interface { + ValueReader + RowSeeker + io.Closer +} + +// NewColumnChunkValueReader creates a new ColumnChunkValueReader for the given +// column chunk. +func NewColumnChunkValueReader(column ColumnChunk) ColumnChunkValueReader { + return &columnChunkValueReader{pages: column.Pages(), release: Release} +} + +type columnChunkValueReader struct { + pages Pages + page Page + values ValueReader + release func(Page) +} + +func (r *columnChunkValueReader) clear() { + if r.page != nil { + r.release(r.page) + r.page = nil + r.values = nil + } +} + +func (r *columnChunkValueReader) Reset() { + if r.pages != nil { + // Ignore errors because we are resetting the reader, if the error + // persists we will see it on the next read, and otherwise we can + // read back from the beginning. + r.pages.SeekToRow(0) + } + r.clear() +} + +func (r *columnChunkValueReader) Close() error { + var err error + if r.pages != nil { + err = r.pages.Close() + r.pages = nil + } + r.clear() + return err +} + +func (r *columnChunkValueReader) ReadValues(values []Value) (int, error) { + if r.pages == nil { + return 0, io.EOF + } + + for { + if r.values == nil { + p, err := r.pages.ReadPage() + if err != nil { + return 0, err + } + r.page = p + r.values = p.Values() + } + + n, err := r.values.ReadValues(values) + if n > 0 { + return n, nil + } + if err == nil { + return 0, io.ErrNoProgress + } + if err != io.EOF { + return 0, err + } + r.clear() + } +} + +func (r *columnChunkValueReader) SeekToRow(rowIndex int64) error { + if r.pages == nil { + return io.ErrClosedPipe + } + if err := r.pages.SeekToRow(rowIndex); err != nil { + return err + } + r.clear() + return nil +} + +type pageAndValueWriter interface { + PageWriter + ValueWriter +} + +type readRowsFunc func(*rowGroupRows, []Row, byte) (int, error) + +func readRowsFuncOf(node Node, columnIndex int, repetitionDepth byte) (int, readRowsFunc) { + var read readRowsFunc + + if node.Repeated() { + repetitionDepth++ + } + + if node.Leaf() { + columnIndex, read = readRowsFuncOfLeaf(columnIndex, repetitionDepth) + } else { + columnIndex, read = readRowsFuncOfGroup(node, columnIndex, repetitionDepth) + } + + if node.Repeated() { + read = readRowsFuncOfRepeated(read, repetitionDepth) + } + + return columnIndex, read +} + +//go:noinline +func readRowsFuncOfRepeated(read readRowsFunc, repetitionDepth byte) readRowsFunc { + return func(r *rowGroupRows, rows []Row, repetitionLevel byte) (int, error) { + for i := range rows { + // Repeated columns have variable number of values, we must process + // them one row at a time because we cannot predict how many values + // need to be consumed in each iteration. + row := rows[i : i+1] + + // The first pass looks for values marking the beginning of a row by + // having a repetition level equal to the current level. + n, err := read(r, row, repetitionLevel) + if err != nil { + // The error here may likely be io.EOF, the read function may + // also have successfully read a row, which is indicated by a + // non-zero count. In this case, we increment the index to + // indicate to the caller than rows up to i+1 have been read. + if n > 0 { + i++ + } + return i, err + } + + // The read function may return no errors and also read no rows in + // case where it had more values to read but none corresponded to + // the current repetition level. This is an indication that we will + // not be able to read more rows at this stage, we must return to + // the caller to let it set the repetition level to its current + // depth, which may allow us to read more values when called again. + if n == 0 { + return i, nil + } + + // When we reach this stage, we have successfully read the first + // values of a row of repeated columns. We continue consuming more + // repeated values until we get the indication that we consumed + // them all (the read function returns zero and no errors). + for { + n, err := read(r, row, repetitionDepth) + if err != nil { + return i + 1, err + } + if n == 0 { + break + } + } + } + return len(rows), nil + } +} + +//go:noinline +func readRowsFuncOfGroup(node Node, columnIndex int, repetitionDepth byte) (int, readRowsFunc) { + fields := node.Fields() + + if len(fields) == 0 { + return columnIndex, func(*rowGroupRows, []Row, byte) (int, error) { + return 0, io.EOF + } + } + + if len(fields) == 1 { + // Small optimization for a somewhat common case of groups with a single + // column (like nested list elements for example); there is no need to + // loop over the group of a single element, we can simply skip to calling + // the inner read function. + return readRowsFuncOf(fields[0], columnIndex, repetitionDepth) + } + + group := make([]readRowsFunc, len(fields)) + for i := range group { + columnIndex, group[i] = readRowsFuncOf(fields[i], columnIndex, repetitionDepth) + } + + return columnIndex, func(r *rowGroupRows, rows []Row, repetitionLevel byte) (int, error) { + // When reading a group, we use the first column as an indicator of how + // may rows can be read during this call. + n, err := group[0](r, rows, repetitionLevel) + + if n > 0 { + // Read values for all rows that the group is able to consume. + // Getting io.EOF from calling the read functions indicate that + // we consumed all values of that particular column, but there may + // be more to read in other columns, therefore we must always read + // all columns and cannot stop on the first error. + for _, read := range group[1:] { + _, err2 := read(r, rows[:n], repetitionLevel) + if err2 != nil && err2 != io.EOF { + return 0, err2 + } + } + } + + return n, err + } +} + +//go:noinline +func readRowsFuncOfLeaf(columnIndex int, repetitionDepth byte) (int, readRowsFunc) { + var read readRowsFunc + + if repetitionDepth == 0 { + read = func(r *rowGroupRows, rows []Row, _ byte) (int, error) { + // When the repetition depth is zero, we know that there is exactly + // one value per row for this column, and therefore we can consume + // as many values as there are rows to fill. + col := &r.columns[columnIndex] + buf := r.buffer(columnIndex) + + for i := range rows { + if col.offset == col.length { + n, err := col.reader.ReadValues(buf) + col.offset = 0 + col.length = int32(n) + if n == 0 && err != nil { + return 0, err + } + } + + rows[i] = append(rows[i], buf[col.offset]) + col.offset++ + } + + return len(rows), nil + } + } else { + read = func(r *rowGroupRows, rows []Row, repetitionLevel byte) (int, error) { + // When the repetition depth is not zero, we know that we will be + // called with a single row as input. We attempt to read at most one + // value of a single row and return to the caller. + col := &r.columns[columnIndex] + buf := r.buffer(columnIndex) + + if col.offset == col.length { + n, err := col.reader.ReadValues(buf) + col.offset = 0 + col.length = int32(n) + if n == 0 && err != nil { + return 0, err + } + } + + if buf[col.offset].repetitionLevel != repetitionLevel { + return 0, nil + } + + rows[0] = append(rows[0], buf[col.offset]) + col.offset++ + return 1, nil + } + } + + return columnIndex + 1, read +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_index.go b/vendor/github.com/parquet-go/parquet-go/column_index.go new file mode 100644 index 0000000000..81e0c4f6cc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_index.go @@ -0,0 +1,754 @@ +package parquet + +import ( + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding/plain" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type ColumnIndex interface { + // NumPages returns the number of paged in the column index. + NumPages() int + + // Returns the number of null values in the page at the given index. + NullCount(int) int64 + + // Tells whether the page at the given index contains null values only. + NullPage(int) bool + + // PageIndex return min/max bounds for the page at the given index in the + // column. + MinValue(int) Value + MaxValue(int) Value + + // IsAscending returns true if the column index min/max values are sorted + // in ascending order (based on the ordering rules of the column's logical + // type). + IsAscending() bool + + // IsDescending returns true if the column index min/max values are sorted + // in descending order (based on the ordering rules of the column's logical + // type). + IsDescending() bool +} + +// NewColumnIndex constructs a ColumnIndex instance from the given parquet +// format column index. The kind argument configures the type of values +func NewColumnIndex(kind Kind, index *format.ColumnIndex) ColumnIndex { + return &formatColumnIndex{ + kind: kind, + index: index, + } +} + +type formatColumnIndex struct { + kind Kind + index *format.ColumnIndex +} + +func (f *formatColumnIndex) NumPages() int { + return len(f.index.MinValues) +} + +func (f *formatColumnIndex) NullCount(i int) int64 { + if len(f.index.NullCounts) > 0 { + return f.index.NullCounts[i] + } + return 0 +} + +func (f *formatColumnIndex) NullPage(i int) bool { + return len(f.index.NullPages) > 0 && f.index.NullPages[i] +} + +func (f *formatColumnIndex) MinValue(i int) Value { + if f.NullPage(i) { + return Value{} + } + return f.kind.Value(f.index.MinValues[i]) +} + +func (f *formatColumnIndex) MaxValue(i int) Value { + if f.NullPage(i) { + return Value{} + } + return f.kind.Value(f.index.MaxValues[i]) +} + +func (f *formatColumnIndex) IsAscending() bool { + return f.index.BoundaryOrder == format.Ascending +} + +func (f *formatColumnIndex) IsDescending() bool { + return f.index.BoundaryOrder == format.Descending +} + +type FileColumnIndex struct { + index *format.ColumnIndex + kind Kind +} + +func (i *FileColumnIndex) NumPages() int { + return len(i.index.NullPages) +} + +func (i *FileColumnIndex) NullCount(j int) int64 { + if len(i.index.NullCounts) > 0 { + return i.index.NullCounts[j] + } + return 0 +} + +func (i *FileColumnIndex) NullPage(j int) bool { + return isNullPage(j, i.index) +} + +func (i *FileColumnIndex) MinValue(j int) Value { + if i.NullPage(j) { + return Value{} + } + return i.makeValue(i.index.MinValues[j]) +} + +func (i *FileColumnIndex) MaxValue(j int) Value { + if i.NullPage(j) { + return Value{} + } + return i.makeValue(i.index.MaxValues[j]) +} + +func (i *FileColumnIndex) IsAscending() bool { + return i.index.BoundaryOrder == format.Ascending +} + +func (i *FileColumnIndex) IsDescending() bool { + return i.index.BoundaryOrder == format.Descending +} + +func (i *FileColumnIndex) makeValue(b []byte) Value { + return i.kind.Value(b) +} + +func isNullPage(j int, index *format.ColumnIndex) bool { + return len(index.NullPages) > 0 && index.NullPages[j] +} + +type emptyColumnIndex struct{} + +func (emptyColumnIndex) NumPages() int { return 0 } +func (emptyColumnIndex) NullCount(int) int64 { return 0 } +func (emptyColumnIndex) NullPage(int) bool { return false } +func (emptyColumnIndex) MinValue(int) Value { return Value{} } +func (emptyColumnIndex) MaxValue(int) Value { return Value{} } +func (emptyColumnIndex) IsAscending() bool { return false } +func (emptyColumnIndex) IsDescending() bool { return false } + +type booleanColumnIndex struct{ page *booleanPage } + +func (i booleanColumnIndex) NumPages() int { return 1 } +func (i booleanColumnIndex) NullCount(int) int64 { return 0 } +func (i booleanColumnIndex) NullPage(int) bool { return false } +func (i booleanColumnIndex) MinValue(int) Value { return makeValueBoolean(i.page.min()) } +func (i booleanColumnIndex) MaxValue(int) Value { return makeValueBoolean(i.page.max()) } +func (i booleanColumnIndex) IsAscending() bool { return false } +func (i booleanColumnIndex) IsDescending() bool { return false } + +type int32ColumnIndex struct{ page *int32Page } + +func (i int32ColumnIndex) NumPages() int { return 1 } +func (i int32ColumnIndex) NullCount(int) int64 { return 0 } +func (i int32ColumnIndex) NullPage(int) bool { return false } +func (i int32ColumnIndex) MinValue(int) Value { return makeValueInt32(i.page.min()) } +func (i int32ColumnIndex) MaxValue(int) Value { return makeValueInt32(i.page.max()) } +func (i int32ColumnIndex) IsAscending() bool { return false } +func (i int32ColumnIndex) IsDescending() bool { return false } + +type int64ColumnIndex struct{ page *int64Page } + +func (i int64ColumnIndex) NumPages() int { return 1 } +func (i int64ColumnIndex) NullCount(int) int64 { return 0 } +func (i int64ColumnIndex) NullPage(int) bool { return false } +func (i int64ColumnIndex) MinValue(int) Value { return makeValueInt64(i.page.min()) } +func (i int64ColumnIndex) MaxValue(int) Value { return makeValueInt64(i.page.max()) } +func (i int64ColumnIndex) IsAscending() bool { return false } +func (i int64ColumnIndex) IsDescending() bool { return false } + +type int96ColumnIndex struct{ page *int96Page } + +func (i int96ColumnIndex) NumPages() int { return 1 } +func (i int96ColumnIndex) NullCount(int) int64 { return 0 } +func (i int96ColumnIndex) NullPage(int) bool { return false } +func (i int96ColumnIndex) MinValue(int) Value { return makeValueInt96(i.page.min()) } +func (i int96ColumnIndex) MaxValue(int) Value { return makeValueInt96(i.page.max()) } +func (i int96ColumnIndex) IsAscending() bool { return false } +func (i int96ColumnIndex) IsDescending() bool { return false } + +type floatColumnIndex struct{ page *floatPage } + +func (i floatColumnIndex) NumPages() int { return 1 } +func (i floatColumnIndex) NullCount(int) int64 { return 0 } +func (i floatColumnIndex) NullPage(int) bool { return false } +func (i floatColumnIndex) MinValue(int) Value { return makeValueFloat(i.page.min()) } +func (i floatColumnIndex) MaxValue(int) Value { return makeValueFloat(i.page.max()) } +func (i floatColumnIndex) IsAscending() bool { return false } +func (i floatColumnIndex) IsDescending() bool { return false } + +type doubleColumnIndex struct{ page *doublePage } + +func (i doubleColumnIndex) NumPages() int { return 1 } +func (i doubleColumnIndex) NullCount(int) int64 { return 0 } +func (i doubleColumnIndex) NullPage(int) bool { return false } +func (i doubleColumnIndex) MinValue(int) Value { return makeValueDouble(i.page.min()) } +func (i doubleColumnIndex) MaxValue(int) Value { return makeValueDouble(i.page.max()) } +func (i doubleColumnIndex) IsAscending() bool { return false } +func (i doubleColumnIndex) IsDescending() bool { return false } + +type byteArrayColumnIndex struct{ page *byteArrayPage } + +func (i byteArrayColumnIndex) NumPages() int { return 1 } +func (i byteArrayColumnIndex) NullCount(int) int64 { return 0 } +func (i byteArrayColumnIndex) NullPage(int) bool { return false } +func (i byteArrayColumnIndex) MinValue(int) Value { return makeValueBytes(ByteArray, i.page.min()) } +func (i byteArrayColumnIndex) MaxValue(int) Value { return makeValueBytes(ByteArray, i.page.max()) } +func (i byteArrayColumnIndex) IsAscending() bool { return false } +func (i byteArrayColumnIndex) IsDescending() bool { return false } + +type fixedLenByteArrayColumnIndex struct{ page *fixedLenByteArrayPage } + +func (i fixedLenByteArrayColumnIndex) NumPages() int { return 1 } +func (i fixedLenByteArrayColumnIndex) NullCount(int) int64 { return 0 } +func (i fixedLenByteArrayColumnIndex) NullPage(int) bool { return false } +func (i fixedLenByteArrayColumnIndex) MinValue(int) Value { + return makeValueBytes(FixedLenByteArray, i.page.min()) +} +func (i fixedLenByteArrayColumnIndex) MaxValue(int) Value { + return makeValueBytes(FixedLenByteArray, i.page.max()) +} +func (i fixedLenByteArrayColumnIndex) IsAscending() bool { return false } +func (i fixedLenByteArrayColumnIndex) IsDescending() bool { return false } + +type uint32ColumnIndex struct{ page *uint32Page } + +func (i uint32ColumnIndex) NumPages() int { return 1 } +func (i uint32ColumnIndex) NullCount(int) int64 { return 0 } +func (i uint32ColumnIndex) NullPage(int) bool { return false } +func (i uint32ColumnIndex) MinValue(int) Value { return makeValueUint32(i.page.min()) } +func (i uint32ColumnIndex) MaxValue(int) Value { return makeValueUint32(i.page.max()) } +func (i uint32ColumnIndex) IsAscending() bool { return false } +func (i uint32ColumnIndex) IsDescending() bool { return false } + +type uint64ColumnIndex struct{ page *uint64Page } + +func (i uint64ColumnIndex) NumPages() int { return 1 } +func (i uint64ColumnIndex) NullCount(int) int64 { return 0 } +func (i uint64ColumnIndex) NullPage(int) bool { return false } +func (i uint64ColumnIndex) MinValue(int) Value { return makeValueUint64(i.page.min()) } +func (i uint64ColumnIndex) MaxValue(int) Value { return makeValueUint64(i.page.max()) } +func (i uint64ColumnIndex) IsAscending() bool { return false } +func (i uint64ColumnIndex) IsDescending() bool { return false } + +type be128ColumnIndex struct{ page *be128Page } + +func (i be128ColumnIndex) NumPages() int { return 1 } +func (i be128ColumnIndex) NullCount(int) int64 { return 0 } +func (i be128ColumnIndex) NullPage(int) bool { return false } +func (i be128ColumnIndex) MinValue(int) Value { return makeValueBytes(FixedLenByteArray, i.page.min()) } +func (i be128ColumnIndex) MaxValue(int) Value { return makeValueBytes(FixedLenByteArray, i.page.max()) } +func (i be128ColumnIndex) IsAscending() bool { return false } +func (i be128ColumnIndex) IsDescending() bool { return false } + +// The ColumnIndexer interface is implemented by types that support generating +// parquet column indexes. +// +// The package does not export any types that implement this interface, programs +// must call NewColumnIndexer on a Type instance to construct column indexers. +type ColumnIndexer interface { + // Resets the column indexer state. + Reset() + + // Add a page to the column indexer. + IndexPage(numValues, numNulls int64, min, max Value) + + // Generates a format.ColumnIndex value from the current state of the + // column indexer. + // + // The returned value may reference internal buffers, in which case the + // values remain valid until the next call to IndexPage or Reset on the + // column indexer. + ColumnIndex() format.ColumnIndex +} + +type baseColumnIndexer struct { + nullPages []bool + nullCounts []int64 +} + +func (i *baseColumnIndexer) reset() { + i.nullPages = i.nullPages[:0] + i.nullCounts = i.nullCounts[:0] +} + +func (i *baseColumnIndexer) observe(numValues, numNulls int64) { + i.nullPages = append(i.nullPages, numValues == numNulls) + i.nullCounts = append(i.nullCounts, numNulls) +} + +func (i *baseColumnIndexer) columnIndex(minValues, maxValues [][]byte, minOrder, maxOrder int) format.ColumnIndex { + nullPages := make([]bool, len(i.nullPages)) + copy(nullPages, i.nullPages) + nullCounts := make([]int64, len(i.nullCounts)) + copy(nullCounts, i.nullCounts) + return format.ColumnIndex{ + NullPages: nullPages, + NullCounts: nullCounts, + MinValues: minValues, + MaxValues: maxValues, + BoundaryOrder: boundaryOrderOf(minOrder, maxOrder), + } +} + +type booleanColumnIndexer struct { + baseColumnIndexer + minValues []bool + maxValues []bool +} + +func newBooleanColumnIndexer() *booleanColumnIndexer { + return new(booleanColumnIndexer) +} + +func (i *booleanColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *booleanColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.boolean()) + i.maxValues = append(i.maxValues, max.boolean()) +} + +func (i *booleanColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(unsafecast.Slice[byte](i.minValues), 1), + splitFixedLenByteArrays(unsafecast.Slice[byte](i.maxValues), 1), + orderOfBool(i.minValues), + orderOfBool(i.maxValues), + ) +} + +type int32ColumnIndexer struct { + baseColumnIndexer + minValues []int32 + maxValues []int32 +} + +func newInt32ColumnIndexer() *int32ColumnIndexer { + return new(int32ColumnIndexer) +} + +func (i *int32ColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *int32ColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.int32()) + i.maxValues = append(i.maxValues, max.int32()) +} + +func (i *int32ColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexInt32Values(i.minValues), 4), + splitFixedLenByteArrays(columnIndexInt32Values(i.maxValues), 4), + orderOfInt32(i.minValues), + orderOfInt32(i.maxValues), + ) +} + +type int64ColumnIndexer struct { + baseColumnIndexer + minValues []int64 + maxValues []int64 +} + +func newInt64ColumnIndexer() *int64ColumnIndexer { + return new(int64ColumnIndexer) +} + +func (i *int64ColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *int64ColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.int64()) + i.maxValues = append(i.maxValues, max.int64()) +} + +func (i *int64ColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexInt64Values(i.minValues), 8), + splitFixedLenByteArrays(columnIndexInt64Values(i.maxValues), 8), + orderOfInt64(i.minValues), + orderOfInt64(i.maxValues), + ) +} + +type int96ColumnIndexer struct { + baseColumnIndexer + minValues []deprecated.Int96 + maxValues []deprecated.Int96 +} + +func newInt96ColumnIndexer() *int96ColumnIndexer { + return new(int96ColumnIndexer) +} + +func (i *int96ColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *int96ColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.Int96()) + i.maxValues = append(i.maxValues, max.Int96()) +} + +func (i *int96ColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexInt96Values(i.minValues), 12), + splitFixedLenByteArrays(columnIndexInt96Values(i.maxValues), 12), + deprecated.OrderOfInt96(i.minValues), + deprecated.OrderOfInt96(i.maxValues), + ) +} + +type floatColumnIndexer struct { + baseColumnIndexer + minValues []float32 + maxValues []float32 +} + +func newFloatColumnIndexer() *floatColumnIndexer { + return new(floatColumnIndexer) +} + +func (i *floatColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *floatColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.float()) + i.maxValues = append(i.maxValues, max.float()) +} + +func (i *floatColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexFloatValues(i.minValues), 4), + splitFixedLenByteArrays(columnIndexFloatValues(i.maxValues), 4), + orderOfFloat32(i.minValues), + orderOfFloat32(i.maxValues), + ) +} + +type doubleColumnIndexer struct { + baseColumnIndexer + minValues []float64 + maxValues []float64 +} + +func newDoubleColumnIndexer() *doubleColumnIndexer { + return new(doubleColumnIndexer) +} + +func (i *doubleColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *doubleColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.double()) + i.maxValues = append(i.maxValues, max.double()) +} + +func (i *doubleColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexDoubleValues(i.minValues), 8), + splitFixedLenByteArrays(columnIndexDoubleValues(i.maxValues), 8), + orderOfFloat64(i.minValues), + orderOfFloat64(i.maxValues), + ) +} + +type byteArrayColumnIndexer struct { + baseColumnIndexer + sizeLimit int + minValues []byte + maxValues []byte +} + +func newByteArrayColumnIndexer(sizeLimit int) *byteArrayColumnIndexer { + return &byteArrayColumnIndexer{sizeLimit: sizeLimit} +} + +func (i *byteArrayColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *byteArrayColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = plain.AppendByteArray(i.minValues, min.byteArray()) + i.maxValues = plain.AppendByteArray(i.maxValues, max.byteArray()) +} + +func (i *byteArrayColumnIndexer) ColumnIndex() format.ColumnIndex { + minValues := splitByteArrays(i.minValues) + maxValues := splitByteArrays(i.maxValues) + if sizeLimit := i.sizeLimit; sizeLimit > 0 { + for i, v := range minValues { + minValues[i] = truncateLargeMinByteArrayValue(v, sizeLimit) + } + for i, v := range maxValues { + maxValues[i] = truncateLargeMaxByteArrayValue(v, sizeLimit) + } + } + return i.columnIndex( + minValues, + maxValues, + orderOfBytes(minValues), + orderOfBytes(maxValues), + ) +} + +type fixedLenByteArrayColumnIndexer struct { + baseColumnIndexer + size int + sizeLimit int + minValues []byte + maxValues []byte +} + +func newFixedLenByteArrayColumnIndexer(size, sizeLimit int) *fixedLenByteArrayColumnIndexer { + return &fixedLenByteArrayColumnIndexer{ + size: size, + sizeLimit: sizeLimit, + } +} + +func (i *fixedLenByteArrayColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *fixedLenByteArrayColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.byteArray()...) + i.maxValues = append(i.maxValues, max.byteArray()...) +} + +func (i *fixedLenByteArrayColumnIndexer) ColumnIndex() format.ColumnIndex { + minValues := splitFixedLenByteArrays(i.minValues, i.size) + maxValues := splitFixedLenByteArrays(i.maxValues, i.size) + if sizeLimit := i.sizeLimit; sizeLimit > 0 { + for i, v := range minValues { + minValues[i] = truncateLargeMinByteArrayValue(v, sizeLimit) + } + for i, v := range maxValues { + maxValues[i] = truncateLargeMaxByteArrayValue(v, sizeLimit) + } + } + return i.columnIndex( + minValues, + maxValues, + orderOfBytes(minValues), + orderOfBytes(maxValues), + ) +} + +type uint32ColumnIndexer struct { + baseColumnIndexer + minValues []uint32 + maxValues []uint32 +} + +func newUint32ColumnIndexer() *uint32ColumnIndexer { + return new(uint32ColumnIndexer) +} + +func (i *uint32ColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *uint32ColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.uint32()) + i.maxValues = append(i.maxValues, max.uint32()) +} + +func (i *uint32ColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexUint32Values(i.minValues), 4), + splitFixedLenByteArrays(columnIndexUint32Values(i.maxValues), 4), + orderOfUint32(i.minValues), + orderOfUint32(i.maxValues), + ) +} + +type uint64ColumnIndexer struct { + baseColumnIndexer + minValues []uint64 + maxValues []uint64 +} + +func newUint64ColumnIndexer() *uint64ColumnIndexer { + return new(uint64ColumnIndexer) +} + +func (i *uint64ColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *uint64ColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + i.minValues = append(i.minValues, min.uint64()) + i.maxValues = append(i.maxValues, max.uint64()) +} + +func (i *uint64ColumnIndexer) ColumnIndex() format.ColumnIndex { + return i.columnIndex( + splitFixedLenByteArrays(columnIndexUint64Values(i.minValues), 8), + splitFixedLenByteArrays(columnIndexUint64Values(i.maxValues), 8), + orderOfUint64(i.minValues), + orderOfUint64(i.maxValues), + ) +} + +type be128ColumnIndexer struct { + baseColumnIndexer + minValues [][16]byte + maxValues [][16]byte +} + +func newBE128ColumnIndexer() *be128ColumnIndexer { + return new(be128ColumnIndexer) +} + +func (i *be128ColumnIndexer) Reset() { + i.reset() + i.minValues = i.minValues[:0] + i.maxValues = i.maxValues[:0] +} + +func (i *be128ColumnIndexer) IndexPage(numValues, numNulls int64, min, max Value) { + i.observe(numValues, numNulls) + if !min.IsNull() { + i.minValues = append(i.minValues, *(*[16]byte)(min.byteArray())) + } + if !max.IsNull() { + i.maxValues = append(i.maxValues, *(*[16]byte)(max.byteArray())) + } +} + +func (i *be128ColumnIndexer) ColumnIndex() format.ColumnIndex { + minValues := splitFixedLenByteArrays(unsafecast.Slice[byte](i.minValues), 16) + maxValues := splitFixedLenByteArrays(unsafecast.Slice[byte](i.maxValues), 16) + return i.columnIndex( + minValues, + maxValues, + orderOfBytes(minValues), + orderOfBytes(maxValues), + ) +} + +func truncateLargeMinByteArrayValue(value []byte, sizeLimit int) []byte { + if len(value) > sizeLimit { + value = value[:sizeLimit] + } + return value +} + +// truncateLargeMaxByteArrayValue truncates the given byte array to the given size limit. +// If the given byte array is truncated, it is incremented by 1 in place. +func truncateLargeMaxByteArrayValue(value []byte, sizeLimit int) []byte { + if len(value) > sizeLimit { + value = value[:sizeLimit] + incrementByteArrayInplace(value) + } + return value +} + +// incrementByteArray increments the given byte array by 1. +// Reference: https://github.com/apache/parquet-java/blob/master/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryTruncator.java#L124 +func incrementByteArrayInplace(value []byte) { + for i := len(value) - 1; i >= 0; i-- { + value[i]++ + if value[i] != 0 { // Did not overflow: 0xFF -> 0x00 + return + } + } + // Fully overflowed, so restore all to 0xFF + for i := range value { + value[i] = 0xFF + } +} + +func splitByteArrays(data []byte) [][]byte { + length := 0 + plain.RangeByteArray(data, func([]byte) error { + length++ + return nil + }) + buffer := make([]byte, 0, len(data)-(4*length)) + values := make([][]byte, 0, length) + plain.RangeByteArray(data, func(value []byte) error { + offset := len(buffer) + buffer = append(buffer, value...) + values = append(values, buffer[offset:]) + return nil + }) + return values +} + +func splitFixedLenByteArrays(data []byte, size int) [][]byte { + data = copyBytes(data) + values := make([][]byte, len(data)/size) + for i := range values { + j := (i + 0) * size + k := (i + 1) * size + values[i] = data[j:k:k] + } + return values +} + +func boundaryOrderOf(minOrder, maxOrder int) format.BoundaryOrder { + if minOrder == maxOrder { + switch { + case minOrder > 0: + return format.Ascending + case minOrder < 0: + return format.Descending + } + } + return format.Unordered +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_index_be.go b/vendor/github.com/parquet-go/parquet-go/column_index_be.go new file mode 100644 index 0000000000..ace82ae320 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_index_be.go @@ -0,0 +1,84 @@ +// This file gets added on all the big-endian CPU architectures. + +//go:build armbe || arm64be || m68k || mips || mips64 || mips64p32 || ppc || ppc64 || s390 || s390x || shbe || sparc || sparc64 + +package parquet + +import ( + "encoding/binary" + "math" + + "github.com/parquet-go/parquet-go/deprecated" +) + +func columnIndexInt32Values(values []int32) []byte { + buf := make([]byte, len(values)*4) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint32(buf[idx:(4+idx)], uint32(values[k])) + idx += 4 + } + return buf +} + +func columnIndexInt64Values(values []int64) []byte { + buf := make([]byte, len(values)*8) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint64(buf[idx:(8+idx)], uint64(values[k])) + idx += 8 + } + return buf +} + +func columnIndexInt96Values(values []deprecated.Int96) []byte { + buf := make([]byte, len(values)*12) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint32(buf[idx:(4+idx)], uint32(values[k][0])) + binary.LittleEndian.PutUint32(buf[(4+idx):(8+idx)], uint32(values[k][1])) + binary.LittleEndian.PutUint32(buf[(8+idx):(12+idx)], uint32(values[k][2])) + idx += 12 + } + return buf +} + +func columnIndexFloatValues(values []float32) []byte { + buf := make([]byte, len(values)*4) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint32(buf[idx:(4+idx)], math.Float32bits(values[k])) + idx += 4 + } + return buf +} + +func columnIndexDoubleValues(values []float64) []byte { + buf := make([]byte, len(values)*8) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint64(buf[idx:(8+idx)], math.Float64bits(values[k])) + idx += 8 + } + return buf +} + +func columnIndexUint32Values(values []uint32) []byte { + buf := make([]byte, len(values)*4) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint32(buf[idx:(4+idx)], values[k]) + idx += 4 + } + return buf +} + +func columnIndexUint64Values(values []uint64) []byte { + buf := make([]byte, len(values)*8) + idx := 0 + for k := range len(values) { + binary.LittleEndian.PutUint64(buf[idx:(8+idx)], values[k]) + idx += 8 + } + return buf +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_index_le.go b/vendor/github.com/parquet-go/parquet-go/column_index_le.go new file mode 100644 index 0000000000..6b6ac8f30b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_index_le.go @@ -0,0 +1,38 @@ +// This file gets added on all the little-endian CPU architectures. + +//go:build 386 || amd64 || amd64p32 || alpha || arm || arm64 || loong64 || mipsle || mips64le || mips64p32le || nios2 || ppc64le || riscv || riscv64 || sh || wasm + +package parquet + +import ( + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +func columnIndexInt32Values(values []int32) []byte { + return unsafecast.Slice[byte](values) +} + +func columnIndexInt64Values(values []int64) []byte { + return unsafecast.Slice[byte](values) +} + +func columnIndexInt96Values(values []deprecated.Int96) []byte { + return unsafecast.Slice[byte](values) +} + +func columnIndexFloatValues(values []float32) []byte { + return unsafecast.Slice[byte](values) +} + +func columnIndexDoubleValues(values []float64) []byte { + return unsafecast.Slice[byte](values) +} + +func columnIndexUint32Values(values []uint32) []byte { + return unsafecast.Slice[byte](values) +} + +func columnIndexUint64Values(values []uint64) []byte { + return unsafecast.Slice[byte](values) +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_mapping.go b/vendor/github.com/parquet-go/parquet-go/column_mapping.go new file mode 100644 index 0000000000..a9345adb15 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_mapping.go @@ -0,0 +1,88 @@ +package parquet + +// LeafColumn is a struct type representing leaf columns of a parquet schema. +type LeafColumn struct { + Node Node + Path []string + ColumnIndex int + MaxRepetitionLevel int + MaxDefinitionLevel int +} + +func columnMappingOf(schema Node) (mapping columnMappingGroup, columns [][]string) { + mapping = make(columnMappingGroup) + columns = make([][]string, 0, 16) + + forEachLeafColumnOf(schema, func(leaf leafColumn) { + path := make(columnPath, len(leaf.path)) + copy(path, leaf.path) + columns = append(columns, path) + + group := mapping + for len(path) > 1 { + columnName := path[0] + g, ok := group[columnName].(columnMappingGroup) + if !ok { + g = make(columnMappingGroup) + group[columnName] = g + } + group, path = g, path[1:] + } + + leaf.path = path // use the copy + group[path[0]] = &columnMappingLeaf{column: leaf} + }) + + return mapping, columns +} + +type columnMapping interface { + lookup(path columnPath) leafColumn +} + +type columnMappingGroup map[string]columnMapping + +func (group columnMappingGroup) lookup(path columnPath) leafColumn { + if len(path) > 0 { + c, ok := group[path[0]] + if ok { + return c.lookup(path[1:]) + } + } + return leafColumn{columnIndex: -1} +} + +func (group columnMappingGroup) lookupClosest(path columnPath) leafColumn { + for len(path) > 0 { + g, ok := group[path[0]].(columnMappingGroup) + if ok { + group, path = g, path[1:] + } else { + firstName := "" + firstLeaf := (*columnMappingLeaf)(nil) + for name, child := range group { + if leaf, ok := child.(*columnMappingLeaf); ok { + if firstLeaf == nil || name < firstName { + firstName, firstLeaf = name, leaf + } + } + } + if firstLeaf != nil { + return firstLeaf.column + } + break + } + } + return leafColumn{columnIndex: -1} +} + +type columnMappingLeaf struct { + column leafColumn +} + +func (leaf *columnMappingLeaf) lookup(path columnPath) leafColumn { + if len(path) == 0 { + return leaf.column + } + return leafColumn{columnIndex: -1} +} diff --git a/vendor/github.com/parquet-go/parquet-go/column_path.go b/vendor/github.com/parquet-go/parquet-go/column_path.go new file mode 100644 index 0000000000..d32fa7cd15 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/column_path.go @@ -0,0 +1,108 @@ +package parquet + +import ( + "slices" + "strings" +) + +type columnPath []string + +func (path columnPath) append(names ...string) columnPath { + return slices.Concat(path, names) +} + +func (path columnPath) equal(other columnPath) bool { + return stringsAreEqual(path, other) +} + +func (path columnPath) less(other columnPath) bool { + return stringsAreOrdered(path, other) +} + +func (path columnPath) String() string { + return strings.Join(path, ".") +} + +func stringsAreEqual(strings1, strings2 []string) bool { + if len(strings1) != len(strings2) { + return false + } + + for i := range strings1 { + if strings1[i] != strings2[i] { + return false + } + } + + return true +} + +func stringsAreOrdered(strings1, strings2 []string) bool { + n := min(len(strings1), len(strings2)) + + for i := range n { + if strings1[i] >= strings2[i] { + return false + } + } + + return len(strings1) <= len(strings2) +} + +type leafColumn struct { + node Node + path columnPath + maxRepetitionLevel byte + maxDefinitionLevel byte + columnIndex int16 +} + +func forEachLeafColumnOf(node Node, do func(leafColumn)) { + forEachLeafColumn(node, nil, 0, 0, 0, do) +} + +func forEachLeafColumn(node Node, path columnPath, columnIndex, maxRepetitionLevel, maxDefinitionLevel int, do func(leafColumn)) int { + switch { + case node.Optional(): + maxDefinitionLevel++ + case node.Repeated(): + maxRepetitionLevel++ + maxDefinitionLevel++ + } + + if node.Leaf() { + do(leafColumn{ + node: node, + path: path, + maxRepetitionLevel: makeRepetitionLevel(maxRepetitionLevel), + maxDefinitionLevel: makeDefinitionLevel(maxDefinitionLevel), + columnIndex: makeColumnIndex(columnIndex), + }) + return columnIndex + 1 + } + + for _, field := range node.Fields() { + columnIndex = forEachLeafColumn( + field, + path.append(field.Name()), + columnIndex, + maxRepetitionLevel, + maxDefinitionLevel, + do, + ) + } + + return columnIndex +} + +func lookupColumnPath(node Node, path columnPath) Node { + for node != nil && len(path) > 0 { + node = fieldByName(node, path[0]) + path = path[1:] + } + return node +} + +func hasColumnPath(node Node, path columnPath) bool { + return lookupColumnPath(node, path) != nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/compare.go b/vendor/github.com/parquet-go/parquet-go/compare.go new file mode 100644 index 0000000000..b503eab805 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compare.go @@ -0,0 +1,351 @@ +package parquet + +import ( + "encoding/binary" + "sync" + + "github.com/parquet-go/parquet-go/deprecated" +) + +// CompareDescending constructs a comparison function which inverses the order +// of values. +// +//go:noinline +func CompareDescending(cmp func(Value, Value) int) func(Value, Value) int { + return func(a, b Value) int { return -cmp(a, b) } +} + +// CompareNullsFirst constructs a comparison function which assumes that null +// values are smaller than all other values. +// +//go:noinline +func CompareNullsFirst(cmp func(Value, Value) int) func(Value, Value) int { + return func(a, b Value) int { + switch { + case a.IsNull(): + if b.IsNull() { + return 0 + } + return -1 + case b.IsNull(): + return +1 + default: + return cmp(a, b) + } + } +} + +// CompareNullsLast constructs a comparison function which assumes that null +// values are greater than all other values. +// +//go:noinline +func CompareNullsLast(cmp func(Value, Value) int) func(Value, Value) int { + return func(a, b Value) int { + switch { + case a.IsNull(): + if b.IsNull() { + return 0 + } + return +1 + case b.IsNull(): + return -1 + default: + return cmp(a, b) + } + } +} + +func compareBool(v1, v2 bool) int { + switch { + case !v1 && v2: + return -1 + case v1 && !v2: + return +1 + default: + return 0 + } +} + +func compareInt32(v1, v2 int32) int { + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + default: + return 0 + } +} + +func compareInt64(v1, v2 int64) int { + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + default: + return 0 + } +} + +func compareInt96(v1, v2 deprecated.Int96) int { + switch { + case v1.Less(v2): + return -1 + case v2.Less(v1): + return +1 + default: + return 0 + } +} + +func compareFloat32(v1, v2 float32) int { + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + default: + return 0 + } +} + +func compareFloat64(v1, v2 float64) int { + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + default: + return 0 + } +} + +func compareUint32(v1, v2 uint32) int { + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + default: + return 0 + } +} + +func compareUint64(v1, v2 uint64) int { + switch { + case v1 < v2: + return -1 + case v1 > v2: + return +1 + default: + return 0 + } +} + +func compareBE128(v1, v2 *[16]byte) int { + x := binary.BigEndian.Uint64(v1[:8]) + y := binary.BigEndian.Uint64(v2[:8]) + switch { + case x < y: + return -1 + case x > y: + return +1 + } + x = binary.BigEndian.Uint64(v1[8:]) + y = binary.BigEndian.Uint64(v2[8:]) + switch { + case x < y: + return -1 + case x > y: + return +1 + default: + return 0 + } +} + +func lessBE128(v1, v2 *[16]byte) bool { + x := binary.BigEndian.Uint64(v1[:8]) + y := binary.BigEndian.Uint64(v2[:8]) + switch { + case x < y: + return true + case x > y: + return false + } + x = binary.BigEndian.Uint64(v1[8:]) + y = binary.BigEndian.Uint64(v2[8:]) + return x < y +} + +func compareRowsFuncOf(schema *Schema, sortingColumns []SortingColumn) func(Row, Row) int { + leafColumns := make([]leafColumn, len(sortingColumns)) + canCompareRows := true + + forEachLeafColumnOf(schema, func(leaf leafColumn) { + if leaf.maxRepetitionLevel > 0 { + canCompareRows = false + } + + if sortingIndex := searchSortingColumn(sortingColumns, leaf.path); sortingIndex < len(sortingColumns) { + leafColumns[sortingIndex] = leaf + + if leaf.maxDefinitionLevel > 0 { + canCompareRows = false + } + } + }) + + // This is an optimization for the common case where rows + // are sorted by non-optional, non-repeated columns. + // + // The sort function can make the assumption that it will + // find the column value at the current column index, and + // does not need to scan the rows looking for values with + // a matching column index. + if canCompareRows { + return compareRowsFuncOfColumnIndexes(leafColumns, sortingColumns) + } + + return compareRowsFuncOfColumnValues(leafColumns, sortingColumns) +} + +func compareRowsUnordered(Row, Row) int { return 0 } + +//go:noinline +func compareRowsFuncOfIndexColumns(compareFuncs []func(Row, Row) int) func(Row, Row) int { + return func(row1, row2 Row) int { + for _, compare := range compareFuncs { + if cmp := compare(row1, row2); cmp != 0 { + return cmp + } + } + return 0 + } +} + +//go:noinline +func compareRowsFuncOfIndexAscending(columnIndex int16, typ Type) func(Row, Row) int { + return func(row1, row2 Row) int { return typ.Compare(row1[columnIndex], row2[columnIndex]) } +} + +//go:noinline +func compareRowsFuncOfIndexDescending(columnIndex int16, typ Type) func(Row, Row) int { + return func(row1, row2 Row) int { return -typ.Compare(row1[columnIndex], row2[columnIndex]) } +} + +//go:noinline +func compareRowsFuncOfColumnIndexes(leafColumns []leafColumn, sortingColumns []SortingColumn) func(Row, Row) int { + compareFuncs := make([]func(Row, Row) int, len(sortingColumns)) + + for sortingIndex, sortingColumn := range sortingColumns { + leaf := leafColumns[sortingIndex] + typ := leaf.node.Type() + + if sortingColumn.Descending() { + compareFuncs[sortingIndex] = compareRowsFuncOfIndexDescending(leaf.columnIndex, typ) + } else { + compareFuncs[sortingIndex] = compareRowsFuncOfIndexAscending(leaf.columnIndex, typ) + } + } + + switch len(compareFuncs) { + case 0: + return compareRowsUnordered + case 1: + return compareFuncs[0] + default: + return compareRowsFuncOfIndexColumns(compareFuncs) + } +} + +var columnPool = &sync.Pool{New: func() any { return make([][2]int32, 0, 128) }} + +//go:noinline +func compareRowsFuncOfColumnValues(leafColumns []leafColumn, sortingColumns []SortingColumn) func(Row, Row) int { + highestColumnIndex := int16(0) + columnIndexes := make([]int16, len(sortingColumns)) + compareFuncs := make([]func(Value, Value) int, len(sortingColumns)) + + for sortingIndex, sortingColumn := range sortingColumns { + leaf := leafColumns[sortingIndex] + compare := leaf.node.Type().Compare + + if sortingColumn.Descending() { + compare = CompareDescending(compare) + } + + if leaf.maxDefinitionLevel > 0 { + if sortingColumn.NullsFirst() { + compare = CompareNullsFirst(compare) + } else { + compare = CompareNullsLast(compare) + } + } + + columnIndexes[sortingIndex] = leaf.columnIndex + compareFuncs[sortingIndex] = compare + + if leaf.columnIndex > highestColumnIndex { + highestColumnIndex = leaf.columnIndex + } + } + + return func(row1, row2 Row) int { + columns1 := columnPool.Get().([][2]int32) + columns2 := columnPool.Get().([][2]int32) + defer func() { + columns1 = columns1[:0] + columns2 = columns2[:0] + columnPool.Put(columns1) + columnPool.Put(columns2) + }() + + i1 := 0 + i2 := 0 + + for columnIndex := int16(0); columnIndex <= highestColumnIndex; columnIndex++ { + j1 := i1 + 1 + j2 := i2 + 1 + + for j1 < len(row1) && row1[j1].columnIndex == ^columnIndex { + j1++ + } + + for j2 < len(row2) && row2[j2].columnIndex == ^columnIndex { + j2++ + } + + columns1 = append(columns1, [2]int32{int32(i1), int32(j1)}) + columns2 = append(columns2, [2]int32{int32(i2), int32(j2)}) + i1 = j1 + i2 = j2 + } + + for i, compare := range compareFuncs { + columnIndex := columnIndexes[i] + offsets1 := columns1[columnIndex] + offsets2 := columns2[columnIndex] + values1 := row1[offsets1[0]:offsets1[1]:offsets1[1]] + values2 := row2[offsets2[0]:offsets2[1]:offsets2[1]] + i1 := 0 + i2 := 0 + + for i1 < len(values1) && i2 < len(values2) { + if cmp := compare(values1[i1], values2[i2]); cmp != 0 { + return cmp + } + i1++ + i2++ + } + + if i1 < len(values1) { + return +1 + } + if i2 < len(values2) { + return -1 + } + } + return 0 + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress.go b/vendor/github.com/parquet-go/parquet-go/compress.go new file mode 100644 index 0000000000..c2b2eb130f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress.go @@ -0,0 +1,96 @@ +package parquet + +import ( + "fmt" + + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/compress/brotli" + "github.com/parquet-go/parquet-go/compress/gzip" + "github.com/parquet-go/parquet-go/compress/lz4" + "github.com/parquet-go/parquet-go/compress/snappy" + "github.com/parquet-go/parquet-go/compress/uncompressed" + "github.com/parquet-go/parquet-go/compress/zstd" + "github.com/parquet-go/parquet-go/format" +) + +var ( + // Uncompressed is a parquet compression codec representing uncompressed + // pages. + Uncompressed uncompressed.Codec + + // Snappy is the SNAPPY parquet compression codec. + Snappy snappy.Codec + + // Gzip is the GZIP parquet compression codec. + Gzip = gzip.Codec{ + Level: gzip.DefaultCompression, + } + + // Brotli is the BROTLI parquet compression codec. + Brotli = brotli.Codec{ + Quality: brotli.DefaultQuality, + LGWin: brotli.DefaultLGWin, + } + + // Zstd is the ZSTD parquet compression codec. + Zstd = zstd.Codec{ + Level: zstd.DefaultLevel, + } + + // Lz4Raw is the LZ4_RAW parquet compression codec. + Lz4Raw = lz4.Codec{ + Level: lz4.DefaultLevel, + } + + // Table of compression codecs indexed by their code in the parquet format. + compressionCodecs = [...]compress.Codec{ + format.Uncompressed: &Uncompressed, + format.Snappy: &Snappy, + format.Gzip: &Gzip, + format.Brotli: &Brotli, + format.Zstd: &Zstd, + format.Lz4Raw: &Lz4Raw, + } +) + +// LookupCompressionCodec returns the compression codec associated with the +// given code. +// +// The function never returns nil. If the encoding is not supported, +// an "unsupported" codec is returned. +func LookupCompressionCodec(codec format.CompressionCodec) compress.Codec { + if codec >= 0 && int(codec) < len(compressionCodecs) { + if c := compressionCodecs[codec]; c != nil { + return c + } + } + return &unsupported{codec} +} + +type unsupported struct { + codec format.CompressionCodec +} + +func (u *unsupported) String() string { + return "UNSUPPORTED" +} + +func (u *unsupported) CompressionCodec() format.CompressionCodec { + return u.codec +} + +func (u *unsupported) Encode(dst, src []byte) ([]byte, error) { + return dst[:0], u.error() +} + +func (u *unsupported) Decode(dst, src []byte) ([]byte, error) { + return dst[:0], u.error() +} + +func (u *unsupported) error() error { + return fmt.Errorf("unsupported compression codec: %s", u.codec) +} + +func isCompressed(c compress.Codec) bool { + return c != nil && c.CompressionCodec() != format.Uncompressed +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress/brotli/brotli.go b/vendor/github.com/parquet-go/parquet-go/compress/brotli/brotli.go new file mode 100644 index 0000000000..db7c0bfdf2 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/brotli/brotli.go @@ -0,0 +1,54 @@ +// Package brotli implements the BROTLI parquet compression codec. +package brotli + +import ( + "io" + + "github.com/andybalholm/brotli" + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/format" +) + +const ( + DefaultQuality = 0 + DefaultLGWin = 0 +) + +type Codec struct { + // Quality controls the compression-speed vs compression-density trade-offs. + // The higher the quality, the slower the compression. Range is 0 to 11. + Quality int + // LGWin is the base 2 logarithm of the sliding window size. + // Range is 10 to 24. 0 indicates automatic configuration based on Quality. + LGWin int + + r compress.Decompressor + w compress.Compressor +} + +func (c *Codec) String() string { + return "BROTLI" +} + +func (c *Codec) CompressionCodec() format.CompressionCodec { + return format.Brotli +} + +func (c *Codec) Encode(dst, src []byte) ([]byte, error) { + return c.w.Encode(dst, src, func(w io.Writer) (compress.Writer, error) { + return brotli.NewWriterOptions(w, brotli.WriterOptions{ + Quality: c.Quality, + LGWin: c.LGWin, + }), nil + }) +} + +func (c *Codec) Decode(dst, src []byte) ([]byte, error) { + return c.r.Decode(dst, src, func(r io.Reader) (compress.Reader, error) { + return reader{brotli.NewReader(r)}, nil + }) +} + +type reader struct{ *brotli.Reader } + +func (reader) Close() error { return nil } diff --git a/vendor/github.com/parquet-go/parquet-go/compress/compress.go b/vendor/github.com/parquet-go/parquet-go/compress/compress.go new file mode 100644 index 0000000000..5cfac8bfb2 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/compress.go @@ -0,0 +1,142 @@ +// Package compress provides the generic APIs implemented by parquet compression +// codecs. +// +// https://github.com/apache/parquet-format/blob/master/Compression.md +package compress + +import ( + "bytes" + "io" + "sync" + + "github.com/parquet-go/parquet-go/format" +) + +// The Codec interface represents parquet compression codecs implemented by the +// compress sub-packages. +// +// Codec instances must be safe to use concurrently from multiple goroutines. +type Codec interface { + // Returns a human-readable name for the codec. + String() string + + // Returns the code of the compression codec in the parquet format. + CompressionCodec() format.CompressionCodec + + // Writes the compressed version of src to dst and returns it. + // + // The method automatically reallocates the output buffer if its capacity + // was too small to hold the compressed data. + Encode(dst, src []byte) ([]byte, error) + + // Writes the uncompressed version of src to dst and returns it. + // + // The method automatically reallocates the output buffer if its capacity + // was too small to hold the uncompressed data. + Decode(dst, src []byte) ([]byte, error) +} + +type Reader interface { + io.ReadCloser + Reset(io.Reader) error +} + +type Writer interface { + io.WriteCloser + Reset(io.Writer) +} + +type Compressor struct { + writers sync.Pool // *writer +} + +type writer struct { + output bytes.Buffer + writer Writer +} + +func (c *Compressor) Encode(dst, src []byte, newWriter func(io.Writer) (Writer, error)) ([]byte, error) { + w, _ := c.writers.Get().(*writer) + if w != nil { + w.output = *bytes.NewBuffer(dst[:0]) + w.writer.Reset(&w.output) + } else { + w = new(writer) + w.output = *bytes.NewBuffer(dst[:0]) + var err error + if w.writer, err = newWriter(&w.output); err != nil { + return dst, err + } + } + + defer func() { + w.output = *bytes.NewBuffer(nil) + w.writer.Reset(io.Discard) + c.writers.Put(w) + }() + + if _, err := w.writer.Write(src); err != nil { + return w.output.Bytes(), err + } + if err := w.writer.Close(); err != nil { + return w.output.Bytes(), err + } + return w.output.Bytes(), nil +} + +type Decompressor struct { + readers sync.Pool // *reader +} + +type reader struct { + input bytes.Reader + reader Reader +} + +func (d *Decompressor) Decode(dst, src []byte, newReader func(io.Reader) (Reader, error)) ([]byte, error) { + r, _ := d.readers.Get().(*reader) + if r != nil { + r.input.Reset(src) + if err := r.reader.Reset(&r.input); err != nil { + return dst, err + } + } else { + r = new(reader) + r.input.Reset(src) + var err error + if r.reader, err = newReader(&r.input); err != nil { + return dst, err + } + } + + defer func() { + r.input.Reset(nil) + if err := r.reader.Reset(nil); err == nil { + d.readers.Put(r) + } + }() + + if cap(dst) == 0 { + dst = make([]byte, 0, 2*len(src)) + } else { + dst = dst[:0] + } + + for { + n, err := r.reader.Read(dst[len(dst):cap(dst)]) + dst = dst[:len(dst)+n] + + if err != nil { + if err == io.EOF { + err = nil + } + return dst, err + } + + if len(dst) == cap(dst) { + tmp := make([]byte, len(dst), 2*len(dst)) + copy(tmp, dst) + dst = tmp + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress/gzip/gzip.go b/vendor/github.com/parquet-go/parquet-go/compress/gzip/gzip.go new file mode 100644 index 0000000000..99780a9509 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/gzip/gzip.go @@ -0,0 +1,67 @@ +// Package gzip implements the GZIP parquet compression codec. +package gzip + +import ( + "io" + "strings" + + "github.com/klauspost/compress/gzip" + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/format" +) + +const ( + emptyGzip = "\x1f\x8b\b\x00\x00\x00\x00\x00\x02\xff\x01\x00\x00\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00" +) + +const ( + NoCompression = gzip.NoCompression + BestSpeed = gzip.BestSpeed + BestCompression = gzip.BestCompression + DefaultCompression = gzip.DefaultCompression + HuffmanOnly = gzip.HuffmanOnly +) + +type Codec struct { + Level int + + r compress.Decompressor + w compress.Compressor +} + +func (c *Codec) String() string { + return "GZIP" +} + +func (c *Codec) CompressionCodec() format.CompressionCodec { + return format.Gzip +} + +func (c *Codec) Encode(dst, src []byte) ([]byte, error) { + return c.w.Encode(dst, src, func(w io.Writer) (compress.Writer, error) { + return gzip.NewWriterLevel(w, c.Level) + }) +} + +func (c *Codec) Decode(dst, src []byte) ([]byte, error) { + return c.r.Decode(dst, src, func(r io.Reader) (compress.Reader, error) { + z, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + return &reader{Reader: z}, nil + }) +} + +type reader struct { + *gzip.Reader + emptyGzip strings.Reader +} + +func (r *reader) Reset(rr io.Reader) error { + if rr == nil { + r.emptyGzip.Reset(emptyGzip) + rr = &r.emptyGzip + } + return r.Reader.Reset(rr) +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress/lz4/lz4.go b/vendor/github.com/parquet-go/parquet-go/compress/lz4/lz4.go new file mode 100644 index 0000000000..975cfed8cc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/lz4/lz4.go @@ -0,0 +1,87 @@ +// Package lz4 implements the LZ4_RAW parquet compression codec. +package lz4 + +import ( + "github.com/parquet-go/parquet-go/format" + "github.com/pierrec/lz4/v4" +) + +type Level = lz4.CompressionLevel + +const ( + Fastest = lz4.CompressionLevel(99) + Fast = lz4.Fast + Level1 = lz4.Level1 + Level2 = lz4.Level2 + Level3 = lz4.Level3 + Level4 = lz4.Level4 + Level5 = lz4.Level5 + Level6 = lz4.Level6 + Level7 = lz4.Level7 + Level8 = lz4.Level8 + Level9 = lz4.Level9 +) + +const ( + DefaultLevel = Fast +) + +type Codec struct { + Level Level +} + +func (c *Codec) String() string { + return "LZ4_RAW" +} + +func (c *Codec) CompressionCodec() format.CompressionCodec { + return format.Lz4Raw +} + +func (c *Codec) Encode(dst, src []byte) ([]byte, error) { + dst = reserveAtLeast(dst, lz4.CompressBlockBound(len(src))) + + var ( + n int + err error + ) + if c.Level == Fastest { + compressor := lz4.Compressor{} + n, err = compressor.CompressBlock(src, dst) + } else { + compressor := lz4.CompressorHC{Level: c.Level} + n, err = compressor.CompressBlock(src, dst) + } + return dst[:n], err +} + +func (c *Codec) Decode(dst, src []byte) ([]byte, error) { + // 3x seems like a common compression ratio, so we optimistically size the + // output buffer to that size. Feel free to change the value if you observe + // different behaviors. + dst = reserveAtLeast(dst, 3*len(src)) + + for { + n, err := lz4.UncompressBlock(src, dst) + // The lz4 package does not expose the error values, they are declared + // in internal/lz4errors. Based on what I read of the implementation, + // the only condition where this function errors is if the output buffer + // was too short. + // + // https://github.com/pierrec/lz4/blob/a5532e5996ee86d17f8ce2694c08fb5bf3c6b471/internal/lz4block/block.go#L45-L53 + if err != nil { + dst = make([]byte, 2*len(dst)) + } else { + return dst[:n], nil + } + } +} + +func reserveAtLeast(b []byte, n int) []byte { + if cap(b) < n { + b = make([]byte, n) + } else { + b = b[:cap(b)] + } + return b +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress/snappy/snappy.go b/vendor/github.com/parquet-go/parquet-go/compress/snappy/snappy.go new file mode 100644 index 0000000000..eb3febda42 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/snappy/snappy.go @@ -0,0 +1,31 @@ +// Package snappy implements the SNAPPY parquet compression codec. +package snappy + +import ( + "github.com/klauspost/compress/snappy" + "github.com/parquet-go/parquet-go/format" +) + +type Codec struct { +} + +// The snappy.Reader and snappy.Writer implement snappy encoding/decoding with +// a framing protocol, but snappy requires the implementation to use the raw +// snappy block encoding. This is why we need to use snappy.Encode/snappy.Decode +// and have to ship custom implementations of the compressed reader and writer. + +func (c *Codec) String() string { + return "SNAPPY" +} + +func (c *Codec) CompressionCodec() format.CompressionCodec { + return format.Snappy +} + +func (c *Codec) Encode(dst, src []byte) ([]byte, error) { + return snappy.Encode(dst, src), nil +} + +func (c *Codec) Decode(dst, src []byte) ([]byte, error) { + return snappy.Decode(dst, src) +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress/uncompressed/uncompressed.go b/vendor/github.com/parquet-go/parquet-go/compress/uncompressed/uncompressed.go new file mode 100644 index 0000000000..2aa0b3538f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/uncompressed/uncompressed.go @@ -0,0 +1,27 @@ +// Package uncompressed provides implementations of the compression codec +// interfaces as pass-through without applying any compression nor +// decompression. +package uncompressed + +import ( + "github.com/parquet-go/parquet-go/format" +) + +type Codec struct { +} + +func (c *Codec) String() string { + return "UNCOMPRESSED" +} + +func (c *Codec) CompressionCodec() format.CompressionCodec { + return format.Uncompressed +} + +func (c *Codec) Encode(dst, src []byte) ([]byte, error) { + return append(dst[:0], src...), nil +} + +func (c *Codec) Decode(dst, src []byte) ([]byte, error) { + return append(dst[:0], src...), nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/compress/zstd/zstd.go b/vendor/github.com/parquet-go/parquet-go/compress/zstd/zstd.go new file mode 100644 index 0000000000..4c2a4e3eae --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/compress/zstd/zstd.go @@ -0,0 +1,102 @@ +// Package zstd implements the ZSTD parquet compression codec. +package zstd + +import ( + "sync" + + "github.com/klauspost/compress/zstd" + "github.com/parquet-go/parquet-go/format" +) + +type Level = zstd.EncoderLevel + +const ( + // SpeedFastest will choose the fastest reasonable compression. + // This is roughly equivalent to the fastest Zstandard mode. + SpeedFastest = zstd.SpeedFastest + + // SpeedDefault is the default "pretty fast" compression option. + // This is roughly equivalent to the default Zstandard mode (level 3). + SpeedDefault = zstd.SpeedDefault + + // SpeedBetterCompression will yield better compression than the default. + // Currently it is about zstd level 7-8 with ~ 2x-3x the default CPU usage. + // By using this, notice that CPU usage may go up in the future. + SpeedBetterCompression = zstd.SpeedBetterCompression + + // SpeedBestCompression will choose the best available compression option. + // This will offer the best compression no matter the CPU cost. + SpeedBestCompression = zstd.SpeedBestCompression +) + +const ( + DefaultLevel = SpeedDefault + + DefaultConcurrency = 1 +) + +type Codec struct { + Level Level + + // Concurrency is the number of CPU cores to use for encoding and decoding. + // If Concurrency is 0, it will use DefaultConcurrency. + Concurrency uint + + encoders sync.Pool // *zstd.Encoder + decoders sync.Pool // *zstd.Decoder +} + +func (c *Codec) String() string { + return "ZSTD" +} + +func (c *Codec) CompressionCodec() format.CompressionCodec { + return format.Zstd +} + +func (c *Codec) Encode(dst, src []byte) ([]byte, error) { + e, _ := c.encoders.Get().(*zstd.Encoder) + if e == nil { + var err error + e, err = zstd.NewWriter(nil, + zstd.WithEncoderConcurrency(c.concurrency()), + zstd.WithEncoderLevel(c.level()), + zstd.WithZeroFrames(true), + zstd.WithEncoderCRC(false), + ) + if err != nil { + return dst[:0], err + } + } + defer c.encoders.Put(e) + return e.EncodeAll(src, dst[:0]), nil +} + +func (c *Codec) Decode(dst, src []byte) ([]byte, error) { + d, _ := c.decoders.Get().(*zstd.Decoder) + if d == nil { + var err error + d, err = zstd.NewReader(nil, + zstd.WithDecoderConcurrency(c.concurrency()), + ) + if err != nil { + return dst[:0], err + } + } + defer c.decoders.Put(d) + return d.DecodeAll(src, dst[:0]) +} + +func (c *Codec) level() Level { + if c.Level != 0 { + return c.Level + } + return DefaultLevel +} + +func (c *Codec) concurrency() int { + if c.Concurrency != 0 { + return int(c.Concurrency) + } + return DefaultConcurrency +} diff --git a/vendor/github.com/parquet-go/parquet-go/config.go b/vendor/github.com/parquet-go/parquet-go/config.go new file mode 100644 index 0000000000..a1082e5771 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/config.go @@ -0,0 +1,879 @@ +package parquet + +import ( + "fmt" + "math" + "runtime/debug" + "strings" + "sync" + + "github.com/parquet-go/parquet-go/compress" + "slices" +) + +// ReadMode is an enum that is used to configure the way that a File reads pages. +type ReadMode int + +const ( + ReadModeSync ReadMode = iota // ReadModeSync reads pages synchronously on demand (Default). + ReadModeAsync // ReadModeAsync reads pages asynchronously in the background. +) + +const ( + DefaultColumnIndexSizeLimit = 16 + DefaultColumnBufferCapacity = 16 * 1024 + DefaultPageBufferSize = 256 * 1024 + DefaultWriteBufferSize = 32 * 1024 + DefaultDataPageVersion = 2 + DefaultDataPageStatistics = false + DefaultSkipMagicBytes = false + DefaultSkipPageIndex = false + DefaultSkipBloomFilters = false + DefaultMaxRowsPerRowGroup = math.MaxInt64 + DefaultReadMode = ReadModeSync +) + +const ( + parquetGoModulePath = "github.com/parquet-go/parquet-go" +) + +var ( + defaultCreatedByInfo string + defaultCreatedByOnce sync.Once +) + +func defaultCreatedBy() string { + defaultCreatedByOnce.Do(func() { + createdBy := parquetGoModulePath + build, ok := debug.ReadBuildInfo() + if ok { + for _, mod := range build.Deps { + if mod.Replace == nil && mod.Path == parquetGoModulePath { + semver, _, buildsha := parseModuleVersion(mod.Version) + createdBy = formatCreatedBy(createdBy, semver, buildsha) + break + } + } + } + defaultCreatedByInfo = createdBy + }) + return defaultCreatedByInfo +} + +func parseModuleVersion(version string) (semver, datetime, buildsha string) { + semver, version = splitModuleVersion(version) + datetime, version = splitModuleVersion(version) + buildsha, _ = splitModuleVersion(version) + semver = strings.TrimPrefix(semver, "v") + return +} + +func splitModuleVersion(s string) (head, tail string) { + if i := strings.IndexByte(s, '-'); i < 0 { + head = s + } else { + head, tail = s[:i], s[i+1:] + } + return +} + +func formatCreatedBy(application, version, build string) string { + return application + " version " + version + "(build " + build + ")" +} + +// The FileConfig type carries configuration options for parquet files. +// +// FileConfig implements the FileOption interface so it can be used directly +// as argument to the OpenFile function when needed, for example: +// +// f, err := parquet.OpenFile(reader, size, &parquet.FileConfig{ +// SkipPageIndex: true, +// SkipBloomFilters: true, +// ReadMode: ReadModeAsync, +// }) +type FileConfig struct { + SkipMagicBytes bool + SkipPageIndex bool + SkipBloomFilters bool + OptimisticRead bool + ReadBufferSize int + ReadMode ReadMode + Schema *Schema +} + +// DefaultFileConfig returns a new FileConfig value initialized with the +// default file configuration. +func DefaultFileConfig() *FileConfig { + return &FileConfig{ + SkipMagicBytes: DefaultSkipMagicBytes, + SkipPageIndex: DefaultSkipPageIndex, + SkipBloomFilters: DefaultSkipBloomFilters, + ReadBufferSize: defaultReadBufferSize, + ReadMode: DefaultReadMode, + Schema: nil, + } +} + +// NewFileConfig constructs a new file configuration applying the options passed +// as arguments. +// +// The function returns an non-nil error if some of the options carried invalid +// configuration values. +func NewFileConfig(options ...FileOption) (*FileConfig, error) { + config := DefaultFileConfig() + config.Apply(options...) + return config, config.Validate() +} + +// Apply applies the given list of options to c. +func (c *FileConfig) Apply(options ...FileOption) { + for _, opt := range options { + opt.ConfigureFile(c) + } +} + +// ConfigureFile applies configuration options from c to config. +func (c *FileConfig) ConfigureFile(config *FileConfig) { + *config = FileConfig{ + SkipMagicBytes: c.SkipMagicBytes, + SkipPageIndex: c.SkipPageIndex, + SkipBloomFilters: c.SkipBloomFilters, + ReadBufferSize: coalesceInt(c.ReadBufferSize, config.ReadBufferSize), + ReadMode: ReadMode(coalesceInt(int(c.ReadMode), int(config.ReadMode))), + Schema: coalesceSchema(c.Schema, config.Schema), + } +} + +// Validate returns a non-nil error if the configuration of c is invalid. +func (c *FileConfig) Validate() error { + return nil +} + +// The ReaderConfig type carries configuration options for parquet readers. +// +// ReaderConfig implements the ReaderOption interface so it can be used directly +// as argument to the NewReader function when needed, for example: +// +// reader := parquet.NewReader(output, schema, &parquet.ReaderConfig{ +// // ... +// }) +type ReaderConfig struct { + Schema *Schema +} + +// DefaultReaderConfig returns a new ReaderConfig value initialized with the +// default reader configuration. +func DefaultReaderConfig() *ReaderConfig { + return &ReaderConfig{} +} + +// NewReaderConfig constructs a new reader configuration applying the options +// passed as arguments. +// +// The function returns an non-nil error if some of the options carried invalid +// configuration values. +func NewReaderConfig(options ...ReaderOption) (*ReaderConfig, error) { + config := DefaultReaderConfig() + config.Apply(options...) + return config, config.Validate() +} + +// Apply applies the given list of options to c. +func (c *ReaderConfig) Apply(options ...ReaderOption) { + for _, opt := range options { + opt.ConfigureReader(c) + } +} + +// ConfigureReader applies configuration options from c to config. +func (c *ReaderConfig) ConfigureReader(config *ReaderConfig) { + *config = ReaderConfig{ + Schema: coalesceSchema(c.Schema, config.Schema), + } +} + +// Validate returns a non-nil error if the configuration of c is invalid. +func (c *ReaderConfig) Validate() error { + return nil +} + +// The WriterConfig type carries configuration options for parquet writers. +// +// WriterConfig implements the WriterOption interface so it can be used directly +// as argument to the NewWriter function when needed, for example: +// +// writer := parquet.NewWriter(output, schema, &parquet.WriterConfig{ +// CreatedBy: "my test program", +// }) +type WriterConfig struct { + CreatedBy string + ColumnPageBuffers BufferPool + ColumnIndexSizeLimit int + PageBufferSize int + WriteBufferSize int + DataPageVersion int + DataPageStatistics bool + MaxRowsPerRowGroup int64 + KeyValueMetadata map[string]string + Schema *Schema + BloomFilters []BloomFilterColumn + Compression compress.Codec + Sorting SortingConfig + SkipPageBounds [][]string +} + +// DefaultWriterConfig returns a new WriterConfig value initialized with the +// default writer configuration. +func DefaultWriterConfig() *WriterConfig { + return &WriterConfig{ + CreatedBy: defaultCreatedBy(), + ColumnPageBuffers: &defaultColumnBufferPool, + ColumnIndexSizeLimit: DefaultColumnIndexSizeLimit, + PageBufferSize: DefaultPageBufferSize, + WriteBufferSize: DefaultWriteBufferSize, + DataPageVersion: DefaultDataPageVersion, + DataPageStatistics: DefaultDataPageStatistics, + MaxRowsPerRowGroup: DefaultMaxRowsPerRowGroup, + Sorting: SortingConfig{ + SortingBuffers: &defaultSortingBufferPool, + }, + } +} + +// NewWriterConfig constructs a new writer configuration applying the options +// passed as arguments. +// +// The function returns an non-nil error if some of the options carried invalid +// configuration values. +func NewWriterConfig(options ...WriterOption) (*WriterConfig, error) { + config := DefaultWriterConfig() + config.Apply(options...) + return config, config.Validate() +} + +// Apply applies the given list of options to c. +func (c *WriterConfig) Apply(options ...WriterOption) { + for _, opt := range options { + opt.ConfigureWriter(c) + } +} + +// ConfigureWriter applies configuration options from c to config. +func (c *WriterConfig) ConfigureWriter(config *WriterConfig) { + keyValueMetadata := config.KeyValueMetadata + if len(c.KeyValueMetadata) > 0 { + if keyValueMetadata == nil { + keyValueMetadata = make(map[string]string, len(c.KeyValueMetadata)) + } + for k, v := range c.KeyValueMetadata { + keyValueMetadata[k] = v + } + } + + *config = WriterConfig{ + CreatedBy: coalesceString(c.CreatedBy, config.CreatedBy), + ColumnPageBuffers: coalesceBufferPool(c.ColumnPageBuffers, config.ColumnPageBuffers), + ColumnIndexSizeLimit: coalesceInt(c.ColumnIndexSizeLimit, config.ColumnIndexSizeLimit), + PageBufferSize: coalesceInt(c.PageBufferSize, config.PageBufferSize), + WriteBufferSize: coalesceInt(c.WriteBufferSize, config.WriteBufferSize), + DataPageVersion: coalesceInt(c.DataPageVersion, config.DataPageVersion), + DataPageStatistics: coalesceBool(c.DataPageStatistics, config.DataPageStatistics), + MaxRowsPerRowGroup: coalesceInt64(c.MaxRowsPerRowGroup, config.MaxRowsPerRowGroup), + KeyValueMetadata: keyValueMetadata, + Schema: coalesceSchema(c.Schema, config.Schema), + BloomFilters: coalesceBloomFilters(c.BloomFilters, config.BloomFilters), + Compression: coalesceCompression(c.Compression, config.Compression), + Sorting: coalesceSortingConfig(c.Sorting, config.Sorting), + } +} + +// Validate returns a non-nil error if the configuration of c is invalid. +func (c *WriterConfig) Validate() error { + const baseName = "parquet.(*WriterConfig)." + return errorInvalidConfiguration( + validateNotNil(baseName+"ColumnPageBuffers", c.ColumnPageBuffers), + validatePositiveInt(baseName+"ColumnIndexSizeLimit", c.ColumnIndexSizeLimit), + validatePositiveInt(baseName+"PageBufferSize", c.PageBufferSize), + validateOneOfInt(baseName+"DataPageVersion", c.DataPageVersion, 1, 2), + c.Sorting.Validate(), + ) +} + +// The RowGroupConfig type carries configuration options for parquet row groups. +// +// RowGroupConfig implements the RowGroupOption interface so it can be used +// directly as argument to the NewBuffer function when needed, for example: +// +// buffer := parquet.NewBuffer(&parquet.RowGroupConfig{ +// ColumnBufferCapacity: 10_000, +// }) +type RowGroupConfig struct { + ColumnBufferCapacity int + Schema *Schema + Sorting SortingConfig +} + +// DefaultRowGroupConfig returns a new RowGroupConfig value initialized with the +// default row group configuration. +func DefaultRowGroupConfig() *RowGroupConfig { + return &RowGroupConfig{ + ColumnBufferCapacity: DefaultColumnBufferCapacity, + Sorting: SortingConfig{ + SortingBuffers: &defaultSortingBufferPool, + }, + } +} + +// NewRowGroupConfig constructs a new row group configuration applying the +// options passed as arguments. +// +// The function returns an non-nil error if some of the options carried invalid +// configuration values. +func NewRowGroupConfig(options ...RowGroupOption) (*RowGroupConfig, error) { + config := DefaultRowGroupConfig() + config.Apply(options...) + return config, config.Validate() +} + +// Validate returns a non-nil error if the configuration of c is invalid. +func (c *RowGroupConfig) Validate() error { + const baseName = "parquet.(*RowGroupConfig)." + return errorInvalidConfiguration( + validatePositiveInt(baseName+"ColumnBufferCapacity", c.ColumnBufferCapacity), + c.Sorting.Validate(), + ) +} + +func (c *RowGroupConfig) Apply(options ...RowGroupOption) { + for _, opt := range options { + opt.ConfigureRowGroup(c) + } +} + +func (c *RowGroupConfig) ConfigureRowGroup(config *RowGroupConfig) { + *config = RowGroupConfig{ + ColumnBufferCapacity: coalesceInt(c.ColumnBufferCapacity, config.ColumnBufferCapacity), + Schema: coalesceSchema(c.Schema, config.Schema), + Sorting: coalesceSortingConfig(c.Sorting, config.Sorting), + } +} + +// The SortingConfig type carries configuration options for parquet row groups. +// +// SortingConfig implements the SortingOption interface so it can be used +// directly as argument to the NewSortingWriter function when needed, +// for example: +// +// buffer := parquet.NewSortingWriter[Row]( +// parquet.SortingWriterConfig( +// parquet.DropDuplicatedRows(true), +// ), +// }) +type SortingConfig struct { + SortingBuffers BufferPool + SortingColumns []SortingColumn + DropDuplicatedRows bool +} + +// DefaultSortingConfig returns a new SortingConfig value initialized with the +// default row group configuration. +func DefaultSortingConfig() *SortingConfig { + return &SortingConfig{ + SortingBuffers: &defaultSortingBufferPool, + } +} + +// NewSortingConfig constructs a new sorting configuration applying the +// options passed as arguments. +// +// The function returns an non-nil error if some of the options carried invalid +// configuration values. +func NewSortingConfig(options ...SortingOption) (*SortingConfig, error) { + config := DefaultSortingConfig() + config.Apply(options...) + return config, config.Validate() +} + +func (c *SortingConfig) Validate() error { + const baseName = "parquet.(*SortingConfig)." + return errorInvalidConfiguration( + validateNotNil(baseName+"SortingBuffers", c.SortingBuffers), + ) +} + +func (c *SortingConfig) Apply(options ...SortingOption) { + for _, opt := range options { + opt.ConfigureSorting(c) + } +} + +func (c *SortingConfig) ConfigureSorting(config *SortingConfig) { + *config = coalesceSortingConfig(*c, *config) +} + +// FileOption is an interface implemented by types that carry configuration +// options for parquet files. +type FileOption interface { + ConfigureFile(*FileConfig) +} + +// ReaderOption is an interface implemented by types that carry configuration +// options for parquet readers. +type ReaderOption interface { + ConfigureReader(*ReaderConfig) +} + +// WriterOption is an interface implemented by types that carry configuration +// options for parquet writers. +type WriterOption interface { + ConfigureWriter(*WriterConfig) +} + +// RowGroupOption is an interface implemented by types that carry configuration +// options for parquet row groups. +type RowGroupOption interface { + ConfigureRowGroup(*RowGroupConfig) +} + +// SortingOption is an interface implemented by types that carry configuration +// options for parquet sorting writers. +type SortingOption interface { + ConfigureSorting(*SortingConfig) +} + +// SkipMagicBytes is a file configuration option which prevents automatically +// reading the magic bytes when opening a parquet file, when set to true. This +// is useful as an optimization when programs can trust that they are dealing +// with parquet files and do not need to verify the first 4 bytes. +func SkipMagicBytes(skip bool) FileOption { + return fileOption(func(config *FileConfig) { config.SkipMagicBytes = skip }) +} + +// SkipPageIndex is a file configuration option which prevents automatically +// reading the page index when opening a parquet file, when set to true. This is +// useful as an optimization when programs know that they will not need to +// consume the page index. +// +// Defaults to false. +func SkipPageIndex(skip bool) FileOption { + return fileOption(func(config *FileConfig) { config.SkipPageIndex = skip }) +} + +// SkipBloomFilters is a file configuration option which prevents automatically +// reading the bloom filters when opening a parquet file, when set to true. +// This is useful as an optimization when programs know that they will not need +// to consume the bloom filters. +// +// Defaults to false. +func SkipBloomFilters(skip bool) FileOption { + return fileOption(func(config *FileConfig) { config.SkipBloomFilters = skip }) +} + +// OptimisticRead configures a file to optimistically perform larger buffered +// reads to improve performance. This is useful when reading from remote storage +// and amortize the cost of network round trips. +// +// This is an option instead of enabled by default because dependents of this +// package have historically relied on the read patterns to provide external +// caches and achieve similar results (e.g., Tempo). +func OptimisticRead(enabled bool) FileOption { + return fileOption(func(config *FileConfig) { config.OptimisticRead = enabled }) +} + +// FileReadMode is a file configuration option which controls the way pages +// are read. Currently the only two options are ReadModeAsync and ReadModeSync +// which control whether or not pages are loaded asynchronously. It can be +// advantageous to use ReadModeAsync if your reader is backed by network +// storage. +// +// Defaults to ReadModeSync. +func FileReadMode(mode ReadMode) FileOption { + return fileOption(func(config *FileConfig) { config.ReadMode = mode }) +} + +// ReadBufferSize is a file configuration option which controls the default +// buffer sizes for reads made to the provided io.Reader. The default of 4096 +// is appropriate for disk based access but if your reader is backed by network +// storage it can be advantageous to increase this value to something more like +// 4 MiB. +// +// Defaults to 4096. +func ReadBufferSize(size int) FileOption { + return fileOption(func(config *FileConfig) { config.ReadBufferSize = size }) +} + +// FileSchema is used to pass a known schema in while opening a Parquet file. +// This optimization is only useful if your application is currently opening +// an extremely large number of parquet files with the same, known schema. +// +// Defaults to nil. +func FileSchema(schema *Schema) FileOption { + return fileOption(func(config *FileConfig) { config.Schema = schema }) +} + +// PageBufferSize configures the size of column page buffers on parquet writers. +// +// Note that the page buffer size refers to the in-memory buffers where pages +// are generated, not the size of pages after encoding and compression. +// This design choice was made to help control the amount of memory needed to +// read and write pages rather than controlling the space used by the encoded +// representation on disk. +// +// Defaults to 256KiB. +func PageBufferSize(size int) WriterOption { + return writerOption(func(config *WriterConfig) { config.PageBufferSize = size }) +} + +// WriteBufferSize configures the size of the write buffer. +// +// Setting the writer buffer size to zero deactivates buffering, all writes are +// immediately sent to the output io.Writer. +// +// Defaults to 32KiB. +func WriteBufferSize(size int) WriterOption { + return writerOption(func(config *WriterConfig) { config.WriteBufferSize = size }) +} + +// MaxRowsPerRowGroup configures the maximum number of rows that a writer will +// produce in each row group. +// +// This limit is useful to control size of row groups in both number of rows and +// byte size. While controlling the byte size of a row group is difficult to +// achieve with parquet due to column encoding and compression, the number of +// rows remains a useful proxy. +// +// Defaults to unlimited. +func MaxRowsPerRowGroup(numRows int64) WriterOption { + if numRows <= 0 { + numRows = DefaultMaxRowsPerRowGroup + } + return writerOption(func(config *WriterConfig) { config.MaxRowsPerRowGroup = numRows }) +} + +// CreatedBy creates a configuration option which sets the name of the +// application that created a parquet file. +// +// The option formats the "CreatedBy" file metadata according to the convention +// described by the parquet spec: +// +// " version (build )" +// +// By default, the option is set to the parquet-go module name, version, and +// build hash. +func CreatedBy(application, version, build string) WriterOption { + createdBy := formatCreatedBy(application, version, build) + return writerOption(func(config *WriterConfig) { config.CreatedBy = createdBy }) +} + +// ColumnPageBuffers creates a configuration option to customize the buffer pool +// used when constructing row groups. This can be used to provide on-disk buffers +// as swap space to ensure that the parquet file creation will no be bottlenecked +// on the amount of memory available. +// +// Defaults to using in-memory buffers. +func ColumnPageBuffers(buffers BufferPool) WriterOption { + return writerOption(func(config *WriterConfig) { config.ColumnPageBuffers = buffers }) +} + +// ColumnIndexSizeLimit creates a configuration option to customize the size +// limit of page boundaries recorded in column indexes. +// +// Defaults to 16. +func ColumnIndexSizeLimit(sizeLimit int) WriterOption { + return writerOption(func(config *WriterConfig) { config.ColumnIndexSizeLimit = sizeLimit }) +} + +// DataPageVersion creates a configuration option which configures the version of +// data pages used when creating a parquet file. +// +// Defaults to version 2. +func DataPageVersion(version int) WriterOption { + return writerOption(func(config *WriterConfig) { config.DataPageVersion = version }) +} + +// DataPageStatistics creates a configuration option which defines whether data +// page statistics are emitted. This option is useful when generating parquet +// files that intend to be backward compatible with older readers which may not +// have the ability to load page statistics from the column index. +// +// Defaults to false. +func DataPageStatistics(enabled bool) WriterOption { + return writerOption(func(config *WriterConfig) { config.DataPageStatistics = enabled }) +} + +// KeyValueMetadata creates a configuration option which adds key/value metadata +// to add to the metadata of parquet files. +// +// This option is additive, it may be used multiple times to add more than one +// key/value pair. +// +// Keys are assumed to be unique, if the same key is repeated multiple times the +// last value is retained. While the parquet format does not require unique keys, +// this design decision was made to optimize for the most common use case where +// applications leverage this extension mechanism to associate single values to +// keys. This may create incompatibilities with other parquet libraries, or may +// cause some key/value pairs to be lost when open parquet files written with +// repeated keys. We can revisit this decision if it ever becomes a blocker. +func KeyValueMetadata(key, value string) WriterOption { + return writerOption(func(config *WriterConfig) { + if config.KeyValueMetadata == nil { + config.KeyValueMetadata = map[string]string{key: value} + } else { + config.KeyValueMetadata[key] = value + } + }) +} + +// BloomFilters creates a configuration option which defines the bloom filters +// that parquet writers should generate. +// +// The compute and memory footprint of generating bloom filters for all columns +// of a parquet schema can be significant, so by default no filters are created +// and applications need to explicitly declare the columns that they want to +// create filters for. +func BloomFilters(filters ...BloomFilterColumn) WriterOption { + filters = slices.Clone(filters) + return writerOption(func(config *WriterConfig) { config.BloomFilters = filters }) +} + +// Compression creates a configuration option which sets the default compression +// codec used by a writer for columns where none were defined. +func Compression(codec compress.Codec) WriterOption { + return writerOption(func(config *WriterConfig) { config.Compression = codec }) +} + +// SortingWriterConfig is a writer option which applies configuration specific +// to sorting writers. +func SortingWriterConfig(options ...SortingOption) WriterOption { + options = slices.Clone(options) + return writerOption(func(config *WriterConfig) { config.Sorting.Apply(options...) }) +} + +// SkipPageBounds lists the path to a column that shouldn't have bounds written to the +// footer of the parquet file. This is useful for data blobs, like a raw html file, +// where the bounds are not meaningful. +// +// This option is additive, it may be used multiple times to skip multiple columns. +func SkipPageBounds(path ...string) WriterOption { + return writerOption(func(config *WriterConfig) { config.SkipPageBounds = append(config.SkipPageBounds, path) }) +} + +// ColumnBufferCapacity creates a configuration option which defines the size of +// row group column buffers. +// +// Defaults to 16384. +func ColumnBufferCapacity(size int) RowGroupOption { + return rowGroupOption(func(config *RowGroupConfig) { config.ColumnBufferCapacity = size }) +} + +// SortingRowGroupConfig is a row group option which applies configuration +// specific sorting row groups. +func SortingRowGroupConfig(options ...SortingOption) RowGroupOption { + options = slices.Clone(options) + return rowGroupOption(func(config *RowGroupConfig) { config.Sorting.Apply(options...) }) +} + +// SortingColumns creates a configuration option which defines the sorting order +// of columns in a row group. +// +// The order of sorting columns passed as argument defines the ordering +// hierarchy; when elements are equal in the first column, the second column is +// used to order rows, etc... +func SortingColumns(columns ...SortingColumn) SortingOption { + // Make a copy so that we do not retain the input slice generated implicitly + // for the variable argument list, and also avoid having a nil slice when + // the option is passed with no sorting columns, so we can differentiate it + // from it not being passed. + columns = slices.Clone(columns) + return sortingOption(func(config *SortingConfig) { config.SortingColumns = columns }) +} + +// SortingBuffers creates a configuration option which sets the pool of buffers +// used to hold intermediary state when sorting parquet rows. +// +// Defaults to using in-memory buffers. +func SortingBuffers(buffers BufferPool) SortingOption { + return sortingOption(func(config *SortingConfig) { config.SortingBuffers = buffers }) +} + +// DropDuplicatedRows configures whether a sorting writer will keep or remove +// duplicated rows. +// +// Two rows are considered duplicates if the values of their all their sorting +// columns are equal. +// +// Defaults to false +func DropDuplicatedRows(drop bool) SortingOption { + return sortingOption(func(config *SortingConfig) { config.DropDuplicatedRows = drop }) +} + +type fileOption func(*FileConfig) + +func (opt fileOption) ConfigureFile(config *FileConfig) { opt(config) } + +type readerOption func(*ReaderConfig) + +func (opt readerOption) ConfigureReader(config *ReaderConfig) { opt(config) } + +type writerOption func(*WriterConfig) + +func (opt writerOption) ConfigureWriter(config *WriterConfig) { opt(config) } + +type rowGroupOption func(*RowGroupConfig) + +func (opt rowGroupOption) ConfigureRowGroup(config *RowGroupConfig) { opt(config) } + +type sortingOption func(*SortingConfig) + +func (opt sortingOption) ConfigureSorting(config *SortingConfig) { opt(config) } + +func coalesceBool(i1, i2 bool) bool { + return i1 || i2 +} + +func coalesceInt(i1, i2 int) int { + if i1 != 0 { + return i1 + } + return i2 +} + +func coalesceInt64(i1, i2 int64) int64 { + if i1 != 0 { + return i1 + } + return i2 +} + +func coalesceString(s1, s2 string) string { + if s1 != "" { + return s1 + } + return s2 +} + +func coalesceBytes(b1, b2 []byte) []byte { + if b1 != nil { + return b1 + } + return b2 +} + +func coalesceBufferPool(p1, p2 BufferPool) BufferPool { + if p1 != nil { + return p1 + } + return p2 +} + +func coalesceSchema(s1, s2 *Schema) *Schema { + if s1 != nil { + return s1 + } + return s2 +} + +func coalesceSortingColumns(s1, s2 []SortingColumn) []SortingColumn { + if s1 != nil { + return s1 + } + return s2 +} + +func coalesceSortingConfig(c1, c2 SortingConfig) SortingConfig { + return SortingConfig{ + SortingBuffers: coalesceBufferPool(c1.SortingBuffers, c2.SortingBuffers), + SortingColumns: coalesceSortingColumns(c1.SortingColumns, c2.SortingColumns), + DropDuplicatedRows: c1.DropDuplicatedRows, + } +} + +func coalesceBloomFilters(f1, f2 []BloomFilterColumn) []BloomFilterColumn { + if f1 != nil { + return f1 + } + return f2 +} + +func coalesceCompression(c1, c2 compress.Codec) compress.Codec { + if c1 != nil { + return c1 + } + return c2 +} + +func validatePositiveInt(optionName string, optionValue int) error { + if optionValue > 0 { + return nil + } + return errorInvalidOptionValue(optionName, optionValue) +} + +func validatePositiveInt64(optionName string, optionValue int64) error { + if optionValue > 0 { + return nil + } + return errorInvalidOptionValue(optionName, optionValue) +} + +func validateOneOfInt(optionName string, optionValue int, supportedValues ...int) error { + if slices.Contains(supportedValues, optionValue) { + return nil + } + return errorInvalidOptionValue(optionName, optionValue) +} + +func validateNotNil(optionName string, optionValue any) error { + if optionValue != nil { + return nil + } + return errorInvalidOptionValue(optionName, optionValue) +} + +func errorInvalidOptionValue(optionName string, optionValue any) error { + return fmt.Errorf("invalid option value: %s: %v", optionName, optionValue) +} + +func errorInvalidConfiguration(reasons ...error) error { + var err *invalidConfiguration + + for _, reason := range reasons { + if reason != nil { + if err == nil { + err = new(invalidConfiguration) + } + err.reasons = append(err.reasons, reason) + } + } + + if err != nil { + return err + } + + return nil +} + +type invalidConfiguration struct { + reasons []error +} + +func (err *invalidConfiguration) Error() string { + errorMessage := new(strings.Builder) + for _, reason := range err.reasons { + errorMessage.WriteString(reason.Error()) + errorMessage.WriteString("\n") + } + errorString := errorMessage.String() + if errorString != "" { + errorString = errorString[:len(errorString)-1] + } + return errorString +} + +var ( + _ FileOption = (*FileConfig)(nil) + _ ReaderOption = (*ReaderConfig)(nil) + _ WriterOption = (*WriterConfig)(nil) + _ RowGroupOption = (*RowGroupConfig)(nil) + _ SortingOption = (*SortingConfig)(nil) +) diff --git a/vendor/github.com/parquet-go/parquet-go/convert.go b/vendor/github.com/parquet-go/parquet-go/convert.go new file mode 100644 index 0000000000..85f2995caf --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/convert.go @@ -0,0 +1,1068 @@ +package parquet + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "math" + "math/big" + "strconv" + "sync" + "time" + + "golang.org/x/sys/cpu" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// ConvertError is an error type returned by calls to Convert when the conversion +// of parquet schemas is impossible or the input row for the conversion is +// malformed. +type ConvertError struct { + Path []string + From Node + To Node +} + +// Error satisfies the error interface. +func (e *ConvertError) Error() string { + sourceType := e.From.Type() + targetType := e.To.Type() + + sourceRepetition := fieldRepetitionTypeOf(e.From) + targetRepetition := fieldRepetitionTypeOf(e.To) + + return fmt.Sprintf("cannot convert parquet column %q from %s %s to %s %s", + columnPath(e.Path), + sourceRepetition, + sourceType, + targetRepetition, + targetType, + ) +} + +// Conversion is an interface implemented by types that provide conversion of +// parquet rows from one schema to another. +// +// Conversion instances must be safe to use concurrently from multiple goroutines. +type Conversion interface { + // Applies the conversion logic on the src row, returning the result + // appended to dst. + Convert(rows []Row) (int, error) + // Converts the given column index in the target schema to the original + // column index in the source schema of the conversion. + Column(int) int + // Returns the target schema of the conversion. + Schema() *Schema +} + +type conversion struct { + columns []conversionColumn + schema *Schema + buffers sync.Pool + // This field is used to size the column buffers held in the sync.Pool since + // they are intended to store the source rows being converted from. + numberOfSourceColumns int +} + +type conversionBuffer struct { + columns [][]Value +} + +type conversionColumn struct { + sourceIndex int + convertValues conversionFunc +} + +type conversionFunc func([]Value) error + +func convertToSelf(column []Value) error { return nil } + +//go:noinline +func convertToType(targetType, sourceType Type) conversionFunc { + return func(column []Value) error { + for i, v := range column { + v, err := targetType.ConvertValue(v, sourceType) + if err != nil { + return err + } + column[i].ptr = v.ptr + column[i].u64 = v.u64 + column[i].kind = v.kind + } + return nil + } +} + +//go:noinline +func convertToValue(value Value) conversionFunc { + return func(column []Value) error { + for i := range column { + column[i] = value + } + return nil + } +} + +//go:noinline +func convertToZero(kind Kind) conversionFunc { + return func(column []Value) error { + for i := range column { + column[i].ptr = nil + column[i].u64 = 0 + column[i].kind = ^int8(kind) + } + return nil + } +} + +//go:noinline +func convertToLevels(repetitionLevels, definitionLevels []byte) conversionFunc { + return func(column []Value) error { + for i := range column { + r := column[i].repetitionLevel + d := column[i].definitionLevel + column[i].repetitionLevel = repetitionLevels[r] + column[i].definitionLevel = definitionLevels[d] + } + return nil + } +} + +//go:noinline +func multiConversionFunc(conversions []conversionFunc) conversionFunc { + switch len(conversions) { + case 0: + return convertToSelf + case 1: + return conversions[0] + default: + return func(column []Value) error { + for _, conv := range conversions { + if err := conv(column); err != nil { + return err + } + } + return nil + } + } +} + +func (c *conversion) getBuffer() *conversionBuffer { + b, _ := c.buffers.Get().(*conversionBuffer) + if b == nil { + b = &conversionBuffer{ + columns: make([][]Value, c.numberOfSourceColumns), + } + values := make([]Value, c.numberOfSourceColumns) + for i := range b.columns { + b.columns[i] = values[i : i : i+1] + } + } + return b +} + +func (c *conversion) putBuffer(b *conversionBuffer) { + c.buffers.Put(b) +} + +// Convert here satisfies the Conversion interface, and does the actual work +// to convert between the source and target Rows. +func (c *conversion) Convert(rows []Row) (int, error) { + source := c.getBuffer() + defer c.putBuffer(source) + + for n, row := range rows { + for i, values := range source.columns { + source.columns[i] = values[:0] + } + row.Range(func(columnIndex int, columnValues []Value) bool { + source.columns[columnIndex] = append(source.columns[columnIndex], columnValues...) + return true + }) + row = row[:0] + + for columnIndex, conv := range c.columns { + columnOffset := len(row) + if conv.sourceIndex < 0 { + // When there is no source column, we put a single value as + // placeholder in the column. This is a condition where the + // target contained a column which did not exist at had not + // other columns existing at that same level. + row = append(row, Value{}) + } else { + // We must copy to the output row first and not mutate the + // source columns because multiple target columns may map to + // the same source column. + row = append(row, source.columns[conv.sourceIndex]...) + } + columnValues := row[columnOffset:] + + if err := conv.convertValues(columnValues); err != nil { + return n, err + } + + // Since the column index may have changed between the source and + // taget columns we ensure that the right value is always written + // to the output row. + for i := range columnValues { + columnValues[i].columnIndex = ^int16(columnIndex) + } + } + + rows[n] = row + } + + return len(rows), nil +} + +func (c *conversion) Column(i int) int { + return c.columns[i].sourceIndex +} + +func (c *conversion) Schema() *Schema { + return c.schema +} + +type identity struct{ schema *Schema } + +func (id identity) Convert(rows []Row) (int, error) { return len(rows), nil } +func (id identity) Column(i int) int { return i } +func (id identity) Schema() *Schema { return id.schema } + +// Convert constructs a conversion function from one parquet schema to another. +// +// The function supports converting between schemas where the source or target +// have extra columns; if there are more columns in the source, they will be +// stripped out of the rows. Extra columns in the target schema will be set to +// null or zero values. +// +// The returned function is intended to be used to append the converted source +// row to the destination buffer. +func Convert(to, from Node) (conv Conversion, err error) { + schema, _ := to.(*Schema) + if schema == nil { + schema = NewSchema("", to) + } + + if EqualNodes(to, from) { + return identity{schema}, nil + } + + targetMapping, targetColumns := columnMappingOf(to) + sourceMapping, sourceColumns := columnMappingOf(from) + columns := make([]conversionColumn, len(targetColumns)) + + for i, path := range targetColumns { + targetColumn := targetMapping.lookup(path) + sourceColumn := sourceMapping.lookup(path) + + conversions := []conversionFunc{} + if sourceColumn.node != nil { + targetType := targetColumn.node.Type() + sourceType := sourceColumn.node.Type() + if !typesAreEqual(targetType, sourceType) { + conversions = append(conversions, + convertToType(targetType, sourceType), + ) + } + + repetitionLevels := make([]byte, len(path)+1) + definitionLevels := make([]byte, len(path)+1) + targetRepetitionLevel := byte(0) + targetDefinitionLevel := byte(0) + sourceRepetitionLevel := byte(0) + sourceDefinitionLevel := byte(0) + targetNode := to + sourceNode := from + + for j := range path { + targetNode = fieldByName(targetNode, path[j]) + sourceNode = fieldByName(sourceNode, path[j]) + + targetRepetitionLevel, targetDefinitionLevel = applyFieldRepetitionType( + fieldRepetitionTypeOf(targetNode), + targetRepetitionLevel, + targetDefinitionLevel, + ) + sourceRepetitionLevel, sourceDefinitionLevel = applyFieldRepetitionType( + fieldRepetitionTypeOf(sourceNode), + sourceRepetitionLevel, + sourceDefinitionLevel, + ) + + repetitionLevels[sourceRepetitionLevel] = targetRepetitionLevel + definitionLevels[sourceDefinitionLevel] = targetDefinitionLevel + } + + repetitionLevels = repetitionLevels[:sourceRepetitionLevel+1] + definitionLevels = definitionLevels[:sourceDefinitionLevel+1] + + if !isDirectLevelMapping(repetitionLevels) || !isDirectLevelMapping(definitionLevels) { + conversions = append(conversions, + convertToLevels(repetitionLevels, definitionLevels), + ) + } + + } else { + targetType := targetColumn.node.Type() + targetKind := targetType.Kind() + sourceColumn = sourceMapping.lookupClosest(path) + if sourceColumn.node != nil { + conversions = append(conversions, + convertToZero(targetKind), + ) + } else { + conversions = append(conversions, + convertToValue(ZeroValue(targetKind)), + ) + } + } + + columns[i] = conversionColumn{ + sourceIndex: int(sourceColumn.columnIndex), + convertValues: multiConversionFunc(conversions), + } + } + + c := &conversion{ + columns: columns, + schema: schema, + numberOfSourceColumns: len(sourceColumns), + } + return c, nil +} + +func isDirectLevelMapping(levels []byte) bool { + for i, level := range levels { + if level != byte(i) { + return false + } + } + return true +} + +// ConvertRowGroup constructs a wrapper of the given row group which applies +// the given schema conversion to its rows. +func ConvertRowGroup(rowGroup RowGroup, conv Conversion) RowGroup { + schema := conv.Schema() + numRows := rowGroup.NumRows() + rowGroupColumns := rowGroup.ColumnChunks() + + columns := make([]ColumnChunk, numLeafColumnsOf(schema)) + forEachLeafColumnOf(schema, func(leaf leafColumn) { + i := leaf.columnIndex + j := conv.Column(int(leaf.columnIndex)) + if j < 0 { + columns[i] = &missingColumnChunk{ + typ: leaf.node.Type(), + column: i, + // TODO: we assume the number of values is the same as the + // number of rows, which may not be accurate when the column is + // part of a repeated group; neighbor columns may be repeated in + // which case it would be impossible for this chunk not to be. + numRows: numRows, + numValues: numRows, + numNulls: numRows, + } + } else { + columns[i] = rowGroupColumns[j] + } + }) + + // Sorting columns must exist on the conversion schema in order to be + // advertised on the converted row group otherwise the resulting rows + // would not be in the right order. + sorting := []SortingColumn{} + for _, col := range rowGroup.SortingColumns() { + if !hasColumnPath(schema, col.Path()) { + break + } + sorting = append(sorting, col) + } + + return &convertedRowGroup{ + // The pair of rowGroup+conv is retained to construct a converted row + // reader by wrapping the underlying row reader of the row group because + // it allows proper reconstruction of the repetition and definition + // levels. + // + // TODO: can we figure out how to set the repetition and definition + // levels when reading values from missing column pages? At first sight + // it appears complex to do, however: + // + // * It is possible that having these levels when reading values of + // missing column pages is not necessary in some scenarios (e.g. when + // merging row groups). + // + // * We may be able to assume the repetition and definition levels at + // the call site (e.g. in the functions reading rows from columns). + // + // Columns of the source row group which do not exist in the target are + // masked to prevent loading unneeded pages when reading rows from the + // converted row group. + rowGroup: maskMissingRowGroupColumns(rowGroup, len(columns), conv), + columns: columns, + sorting: sorting, + conv: conv, + } +} + +func maskMissingRowGroupColumns(r RowGroup, numColumns int, conv Conversion) RowGroup { + rowGroupColumns := r.ColumnChunks() + columns := make([]ColumnChunk, len(rowGroupColumns)) + missing := make([]missingColumnChunk, len(columns)) + numRows := r.NumRows() + + for i := range missing { + missing[i] = missingColumnChunk{ + typ: rowGroupColumns[i].Type(), + column: int16(i), + numRows: numRows, + numValues: numRows, + numNulls: numRows, + } + } + + for i := range columns { + columns[i] = &missing[i] + } + + for i := range numColumns { + j := conv.Column(i) + if j >= 0 && j < len(columns) { + columns[j] = rowGroupColumns[j] + } + } + + return &rowGroup{ + schema: r.Schema(), + numRows: numRows, + columns: columns, + } +} + +type missingColumnChunk struct { + typ Type + column int16 + numRows int64 + numValues int64 + numNulls int64 +} + +func (c *missingColumnChunk) Type() Type { return c.typ } +func (c *missingColumnChunk) Column() int { return int(c.column) } +func (c *missingColumnChunk) Pages() Pages { return onePage(missingPage{c}) } +func (c *missingColumnChunk) ColumnIndex() (ColumnIndex, error) { return missingColumnIndex{c}, nil } +func (c *missingColumnChunk) OffsetIndex() (OffsetIndex, error) { return missingOffsetIndex{}, nil } +func (c *missingColumnChunk) BloomFilter() BloomFilter { return missingBloomFilter{} } +func (c *missingColumnChunk) NumValues() int64 { return c.numValues } + +type missingColumnIndex struct{ *missingColumnChunk } + +func (i missingColumnIndex) NumPages() int { return 1 } +func (i missingColumnIndex) NullCount(int) int64 { return i.numNulls } +func (i missingColumnIndex) NullPage(int) bool { return true } +func (i missingColumnIndex) MinValue(int) Value { return Value{} } +func (i missingColumnIndex) MaxValue(int) Value { return Value{} } +func (i missingColumnIndex) IsAscending() bool { return true } +func (i missingColumnIndex) IsDescending() bool { return false } + +type missingOffsetIndex struct{} + +func (missingOffsetIndex) NumPages() int { return 1 } +func (missingOffsetIndex) Offset(int) int64 { return 0 } +func (missingOffsetIndex) CompressedPageSize(int) int64 { return 0 } +func (missingOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type missingBloomFilter struct{} + +func (missingBloomFilter) ReadAt([]byte, int64) (int, error) { return 0, io.EOF } +func (missingBloomFilter) Size() int64 { return 0 } +func (missingBloomFilter) Check(Value) (bool, error) { return false, nil } + +type missingPage struct{ *missingColumnChunk } + +func (p missingPage) Column() int { return int(p.column) } +func (p missingPage) Dictionary() Dictionary { return nil } +func (p missingPage) NumRows() int64 { return p.numRows } +func (p missingPage) NumValues() int64 { return p.numValues } +func (p missingPage) NumNulls() int64 { return p.numNulls } +func (p missingPage) Bounds() (min, max Value, ok bool) { return } +func (p missingPage) Slice(i, j int64) Page { + return missingPage{ + &missingColumnChunk{ + typ: p.typ, + column: p.column, + numRows: j - i, + numValues: j - i, + numNulls: j - i, + }, + } +} +func (p missingPage) Size() int64 { return 0 } +func (p missingPage) RepetitionLevels() []byte { return nil } +func (p missingPage) DefinitionLevels() []byte { return nil } +func (p missingPage) Data() encoding.Values { return p.typ.NewValues(nil, nil) } +func (p missingPage) Values() ValueReader { return &missingPageValues{page: p} } + +type missingPageValues struct { + page missingPage + read int64 +} + +func (r *missingPageValues) ReadValues(values []Value) (int, error) { + remain := r.page.numValues - r.read + if int64(len(values)) > remain { + values = values[:remain] + } + for i := range values { + // TODO: how do we set the repetition and definition levels here? + values[i] = Value{columnIndex: ^r.page.column} + } + if r.read += int64(len(values)); r.read == r.page.numValues { + return len(values), io.EOF + } + return len(values), nil +} + +func (r *missingPageValues) Close() error { + r.read = r.page.numValues + return nil +} + +type convertedRowGroup struct { + rowGroup RowGroup + columns []ColumnChunk + sorting []SortingColumn + conv Conversion +} + +func (c *convertedRowGroup) NumRows() int64 { return c.rowGroup.NumRows() } +func (c *convertedRowGroup) ColumnChunks() []ColumnChunk { return c.columns } +func (c *convertedRowGroup) Schema() *Schema { return c.conv.Schema() } +func (c *convertedRowGroup) SortingColumns() []SortingColumn { return c.sorting } +func (c *convertedRowGroup) Rows() Rows { + rows := c.rowGroup.Rows() + return &convertedRows{ + Closer: rows, + rows: rows, + conv: c.conv, + } +} + +// ConvertRowReader constructs a wrapper of the given row reader which applies +// the given schema conversion to the rows. +func ConvertRowReader(rows RowReader, conv Conversion) RowReaderWithSchema { + return &convertedRows{rows: &forwardRowSeeker{rows: rows}, conv: conv} +} + +type convertedRows struct { + io.Closer + rows RowReadSeeker + conv Conversion +} + +func (c *convertedRows) ReadRows(rows []Row) (int, error) { + n, err := c.rows.ReadRows(rows) + if n > 0 { + var convErr error + n, convErr = c.conv.Convert(rows[:n]) + if convErr != nil { + err = convErr + } + } + return n, err +} + +func (c *convertedRows) Schema() *Schema { + return c.conv.Schema() +} + +func (c *convertedRows) SeekToRow(rowIndex int64) error { + return c.rows.SeekToRow(rowIndex) +} + +var ( + trueBytes = []byte(`true`) + falseBytes = []byte(`false`) + unixEpoch = time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC) +) + +func convertBooleanToInt32(v Value) (Value, error) { + return v.convertToInt32(int32(v.byte())), nil +} + +func convertBooleanToInt64(v Value) (Value, error) { + return v.convertToInt64(int64(v.byte())), nil +} + +func convertBooleanToInt96(v Value) (Value, error) { + return v.convertToInt96(deprecated.Int96{0: uint32(v.byte())}), nil +} + +func convertBooleanToFloat(v Value) (Value, error) { + return v.convertToFloat(float32(v.byte())), nil +} + +func convertBooleanToDouble(v Value) (Value, error) { + return v.convertToDouble(float64(v.byte())), nil +} + +func convertBooleanToByteArray(v Value) (Value, error) { + return v.convertToByteArray([]byte{v.byte()}), nil +} + +func convertBooleanToFixedLenByteArray(v Value, size int) (Value, error) { + b := []byte{v.byte()} + c := make([]byte, size) + copy(c, b) + return v.convertToFixedLenByteArray(c), nil +} + +func convertBooleanToString(v Value) (Value, error) { + b := ([]byte)(nil) + if v.boolean() { + b = trueBytes + } else { + b = falseBytes + } + return v.convertToByteArray(b), nil +} + +func convertInt32ToBoolean(v Value) (Value, error) { + return v.convertToBoolean(v.int32() != 0), nil +} + +func convertInt32ToInt64(v Value) (Value, error) { + return v.convertToInt64(int64(v.int32())), nil +} + +func convertInt32ToInt96(v Value) (Value, error) { + return v.convertToInt96(deprecated.Int32ToInt96(v.int32())), nil +} + +func convertInt32ToFloat(v Value) (Value, error) { + return v.convertToFloat(float32(v.int32())), nil +} + +func convertInt32ToDouble(v Value) (Value, error) { + return v.convertToDouble(float64(v.int32())), nil +} + +func convertInt32ToByteArray(v Value) (Value, error) { + b := make([]byte, 4) + binary.LittleEndian.PutUint32(b, v.uint32()) + return v.convertToByteArray(b), nil +} + +func convertInt32ToFixedLenByteArray(v Value, size int) (Value, error) { + b := make([]byte, 4) + c := make([]byte, size) + binary.LittleEndian.PutUint32(b, v.uint32()) + copy(c, b) + return v.convertToFixedLenByteArray(c), nil +} + +func convertInt32ToString(v Value) (Value, error) { + return v.convertToByteArray(strconv.AppendInt(nil, int64(v.int32()), 10)), nil +} + +func convertInt64ToBoolean(v Value) (Value, error) { + return v.convertToBoolean(v.int64() != 0), nil +} + +func convertInt64ToInt32(v Value) (Value, error) { + return v.convertToInt32(int32(v.int64())), nil +} + +func convertInt64ToInt96(v Value) (Value, error) { + return v.convertToInt96(deprecated.Int64ToInt96(v.int64())), nil +} + +func convertInt64ToFloat(v Value) (Value, error) { + return v.convertToFloat(float32(v.int64())), nil +} + +func convertInt64ToDouble(v Value) (Value, error) { + return v.convertToDouble(float64(v.int64())), nil +} + +func convertInt64ToByteArray(v Value) (Value, error) { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, v.uint64()) + return v.convertToByteArray(b), nil +} + +func convertInt64ToFixedLenByteArray(v Value, size int) (Value, error) { + b := make([]byte, 8) + c := make([]byte, size) + binary.LittleEndian.PutUint64(b, v.uint64()) + copy(c, b) + return v.convertToFixedLenByteArray(c), nil +} + +func convertInt64ToString(v Value) (Value, error) { + return v.convertToByteArray(strconv.AppendInt(nil, v.int64(), 10)), nil +} + +func convertInt96ToBoolean(v Value) (Value, error) { + return v.convertToBoolean(!v.int96().IsZero()), nil +} + +func convertInt96ToInt32(v Value) (Value, error) { + return v.convertToInt32(v.int96().Int32()), nil +} + +func convertInt96ToInt64(v Value) (Value, error) { + return v.convertToInt64(v.int96().Int64()), nil +} + +func convertInt96ToFloat(v Value) (Value, error) { + return v, invalidConversion(v, "INT96", "FLOAT") +} + +func convertInt96ToDouble(v Value) (Value, error) { + return v, invalidConversion(v, "INT96", "DOUBLE") +} + +func convertInt96ToByteArray(v Value) (Value, error) { + return v.convertToByteArray(v.byteArray()), nil +} + +func convertInt96ToFixedLenByteArray(v Value, size int) (Value, error) { + b := v.byteArray() + if len(b) < size { + c := make([]byte, size) + copy(c, b) + b = c + } else { + b = b[:size] + } + return v.convertToFixedLenByteArray(b), nil +} + +func convertInt96ToString(v Value) (Value, error) { + return v.convertToByteArray([]byte(v.String())), nil +} + +func convertFloatToBoolean(v Value) (Value, error) { + return v.convertToBoolean(v.float() != 0), nil +} + +func convertFloatToInt32(v Value) (Value, error) { + return v.convertToInt32(int32(v.float())), nil +} + +func convertFloatToInt64(v Value) (Value, error) { + return v.convertToInt64(int64(v.float())), nil +} + +func convertFloatToInt96(v Value) (Value, error) { + return v, invalidConversion(v, "FLOAT", "INT96") +} + +func convertFloatToDouble(v Value) (Value, error) { + return v.convertToDouble(float64(v.float())), nil +} + +func convertFloatToByteArray(v Value) (Value, error) { + b := make([]byte, 4) + binary.LittleEndian.PutUint32(b, v.uint32()) + return v.convertToByteArray(b), nil +} + +func convertFloatToFixedLenByteArray(v Value, size int) (Value, error) { + b := make([]byte, 4) + c := make([]byte, size) + binary.LittleEndian.PutUint32(b, v.uint32()) + copy(c, b) + return v.convertToFixedLenByteArray(c), nil +} + +func convertFloatToString(v Value) (Value, error) { + return v.convertToByteArray(strconv.AppendFloat(nil, float64(v.float()), 'g', -1, 32)), nil +} + +func convertDoubleToBoolean(v Value) (Value, error) { + return v.convertToBoolean(v.double() != 0), nil +} + +func convertDoubleToInt32(v Value) (Value, error) { + return v.convertToInt32(int32(v.double())), nil +} + +func convertDoubleToInt64(v Value) (Value, error) { + return v.convertToInt64(int64(v.double())), nil +} + +func convertDoubleToInt96(v Value) (Value, error) { + return v, invalidConversion(v, "FLOAT", "INT96") +} + +func convertDoubleToFloat(v Value) (Value, error) { + return v.convertToFloat(float32(v.double())), nil +} + +func convertDoubleToByteArray(v Value) (Value, error) { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, v.uint64()) + return v.convertToByteArray(b), nil +} + +func convertDoubleToFixedLenByteArray(v Value, size int) (Value, error) { + b := make([]byte, 8) + c := make([]byte, size) + binary.LittleEndian.PutUint64(b, v.uint64()) + copy(c, b) + return v.convertToFixedLenByteArray(c), nil +} + +func convertDoubleToString(v Value) (Value, error) { + return v.convertToByteArray(strconv.AppendFloat(nil, v.double(), 'g', -1, 64)), nil +} + +func convertByteArrayToBoolean(v Value) (Value, error) { + return v.convertToBoolean(!isZero(v.byteArray())), nil +} + +func convertByteArrayToInt32(v Value) (Value, error) { + b := make([]byte, 4) + copy(b, v.byteArray()) + return v.convertToInt32(int32(binary.LittleEndian.Uint32(b))), nil +} + +func convertByteArrayToInt64(v Value) (Value, error) { + b := make([]byte, 8) + copy(b, v.byteArray()) + return v.convertToInt64(int64(binary.LittleEndian.Uint64(b))), nil +} + +func convertByteArrayToInt96(v Value) (Value, error) { + b := make([]byte, 12) + copy(b, v.byteArray()) + return v.convertToInt96(deprecated.Int96{ + 0: binary.LittleEndian.Uint32(b[0:4]), + 1: binary.LittleEndian.Uint32(b[4:8]), + 2: binary.LittleEndian.Uint32(b[8:12]), + }), nil +} + +func convertByteArrayToFloat(v Value) (Value, error) { + b := make([]byte, 4) + copy(b, v.byteArray()) + return v.convertToFloat(math.Float32frombits(binary.LittleEndian.Uint32(b))), nil +} + +func convertByteArrayToDouble(v Value) (Value, error) { + b := make([]byte, 8) + copy(b, v.byteArray()) + return v.convertToDouble(math.Float64frombits(binary.LittleEndian.Uint64(b))), nil +} + +func convertByteArrayToFixedLenByteArray(v Value, size int) (Value, error) { + b := v.byteArray() + if len(b) < size { + c := make([]byte, size) + copy(c, b) + b = c + } else { + b = b[:size] + } + return v.convertToFixedLenByteArray(b), nil +} + +func convertFixedLenByteArrayToString(v Value) (Value, error) { + b := v.byteArray() + c := make([]byte, hex.EncodedLen(len(b))) + hex.Encode(c, b) + return v.convertToByteArray(c), nil +} + +func convertStringToBoolean(v Value) (Value, error) { + b, err := strconv.ParseBool(v.string()) + if err != nil { + return v, conversionError(v, "STRING", "BOOLEAN", err) + } + return v.convertToBoolean(b), nil +} + +func convertStringToInt32(v Value) (Value, error) { + i, err := strconv.ParseInt(v.string(), 10, 32) + if err != nil { + return v, conversionError(v, "STRING", "INT32", err) + } + return v.convertToInt32(int32(i)), nil +} + +func convertStringToInt64(v Value) (Value, error) { + i, err := strconv.ParseInt(v.string(), 10, 64) + if err != nil { + return v, conversionError(v, "STRING", "INT64", err) + } + return v.convertToInt64(i), nil +} + +func convertStringToInt96(v Value) (Value, error) { + i, ok := new(big.Int).SetString(v.string(), 10) + if !ok { + return v, conversionError(v, "STRING", "INT96", strconv.ErrSyntax) + } + b := i.Bytes() + c := make([]byte, 12) + copy(c, b) + if cpu.IsBigEndian { + bufLen := len(c) + for idx := 0; idx < bufLen; idx = idx + 4 { + for m, n := (idx + 0), (idx + 3); m < n; m, n = m+1, n-1 { + c[m], c[n] = c[n], c[m] + } + } + } + i96 := unsafecast.Slice[deprecated.Int96](c) + return v.convertToInt96(i96[0]), nil +} + +func convertStringToFloat(v Value) (Value, error) { + f, err := strconv.ParseFloat(v.string(), 32) + if err != nil { + return v, conversionError(v, "STRING", "FLOAT", err) + } + return v.convertToFloat(float32(f)), nil +} + +func convertStringToDouble(v Value) (Value, error) { + f, err := strconv.ParseFloat(v.string(), 64) + if err != nil { + return v, conversionError(v, "STRING", "DOUBLE", err) + } + return v.convertToDouble(f), nil +} + +func convertStringToFixedLenByteArray(v Value, size int) (Value, error) { + b := v.byteArray() + c := make([]byte, size) + _, err := hex.Decode(c, b) + if err != nil { + return v, conversionError(v, "STRING", "BYTE_ARRAY", err) + } + return v.convertToFixedLenByteArray(c), nil +} + +func convertStringToDate(v Value, tz *time.Location) (Value, error) { + t, err := time.ParseInLocation("2006-01-02", v.string(), tz) + if err != nil { + return v, conversionError(v, "STRING", "DATE", err) + } + d := daysSinceUnixEpoch(t) + return v.convertToInt32(int32(d)), nil +} + +func convertStringToTimeMillis(v Value, tz *time.Location) (Value, error) { + t, err := time.ParseInLocation("15:04:05.999", v.string(), tz) + if err != nil { + return v, conversionError(v, "STRING", "TIME", err) + } + m := nearestMidnightLessThan(t) + milliseconds := t.Sub(m).Milliseconds() + return v.convertToInt32(int32(milliseconds)), nil +} + +func convertStringToTimeMicros(v Value, tz *time.Location) (Value, error) { + t, err := time.ParseInLocation("15:04:05.999999", v.string(), tz) + if err != nil { + return v, conversionError(v, "STRING", "TIME", err) + } + m := nearestMidnightLessThan(t) + microseconds := t.Sub(m).Microseconds() + return v.convertToInt64(microseconds), nil +} + +func convertDateToTimestamp(v Value, u format.TimeUnit, tz *time.Location) (Value, error) { + t := unixEpoch.AddDate(0, 0, int(v.int32())) + d := timeUnitDuration(u) + return v.convertToInt64(int64(t.In(tz).Sub(unixEpoch) / d)), nil +} + +func convertDateToString(v Value) (Value, error) { + t := unixEpoch.AddDate(0, 0, int(v.int32())) + b := t.AppendFormat(make([]byte, 0, 10), "2006-01-02") + return v.convertToByteArray(b), nil +} + +func convertTimeMillisToString(v Value, tz *time.Location) (Value, error) { + t := time.UnixMilli(int64(v.int32())).In(tz) + b := t.AppendFormat(make([]byte, 0, 12), "15:04:05.999") + return v.convertToByteArray(b), nil +} + +func convertTimeMicrosToString(v Value, tz *time.Location) (Value, error) { + t := time.UnixMicro(v.int64()).In(tz) + b := t.AppendFormat(make([]byte, 0, 15), "15:04:05.999999") + return v.convertToByteArray(b), nil +} + +func convertTimestampToDate(v Value, u format.TimeUnit, tz *time.Location) (Value, error) { + t := timestamp(v, u, tz) + d := daysSinceUnixEpoch(t) + return v.convertToInt32(int32(d)), nil +} + +func convertTimestampToTimeMillis(v Value, u format.TimeUnit, sourceZone, targetZone *time.Location) (Value, error) { + t := timestamp(v, u, sourceZone) + m := nearestMidnightLessThan(t) + milliseconds := t.In(targetZone).Sub(m).Milliseconds() + return v.convertToInt32(int32(milliseconds)), nil +} + +func convertTimestampToTimeMicros(v Value, u format.TimeUnit, sourceZone, targetZone *time.Location) (Value, error) { + t := timestamp(v, u, sourceZone) + m := nearestMidnightLessThan(t) + microseconds := t.In(targetZone).Sub(m).Microseconds() + return v.convertToInt64(int64(microseconds)), nil +} + +func convertTimestampToTimestamp(v Value, sourceUnit, targetUnit format.TimeUnit) (Value, error) { + sourceScale := timeUnitDuration(sourceUnit).Nanoseconds() + targetScale := timeUnitDuration(targetUnit).Nanoseconds() + targetValue := (v.int64() * sourceScale) / targetScale + return v.convertToInt64(targetValue), nil +} + +const nanosecondsPerDay = 24 * 60 * 60 * 1e9 + +func daysSinceUnixEpoch(t time.Time) int { + return int(t.Sub(unixEpoch).Hours()) / 24 +} + +func nearestMidnightLessThan(t time.Time) time.Time { + y, m, d := t.Date() + return time.Date(y, m, d, 0, 0, 0, 0, t.Location()) +} + +func timestamp(v Value, u format.TimeUnit, tz *time.Location) time.Time { + return unixEpoch.In(tz).Add(time.Duration(v.int64()) * timeUnitDuration(u)) +} + +func timeUnitDuration(unit format.TimeUnit) time.Duration { + switch { + case unit.Millis != nil: + return time.Millisecond + case unit.Micros != nil: + return time.Microsecond + default: + return time.Nanosecond + } +} + +func invalidConversion(value Value, from, to string) error { + return fmt.Errorf("%s to %s: %s: %w", from, to, value, ErrInvalidConversion) +} + +func conversionError(value Value, from, to string, err error) error { + return fmt.Errorf("%s to %s: %q: %s: %w", from, to, value.string(), err, ErrInvalidConversion) +} diff --git a/vendor/github.com/parquet-go/parquet-go/dedupe.go b/vendor/github.com/parquet-go/parquet-go/dedupe.go new file mode 100644 index 0000000000..0f43439675 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/dedupe.go @@ -0,0 +1,107 @@ +package parquet + +// DedupeRowReader constructs a row reader which drops duplicated consecutive +// rows, according to the comparator function passed as argument. +// +// If the underlying reader produces a sequence of rows sorted by the same +// comparison predicate, the output is guaranteed to produce unique rows only. +func DedupeRowReader(reader RowReader, compare func(Row, Row) int) RowReader { + return &dedupeRowReader{reader: reader, compare: compare} +} + +type dedupeRowReader struct { + reader RowReader + compare func(Row, Row) int + dedupe +} + +func (d *dedupeRowReader) ReadRows(rows []Row) (int, error) { + for { + n, err := d.reader.ReadRows(rows) + n = d.deduplicate(rows[:n], d.compare) + + if n > 0 || err != nil { + return n, err + } + } +} + +// DedupeRowWriter constructs a row writer which drops duplicated consecutive +// rows, according to the comparator function passed as argument. +// +// If the writer is given a sequence of rows sorted by the same comparison +// predicate, the output is guaranteed to contain unique rows only. +func DedupeRowWriter(writer RowWriter, compare func(Row, Row) int) RowWriter { + return &dedupeRowWriter{writer: writer, compare: compare} +} + +type dedupeRowWriter struct { + writer RowWriter + compare func(Row, Row) int + dedupe + rows []Row +} + +func (d *dedupeRowWriter) WriteRows(rows []Row) (int, error) { + // We need to make a copy because we cannot modify the rows slice received + // as argument to respect the RowWriter contract. + d.rows = append(d.rows[:0], rows...) + defer func() { + for i := range d.rows { + d.rows[i] = Row{} + } + }() + + if n := d.deduplicate(d.rows, d.compare); n > 0 { + w, err := d.writer.WriteRows(d.rows[:n]) + if err != nil { + return w, err + } + } + + // Return the number of rows received instead of the number of deduplicated + // rows actually written to the underlying writer because we have to repsect + // the RowWriter contract. + return len(rows), nil +} + +type dedupe struct { + lastRow Row + uniq []Row + dupe []Row +} + +func (d *dedupe) reset() { + d.lastRow = d.lastRow[:0] +} + +func (d *dedupe) deduplicate(rows []Row, compare func(Row, Row) int) int { + defer func() { + for i := range d.uniq { + d.uniq[i] = Row{} + } + for i := range d.dupe { + d.dupe[i] = Row{} + } + d.uniq = d.uniq[:0] + d.dupe = d.dupe[:0] + }() + + lastRow := d.lastRow + + for _, row := range rows { + if len(lastRow) != 0 && compare(row, lastRow) == 0 { + d.dupe = append(d.dupe, row) + } else { + lastRow = row + d.uniq = append(d.uniq, row) + } + } + + rows = rows[:0] + rows = append(rows, d.uniq...) + rows = append(rows, d.dupe...) + + d.lastRow = append(d.lastRow[:0], lastRow...) + return len(d.uniq) +} diff --git a/vendor/github.com/parquet-go/parquet-go/deprecated/int96.go b/vendor/github.com/parquet-go/parquet-go/deprecated/int96.go new file mode 100644 index 0000000000..fc6d40648e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/deprecated/int96.go @@ -0,0 +1,179 @@ +package deprecated + +import ( + "math/big" + "math/bits" +) + +// Int96 is an implementation of the deprecated INT96 parquet type. +type Int96 [3]uint32 + +// Int32ToInt96 converts a int32 value to a Int96. +func Int32ToInt96(value int32) (i96 Int96) { + if value < 0 { + i96[2] = 0xFFFFFFFF + i96[1] = 0xFFFFFFFF + } + i96[0] = uint32(value) + return +} + +// Int64ToInt96 converts a int64 value to Int96. +func Int64ToInt96(value int64) (i96 Int96) { + if value < 0 { + i96[2] = 0xFFFFFFFF + } + i96[1] = uint32(value >> 32) + i96[0] = uint32(value) + return +} + +// IsZero returns true if i is the zero-value. +func (i Int96) IsZero() bool { return i == Int96{} } + +// Negative returns true if i is a negative value. +func (i Int96) Negative() bool { + return (i[2] >> 31) != 0 +} + +// Less returns true if i < j. +// +// The method implements a signed comparison between the two operands. +func (i Int96) Less(j Int96) bool { + if i.Negative() { + if !j.Negative() { + return true + } + } else { + if j.Negative() { + return false + } + } + for k := 2; k >= 0; k-- { + a, b := i[k], j[k] + switch { + case a < b: + return true + case a > b: + return false + } + } + return false +} + +// Int converts i to a big.Int representation. +func (i Int96) Int() *big.Int { + z := new(big.Int) + z.Or(z, big.NewInt(int64(i[2])<<32|int64(i[1]))) + z.Lsh(z, 32) + z.Or(z, big.NewInt(int64(i[0]))) + return z +} + +// Int32 converts i to a int32, potentially truncating the value. +func (i Int96) Int32() int32 { + return int32(i[0]) +} + +// Int64 converts i to a int64, potentially truncating the value. +func (i Int96) Int64() int64 { + return int64(i[1])<<32 | int64(i[0]) +} + +// String returns a string representation of i. +func (i Int96) String() string { + return i.Int().String() +} + +// Len returns the minimum length in bits required to store the value of i. +func (i Int96) Len() int { + switch { + case i[2] != 0: + return 64 + bits.Len32(i[2]) + case i[1] != 0: + return 32 + bits.Len32(i[1]) + default: + return bits.Len32(i[0]) + } +} + +func MaxLenInt96(data []Int96) int { + max := 0 + for i := range data { + n := data[i].Len() + if n > max { + max = n + } + } + return max +} + +func MinInt96(data []Int96) (min Int96) { + if len(data) > 0 { + min = data[0] + for _, v := range data[1:] { + if v.Less(min) { + min = v + } + } + } + return min +} + +func MaxInt96(data []Int96) (max Int96) { + if len(data) > 0 { + max = data[0] + for _, v := range data[1:] { + if max.Less(v) { + max = v + } + } + } + return max +} + +func MinMaxInt96(data []Int96) (min, max Int96) { + if len(data) > 0 { + min = data[0] + max = data[0] + for _, v := range data[1:] { + if v.Less(min) { + min = v + } + if max.Less(v) { + max = v + } + } + } + return min, max +} + +func OrderOfInt96(data []Int96) int { + if len(data) > 1 { + if int96AreInAscendingOrder(data) { + return +1 + } + if int96AreInDescendingOrder(data) { + return -1 + } + } + return 0 +} + +func int96AreInAscendingOrder(data []Int96) bool { + for i := len(data) - 1; i > 0; i-- { + if data[i].Less(data[i-1]) { + return false + } + } + return true +} + +func int96AreInDescendingOrder(data []Int96) bool { + for i := len(data) - 1; i > 0; i-- { + if data[i-1].Less(data[i]) { + return false + } + } + return true +} diff --git a/vendor/github.com/parquet-go/parquet-go/deprecated/parquet.go b/vendor/github.com/parquet-go/parquet-go/deprecated/parquet.go new file mode 100644 index 0000000000..b2c60072c7 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/deprecated/parquet.go @@ -0,0 +1,112 @@ +package deprecated + +// DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet. +// ConvertedType is superseded by LogicalType. This enum should not be extended. +// +// See LogicalTypes.md for conversion between ConvertedType and LogicalType. +type ConvertedType int32 + +const ( + // a BYTE_ARRAY actually contains UTF8 encoded chars + UTF8 ConvertedType = 0 + + // a map is converted as an optional field containing a repeated key/value pair + Map ConvertedType = 1 + + // a key/value pair is converted into a group of two fields + MapKeyValue ConvertedType = 2 + + // a list is converted into an optional field containing a repeated field for its + // values + List ConvertedType = 3 + + // an enum is converted into a binary field + Enum ConvertedType = 4 + + // A decimal value. + // + // This may be used to annotate binary or fixed primitive types. The + // underlying byte array stores the unscaled value encoded as two's + // complement using big-endian byte order (the most significant byte is the + // zeroth element). The value of the decimal is the value * 10^{-scale}. + // + // This must be accompanied by a (maximum) precision and a scale in the + // SchemaElement. The precision specifies the number of digits in the decimal + // and the scale stores the location of the decimal point. For example 1.23 + // would have precision 3 (3 total digits) and scale 2 (the decimal point is + // 2 digits over). + Decimal ConvertedType = 5 + + // A Date + // + // Stored as days since Unix epoch, encoded as the INT32 physical type. + Date ConvertedType = 6 + + // A time + // + // The total number of milliseconds since midnight. The value is stored + // as an INT32 physical type. + TimeMillis ConvertedType = 7 + + // A time. + // + // The total number of microseconds since midnight. The value is stored as + // an INT64 physical type. + TimeMicros ConvertedType = 8 + + // A date/time combination + // + // Date and time recorded as milliseconds since the Unix epoch. Recorded as + // a physical type of INT64. + TimestampMillis ConvertedType = 9 + + // A date/time combination + // + // Date and time recorded as microseconds since the Unix epoch. The value is + // stored as an INT64 physical type. + TimestampMicros ConvertedType = 10 + + // An unsigned integer value. + // + // The number describes the maximum number of meaningful data bits in + // the stored value. 8, 16 and 32 bit values are stored using the + // INT32 physical type. 64 bit values are stored using the INT64 + // physical type. + Uint8 ConvertedType = 11 + Uint16 ConvertedType = 12 + Uint32 ConvertedType = 13 + Uint64 ConvertedType = 14 + + // A signed integer value. + // + // The number describes the maximum number of meaningful data bits in + // the stored value. 8, 16 and 32 bit values are stored using the + // INT32 physical type. 64 bit values are stored using the INT64 + // physical type. + Int8 ConvertedType = 15 + Int16 ConvertedType = 16 + Int32 ConvertedType = 17 + Int64 ConvertedType = 18 + + // An embedded JSON document + // + // A JSON document embedded within a single UTF8 column. + Json ConvertedType = 19 + + // An embedded BSON document + // + // A BSON document embedded within a single BINARY column. + Bson ConvertedType = 20 + + // An interval of time + // + // This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12 + // This data is composed of three separate little endian unsigned + // integers. Each stores a component of a duration of time. The first + // integer identifies the number of months associated with the duration, + // the second identifies the number of days associated with the duration + // and the third identifies the number of milliseconds associated with + // the provided duration. This duration of time is independent of any + // particular timezone or date. + Interval ConvertedType = 21 +) diff --git a/vendor/github.com/parquet-go/parquet-go/dictionary.go b/vendor/github.com/parquet-go/parquet-go/dictionary.go new file mode 100644 index 0000000000..55c82f5087 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/dictionary.go @@ -0,0 +1,1483 @@ +package parquet + +import ( + "io" + "math/bits" + "unsafe" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/encoding/plain" + "github.com/parquet-go/parquet-go/hashprobe" + "github.com/parquet-go/parquet-go/internal/bitpack" + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" + "slices" +) + +const ( + // Maximum load of probing tables. This parameter configures the balance + // between memory density and compute time of probing operations. Valid + // values are floating point numbers between 0 and 1. + // + // Smaller values result in lower collision probability when inserting + // values in probing tables, but also increase memory utilization. + // + // TODO: make this configurable by the application? + hashprobeTableMaxLoad = 0.85 + + // An estimate of the CPU cache footprint used by insert operations. + // + // This constant is used to determine a useful chunk size depending on the + // size of values being inserted in dictionaries. More values of small size + // can fit in CPU caches, so the inserts can operate on larger chunks. + insertsTargetCacheFootprint = 8192 +) + +// The Dictionary interface represents type-specific implementations of parquet +// dictionaries. +// +// Programs can instantiate dictionaries by call the NewDictionary method of a +// Type object. +// +// The current implementation has a limitation which prevents applications from +// providing custom versions of this interface because it contains unexported +// methods. The only way to create Dictionary values is to call the +// NewDictionary of Type instances. This limitation may be lifted in future +// releases. +type Dictionary interface { + // Returns the type that the dictionary was created from. + Type() Type + + // Returns the number of value indexed in the dictionary. + Len() int + + // Returns the dictionary value at the given index. + Index(index int32) Value + + // Inserts values from the second slice to the dictionary and writes the + // indexes at which each value was inserted to the first slice. + // + // The method panics if the length of the indexes slice is smaller than the + // length of the values slice. + Insert(indexes []int32, values []Value) + + // Given an array of dictionary indexes, lookup the values into the array + // of values passed as second argument. + // + // The method panics if len(indexes) > len(values), or one of the indexes + // is negative or greater than the highest index in the dictionary. + Lookup(indexes []int32, values []Value) + + // Returns the min and max values found in the given indexes. + Bounds(indexes []int32) (min, max Value) + + // Resets the dictionary to its initial state, removing all values. + Reset() + + // Returns a Page representing the content of the dictionary. + // + // The returned page shares the underlying memory of the buffer, it remains + // valid to use until the dictionary's Reset method is called. + Page() Page + + // See ColumnBuffer.writeValues for details on the use of unexported methods + // on interfaces. + insert(indexes []int32, rows sparse.Array) + //lookup(indexes []int32, rows sparse.Array) +} + +func checkLookupIndexBounds(indexes []int32, rows sparse.Array) { + if rows.Len() < len(indexes) { + panic("dictionary lookup with more indexes than values") + } +} + +// The boolean dictionary always contains two values for true and false. +type booleanDictionary struct { + booleanPage + // There are only two possible values for booleans, false and true. + // Rather than using a Go map, we track the indexes of each values + // in an array of two 32 bits integers. When inserting values in the + // dictionary, we ensure that an index exist for each boolean value, + // then use the value 0 or 1 (false or true) to perform a lookup in + // the dictionary's map. + table [2]int32 +} + +func newBooleanDictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *booleanDictionary { + indexOfFalse, indexOfTrue, values := int32(-1), int32(-1), data.Boolean() + + for i := int32(0); i < numValues && indexOfFalse < 0 && indexOfTrue < 0; i += 8 { + v := values[i] + if v != 0x00 { + indexOfTrue = i + int32(bits.TrailingZeros8(v)) + } + if v != 0xFF { + indexOfFalse = i + int32(bits.TrailingZeros8(^v)) + } + } + + return &booleanDictionary{ + booleanPage: booleanPage{ + typ: typ, + bits: values[:bitpack.ByteCount(uint(numValues))], + numValues: numValues, + columnIndex: ^columnIndex, + }, + table: [2]int32{ + 0: indexOfFalse, + 1: indexOfTrue, + }, + } +} + +func (d *booleanDictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *booleanDictionary) Len() int { return int(d.numValues) } + +func (d *booleanDictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *booleanDictionary) index(i int32) bool { return d.valueAt(int(i)) } + +func (d *booleanDictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfBool)) +} + +func (d *booleanDictionary) insert(indexes []int32, rows sparse.Array) { + _ = indexes[:rows.Len()] + + if d.table[0] < 0 { + d.table[0] = d.numValues + d.numValues++ + d.bits = plain.AppendBoolean(d.bits, int(d.table[0]), false) + } + + if d.table[1] < 0 { + d.table[1] = d.numValues + d.numValues++ + d.bits = plain.AppendBoolean(d.bits, int(d.table[1]), true) + } + + values := rows.Uint8Array() + dict := d.table + + for i := range rows.Len() { + v := values.Index(i) & 1 + indexes[i] = dict[v] + } +} + +func (d *booleanDictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(false) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *booleanDictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*bool)(rows.Index(i)) = d.index(j) + } +} + +func (d *booleanDictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + hasFalse, hasTrue := false, false + + for _, i := range indexes { + v := d.index(i) + if v { + hasTrue = true + } else { + hasFalse = true + } + if hasTrue && hasFalse { + break + } + } + + min = d.makeValue(!hasFalse) + max = d.makeValue(hasTrue) + } + return min, max +} + +func (d *booleanDictionary) Reset() { + d.bits = d.bits[:0] + d.offset = 0 + d.numValues = 0 + d.table = [2]int32{-1, -1} +} + +func (d *booleanDictionary) Page() Page { + return &d.booleanPage +} + +type int32Dictionary struct { + int32Page + table *hashprobe.Int32Table +} + +func newInt32Dictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *int32Dictionary { + return &int32Dictionary{ + int32Page: int32Page{ + typ: typ, + values: data.Int32()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *int32Dictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *int32Dictionary) Len() int { return len(d.values) } + +func (d *int32Dictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *int32Dictionary) index(i int32) int32 { return d.values[i] } + +func (d *int32Dictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfU32)) +} + +func (d *int32Dictionary) init(indexes []int32) { + d.table = hashprobe.NewInt32Table(len(d.values), hashprobeTableMaxLoad) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *int32Dictionary) insert(indexes []int32, rows sparse.Array) { + // Iterating over the input in chunks helps keep relevant data in CPU + // caches when a large number of values are inserted into the dictionary with + // a single method call. + // + // Without this chunking, memory areas from the head of the indexes and + // values arrays end up being evicted from CPU caches as the probing + // operation iterates through the array. The subsequent scan of the indexes + // required to determine which values must be inserted into the page then + // stalls on retrieving data from main memory. + // + // We measured as much as ~37% drop in throughput when disabling the + // chunking, and did not observe any penalties from having it on smaller + // inserts. + const chunkSize = insertsTargetCacheFootprint / 4 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Int32Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *int32Dictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(0) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU32)) +} + +func (d *int32Dictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *int32Dictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *int32Dictionary) Page() Page { + return &d.int32Page +} + +type int64Dictionary struct { + int64Page + table *hashprobe.Int64Table +} + +func newInt64Dictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *int64Dictionary { + return &int64Dictionary{ + int64Page: int64Page{ + typ: typ, + values: data.Int64()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *int64Dictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *int64Dictionary) Len() int { return len(d.values) } + +func (d *int64Dictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *int64Dictionary) index(i int32) int64 { return d.values[i] } + +func (d *int64Dictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *int64Dictionary) init(indexes []int32) { + d.table = hashprobe.NewInt64Table(len(d.values), hashprobeTableMaxLoad) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *int64Dictionary) insert(indexes []int32, rows sparse.Array) { + const chunkSize = insertsTargetCacheFootprint / 8 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Int64Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *int64Dictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(0) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *int64Dictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *int64Dictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *int64Dictionary) Page() Page { + return &d.int64Page +} + +type int96Dictionary struct { + int96Page + hashmap map[deprecated.Int96]int32 +} + +func newInt96Dictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *int96Dictionary { + return &int96Dictionary{ + int96Page: int96Page{ + typ: typ, + values: data.Int96()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *int96Dictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *int96Dictionary) Len() int { return len(d.values) } + +func (d *int96Dictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *int96Dictionary) index(i int32) deprecated.Int96 { return d.values[i] } + +func (d *int96Dictionary) Insert(indexes []int32, values []Value) { + d.insertValues(indexes, len(values), func(i int) deprecated.Int96 { + return values[i].Int96() + }) +} + +func (d *int96Dictionary) insert(indexes []int32, rows sparse.Array) { + d.insertValues(indexes, rows.Len(), func(i int) deprecated.Int96 { + return *(*deprecated.Int96)(rows.Index(i)) + }) +} + +func (d *int96Dictionary) insertValues(indexes []int32, count int, valueAt func(int) deprecated.Int96) { + _ = indexes[:count] + + if d.hashmap == nil { + d.hashmap = make(map[deprecated.Int96]int32, len(d.values)) + for i, v := range d.values { + d.hashmap[v] = int32(i) + } + } + + for i := range count { + value := valueAt(i) + + index, exists := d.hashmap[value] + if !exists { + index = int32(len(d.values)) + d.values = append(d.values, value) + d.hashmap[value] = index + } + + indexes[i] = index + } +} + +func (d *int96Dictionary) Lookup(indexes []int32, values []Value) { + for i, j := range indexes { + values[i] = d.Index(j) + } +} + +func (d *int96Dictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue := d.index(indexes[0]) + maxValue := minValue + + for _, i := range indexes[1:] { + value := d.index(i) + switch { + case value.Less(minValue): + minValue = value + case maxValue.Less(value): + maxValue = value + } + } + + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *int96Dictionary) Reset() { + d.values = d.values[:0] + d.hashmap = nil +} + +func (d *int96Dictionary) Page() Page { + return &d.int96Page +} + +type floatDictionary struct { + floatPage + table *hashprobe.Float32Table +} + +func newFloatDictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *floatDictionary { + return &floatDictionary{ + floatPage: floatPage{ + typ: typ, + values: data.Float()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *floatDictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *floatDictionary) Len() int { return len(d.values) } + +func (d *floatDictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *floatDictionary) index(i int32) float32 { return d.values[i] } + +func (d *floatDictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfU32)) +} + +func (d *floatDictionary) init(indexes []int32) { + d.table = hashprobe.NewFloat32Table(len(d.values), hashprobeTableMaxLoad) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *floatDictionary) insert(indexes []int32, rows sparse.Array) { + const chunkSize = insertsTargetCacheFootprint / 4 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Float32Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *floatDictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(0) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU32)) +} + +func (d *floatDictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *floatDictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *floatDictionary) Page() Page { + return &d.floatPage +} + +type doubleDictionary struct { + doublePage + table *hashprobe.Float64Table +} + +func newDoubleDictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *doubleDictionary { + return &doubleDictionary{ + doublePage: doublePage{ + typ: typ, + values: data.Double()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *doubleDictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *doubleDictionary) Len() int { return len(d.values) } + +func (d *doubleDictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *doubleDictionary) index(i int32) float64 { return d.values[i] } + +func (d *doubleDictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *doubleDictionary) init(indexes []int32) { + d.table = hashprobe.NewFloat64Table(len(d.values), hashprobeTableMaxLoad) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *doubleDictionary) insert(indexes []int32, rows sparse.Array) { + const chunkSize = insertsTargetCacheFootprint / 8 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Float64Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *doubleDictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(0) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *doubleDictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *doubleDictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *doubleDictionary) Page() Page { + return &d.doublePage +} + +type byteArrayDictionary struct { + byteArrayPage + table map[string]int32 + alloc allocator +} + +func newByteArrayDictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *byteArrayDictionary { + values, offsets := data.ByteArray() + // The first offset must always be zero, and the last offset is the length + // of the values in bytes. + // + // As an optimization we make the assumption that the backing array of the + // offsets slice belongs to the dictionary. + switch { + case cap(offsets) == 0: + offsets = make([]uint32, 1, 8) + case len(offsets) == 0: + offsets = append(offsets[:0], 0) + } + return &byteArrayDictionary{ + byteArrayPage: byteArrayPage{ + typ: typ, + values: values, + offsets: offsets, + columnIndex: ^columnIndex, + }, + } +} + +func (d *byteArrayDictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *byteArrayDictionary) Len() int { return d.len() } + +func (d *byteArrayDictionary) Index(i int32) Value { return d.makeValueBytes(d.index(int(i))) } + +func (d *byteArrayDictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfPtr)) +} + +func (d *byteArrayDictionary) init() { + numValues := d.len() + d.table = make(map[string]int32, numValues) + + for i := range numValues { + d.table[string(d.index(i))] = int32(len(d.table)) + } +} + +func (d *byteArrayDictionary) insert(indexes []int32, rows sparse.Array) { + if d.table == nil { + d.init() + } + + values := rows.StringArray() + + for i := range indexes { + value := values.Index(i) + + index, exists := d.table[value] + if !exists { + value = d.alloc.copyString(value) + index = int32(len(d.table)) + d.table[value] = index + d.values = append(d.values, value...) + d.offsets = append(d.offsets, uint32(len(d.values))) + } + + indexes[i] = index + } +} + +func (d *byteArrayDictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValueString("") + memsetValues(values, model) + d.lookupString(indexes, makeArrayValue(values, offsetOfPtr)) +} + +func (d *byteArrayDictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + base := d.index(int(indexes[0])) + minValue := unsafecast.String(base) + maxValue := minValue + values := [64]string{} + + for i := 1; i < len(indexes); i += len(values) { + n := len(indexes) - i + if n > len(values) { + n = len(values) + } + j := i + n + d.lookupString(indexes[i:j:j], makeArrayString(values[:n:n])) + + for _, value := range values[:n:n] { + switch { + case value < minValue: + minValue = value + case value > maxValue: + maxValue = value + } + } + } + + min = d.makeValueString(minValue) + max = d.makeValueString(maxValue) + } + return min, max +} + +func (d *byteArrayDictionary) Reset() { + d.offsets = d.offsets[:1] + d.values = d.values[:0] + for k := range d.table { + delete(d.table, k) + } + d.alloc.reset() +} + +func (d *byteArrayDictionary) Page() Page { + return &d.byteArrayPage +} + +type fixedLenByteArrayDictionary struct { + fixedLenByteArrayPage + hashmap map[string]int32 +} + +func newFixedLenByteArrayDictionary(typ Type, columnIndex int16, numValues int32, values encoding.Values) *fixedLenByteArrayDictionary { + data, size := values.FixedLenByteArray() + return &fixedLenByteArrayDictionary{ + fixedLenByteArrayPage: fixedLenByteArrayPage{ + typ: typ, + size: size, + data: data, + columnIndex: ^columnIndex, + }, + } +} + +func (d *fixedLenByteArrayDictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *fixedLenByteArrayDictionary) Len() int { return len(d.data) / d.size } + +func (d *fixedLenByteArrayDictionary) Index(i int32) Value { + return d.makeValueBytes(d.index(i)) +} + +func (d *fixedLenByteArrayDictionary) index(i int32) []byte { + j := (int(i) + 0) * d.size + k := (int(i) + 1) * d.size + return d.data[j:k:k] +} + +func (d *fixedLenByteArrayDictionary) Insert(indexes []int32, values []Value) { + d.insertValues(indexes, len(values), func(i int) *byte { + return values[i].ptr + }) +} + +func (d *fixedLenByteArrayDictionary) insert(indexes []int32, rows sparse.Array) { + d.insertValues(indexes, rows.Len(), func(i int) *byte { + return (*byte)(rows.Index(i)) + }) +} + +func (d *fixedLenByteArrayDictionary) insertValues(indexes []int32, count int, valueAt func(int) *byte) { + _ = indexes[:count] + + if d.hashmap == nil { + d.hashmap = make(map[string]int32, cap(d.data)/d.size) + for i, j := 0, int32(0); i < len(d.data); i += d.size { + d.hashmap[string(d.data[i:i+d.size])] = j + j++ + } + } + + for i := range count { + value := unsafe.Slice(valueAt(i), d.size) + + index, exists := d.hashmap[string(value)] + if !exists { + index = int32(d.Len()) + start := len(d.data) + d.data = append(d.data, value...) + d.hashmap[string(d.data[start:])] = index + } + + indexes[i] = index + } +} + +func (d *fixedLenByteArrayDictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValueString("") + memsetValues(values, model) + d.lookupString(indexes, makeArrayValue(values, offsetOfPtr)) +} + +func (d *fixedLenByteArrayDictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + base := d.index(indexes[0]) + minValue := unsafecast.String(base) + maxValue := minValue + values := [64]string{} + + for i := 1; i < len(indexes); i += len(values) { + n := len(indexes) - i + if n > len(values) { + n = len(values) + } + j := i + n + d.lookupString(indexes[i:j:j], makeArrayString(values[:n:n])) + + for _, value := range values[:n:n] { + switch { + case value < minValue: + minValue = value + case value > maxValue: + maxValue = value + } + } + } + + min = d.makeValueString(minValue) + max = d.makeValueString(maxValue) + } + return min, max +} + +func (d *fixedLenByteArrayDictionary) Reset() { + d.data = d.data[:0] + d.hashmap = nil +} + +func (d *fixedLenByteArrayDictionary) Page() Page { + return &d.fixedLenByteArrayPage +} + +type uint32Dictionary struct { + uint32Page + table *hashprobe.Uint32Table +} + +func newUint32Dictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *uint32Dictionary { + return &uint32Dictionary{ + uint32Page: uint32Page{ + typ: typ, + values: data.Uint32()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *uint32Dictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *uint32Dictionary) Len() int { return len(d.values) } + +func (d *uint32Dictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *uint32Dictionary) index(i int32) uint32 { return d.values[i] } + +func (d *uint32Dictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfU32)) +} + +func (d *uint32Dictionary) init(indexes []int32) { + d.table = hashprobe.NewUint32Table(len(d.values), hashprobeTableMaxLoad) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *uint32Dictionary) insert(indexes []int32, rows sparse.Array) { + const chunkSize = insertsTargetCacheFootprint / 4 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Uint32Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *uint32Dictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(0) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU32)) +} + +func (d *uint32Dictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *uint32Dictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *uint32Dictionary) Page() Page { + return &d.uint32Page +} + +type uint64Dictionary struct { + uint64Page + table *hashprobe.Uint64Table +} + +func newUint64Dictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *uint64Dictionary { + return &uint64Dictionary{ + uint64Page: uint64Page{ + typ: typ, + values: data.Uint64()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *uint64Dictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *uint64Dictionary) Len() int { return len(d.values) } + +func (d *uint64Dictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *uint64Dictionary) index(i int32) uint64 { return d.values[i] } + +func (d *uint64Dictionary) Insert(indexes []int32, values []Value) { + d.insert(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *uint64Dictionary) init(indexes []int32) { + d.table = hashprobe.NewUint64Table(len(d.values), hashprobeTableMaxLoad) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *uint64Dictionary) insert(indexes []int32, rows sparse.Array) { + const chunkSize = insertsTargetCacheFootprint / 8 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Uint64Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *uint64Dictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValue(0) + memsetValues(values, model) + d.lookup(indexes, makeArrayValue(values, offsetOfU64)) +} + +func (d *uint64Dictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *uint64Dictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *uint64Dictionary) Page() Page { + return &d.uint64Page +} + +type be128Dictionary struct { + be128Page + table *hashprobe.Uint128Table +} + +func newBE128Dictionary(typ Type, columnIndex int16, numValues int32, data encoding.Values) *be128Dictionary { + return &be128Dictionary{ + be128Page: be128Page{ + typ: typ, + values: data.Uint128()[:numValues], + columnIndex: ^columnIndex, + }, + } +} + +func (d *be128Dictionary) Type() Type { return newIndexedType(d.typ, d) } + +func (d *be128Dictionary) Len() int { return len(d.values) } + +func (d *be128Dictionary) Index(i int32) Value { return d.makeValue(d.index(i)) } + +func (d *be128Dictionary) index(i int32) *[16]byte { return &d.values[i] } + +func (d *be128Dictionary) Insert(indexes []int32, values []Value) { + _ = indexes[:len(values)] + + for _, v := range values { + if v.kind != ^int8(FixedLenByteArray) { + panic("values inserted in BE128 dictionary must be of type BYTE_ARRAY") + } + if v.u64 != 16 { + panic("values inserted in BE128 dictionary must be of length 16") + } + } + + if d.table == nil { + d.init(indexes) + } + + const chunkSize = insertsTargetCacheFootprint / 16 + var buffer [chunkSize][16]byte + + for i := 0; i < len(values); i += chunkSize { + j := min(chunkSize+i, len(values)) + n := min(chunkSize, len(values)-i) + + probe := buffer[:n:n] + writePointersBE128(probe, makeArrayValue(values[i:j], unsafe.Offsetof(values[i].ptr))) + + if d.table.Probe(probe, indexes[i:j:j]) > 0 { + for k, v := range probe { + if indexes[i+k] == int32(len(d.values)) { + d.values = append(d.values, v) + } + } + } + } +} + +func (d *be128Dictionary) init(indexes []int32) { + d.table = hashprobe.NewUint128Table(len(d.values), 0.75) + + n := min(len(d.values), len(indexes)) + + for i := 0; i < len(d.values); i += n { + j := min(i+n, len(d.values)) + d.table.Probe(d.values[i:j:j], indexes[:n:n]) + } +} + +func (d *be128Dictionary) insert(indexes []int32, rows sparse.Array) { + const chunkSize = insertsTargetCacheFootprint / 16 + + if d.table == nil { + d.init(indexes) + } + + values := rows.Uint128Array() + + for i := 0; i < values.Len(); i += chunkSize { + j := min(i+chunkSize, values.Len()) + + if d.table.ProbeArray(values.Slice(i, j), indexes[i:j:j]) > 0 { + for k, index := range indexes[i:j] { + if index == int32(len(d.values)) { + d.values = append(d.values, values.Index(i+k)) + } + } + } + } +} + +func (d *be128Dictionary) Lookup(indexes []int32, values []Value) { + model := d.makeValueString("") + memsetValues(values, model) + d.lookupString(indexes, makeArrayValue(values, offsetOfPtr)) +} + +func (d *be128Dictionary) Bounds(indexes []int32) (min, max Value) { + if len(indexes) > 0 { + minValue, maxValue := d.bounds(indexes) + min = d.makeValue(minValue) + max = d.makeValue(maxValue) + } + return min, max +} + +func (d *be128Dictionary) Reset() { + d.values = d.values[:0] + if d.table != nil { + d.table.Reset() + } +} + +func (d *be128Dictionary) Page() Page { + return &d.be128Page +} + +// indexedType is a wrapper around a Type value which overrides object +// constructors to use indexed versions referencing values in the dictionary +// instead of storing plain values. +type indexedType struct { + Type + dict Dictionary +} + +func newIndexedType(typ Type, dict Dictionary) *indexedType { + return &indexedType{Type: typ, dict: dict} +} + +func (t *indexedType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newIndexedColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t *indexedType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newIndexedPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +// indexedPage is an implementation of the Page interface which stores +// indexes instead of plain value. The indexes reference the values in a +// dictionary that the page was created for. +type indexedPage struct { + typ *indexedType + values []int32 + columnIndex int16 +} + +func newIndexedPage(typ *indexedType, columnIndex int16, numValues int32, data encoding.Values) *indexedPage { + // RLE encoded values that contain dictionary indexes in data pages are + // sometimes truncated when they contain only zeros. We account for this + // special case here and extend the values buffer if it is shorter than + // needed to hold `numValues`. + size := int(numValues) + values := data.Int32() + + if len(values) < size { + if cap(values) < size { + tmp := make([]int32, size) + copy(tmp, values) + values = tmp + } else { + clear := values[len(values):size] + for i := range clear { + clear[i] = 0 + } + } + } + + return &indexedPage{ + typ: typ, + values: values[:size], + columnIndex: ^columnIndex, + } +} + +func (page *indexedPage) Type() Type { return indexedPageType{page.typ} } + +func (page *indexedPage) Column() int { return int(^page.columnIndex) } + +func (page *indexedPage) Dictionary() Dictionary { return page.typ.dict } + +func (page *indexedPage) NumRows() int64 { return int64(len(page.values)) } + +func (page *indexedPage) NumValues() int64 { return int64(len(page.values)) } + +func (page *indexedPage) NumNulls() int64 { return 0 } + +func (page *indexedPage) Size() int64 { return 4 * int64(len(page.values)) } + +func (page *indexedPage) RepetitionLevels() []byte { return nil } + +func (page *indexedPage) DefinitionLevels() []byte { return nil } + +func (page *indexedPage) Data() encoding.Values { return encoding.Int32Values(page.values) } + +func (page *indexedPage) Values() ValueReader { return &indexedPageValues{page: page} } + +func (page *indexedPage) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + min, max = page.typ.dict.Bounds(page.values) + min.columnIndex = page.columnIndex + max.columnIndex = page.columnIndex + } + return min, max, ok +} + +func (page *indexedPage) Slice(i, j int64) Page { + return &indexedPage{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +// indexedPageType is an adapter for the indexedType returned when accessing +// the type of an indexedPage value. It overrides the Encode/Decode methods to +// account for the fact that an indexed page is holding indexes of values into +// its dictionary instead of plain values. +type indexedPageType struct{ *indexedType } + +func (t indexedPageType) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.Int32ValuesFromBytes(values) +} + +func (t indexedPageType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeInt32(dst, src, enc) +} + +func (t indexedPageType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeInt32(dst, src, enc) +} + +func (t indexedPageType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return Int32Type.EstimateDecodeSize(numValues, src, enc) +} + +type indexedPageValues struct { + page *indexedPage + offset int +} + +func (r *indexedPageValues) ReadValues(values []Value) (n int, err error) { + if n = len(r.page.values) - r.offset; n == 0 { + return 0, io.EOF + } + if n > len(values) { + n = len(values) + } + r.page.typ.dict.Lookup(r.page.values[r.offset:r.offset+n], values[:n]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +// indexedColumnBuffer is an implementation of the ColumnBuffer interface which +// builds a page of indexes into a parent dictionary when values are written. +type indexedColumnBuffer struct{ indexedPage } + +func newIndexedColumnBuffer(typ *indexedType, columnIndex int16, numValues int32) *indexedColumnBuffer { + return &indexedColumnBuffer{ + indexedPage: indexedPage{ + typ: typ, + values: make([]int32, 0, numValues), + columnIndex: ^columnIndex, + }, + } +} + +func (col *indexedColumnBuffer) Clone() ColumnBuffer { + return &indexedColumnBuffer{ + indexedPage: indexedPage{ + typ: col.typ, + values: slices.Clone(col.values), + columnIndex: col.columnIndex, + }, + } +} + +func (col *indexedColumnBuffer) Type() Type { return col.typ.Type } + +func (col *indexedColumnBuffer) ColumnIndex() (ColumnIndex, error) { + return indexedColumnIndex{col}, nil +} + +func (col *indexedColumnBuffer) OffsetIndex() (OffsetIndex, error) { + return indexedOffsetIndex{col}, nil +} + +func (col *indexedColumnBuffer) BloomFilter() BloomFilter { return nil } + +func (col *indexedColumnBuffer) Dictionary() Dictionary { return col.typ.dict } + +func (col *indexedColumnBuffer) Pages() Pages { return onePage(col.Page()) } + +func (col *indexedColumnBuffer) Page() Page { return &col.indexedPage } + +func (col *indexedColumnBuffer) Reset() { col.values = col.values[:0] } + +func (col *indexedColumnBuffer) Cap() int { return cap(col.values) } + +func (col *indexedColumnBuffer) Len() int { return len(col.values) } + +func (col *indexedColumnBuffer) Less(i, j int) bool { + u := col.typ.dict.Index(col.values[i]) + v := col.typ.dict.Index(col.values[j]) + return col.typ.Compare(u, v) < 0 +} + +func (col *indexedColumnBuffer) Swap(i, j int) { + col.values[i], col.values[j] = col.values[j], col.values[i] +} + +func (col *indexedColumnBuffer) WriteValues(values []Value) (int, error) { + i := len(col.values) + j := len(col.values) + len(values) + + if j <= cap(col.values) { + col.values = col.values[:j] + } else { + tmp := make([]int32, j, 2*j) + copy(tmp, col.values) + col.values = tmp + } + + col.typ.dict.Insert(col.values[i:], values) + return len(values), nil +} + +func (col *indexedColumnBuffer) writeValues(rows sparse.Array, _ columnLevels) { + i := len(col.values) + j := len(col.values) + rows.Len() + + if j <= cap(col.values) { + col.values = col.values[:j] + } else { + tmp := make([]int32, j, 2*j) + copy(tmp, col.values) + col.values = tmp + } + + col.typ.dict.insert(col.values[i:], rows) +} + +func (col *indexedColumnBuffer) ReadValuesAt(values []Value, offset int64) (n int, err error) { + i := int(offset) + switch { + case i < 0: + return 0, errRowIndexOutOfBounds(offset, int64(len(col.values))) + case i >= len(col.values): + return 0, io.EOF + default: + for n < len(values) && i < len(col.values) { + values[n] = col.typ.dict.Index(col.values[i]) + values[n].columnIndex = col.columnIndex + n++ + i++ + } + if n < len(values) { + err = io.EOF + } + return n, err + } +} + +func (col *indexedColumnBuffer) ReadRowAt(row Row, index int64) (Row, error) { + switch { + case index < 0: + return row, errRowIndexOutOfBounds(index, int64(len(col.values))) + case index >= int64(len(col.values)): + return row, io.EOF + default: + v := col.typ.dict.Index(col.values[index]) + v.columnIndex = col.columnIndex + return append(row, v), nil + } +} + +type indexedColumnIndex struct{ col *indexedColumnBuffer } + +func (index indexedColumnIndex) NumPages() int { return 1 } +func (index indexedColumnIndex) NullCount(int) int64 { return 0 } +func (index indexedColumnIndex) NullPage(int) bool { return false } +func (index indexedColumnIndex) MinValue(int) Value { + min, _, _ := index.col.Bounds() + return min +} +func (index indexedColumnIndex) MaxValue(int) Value { + _, max, _ := index.col.Bounds() + return max +} +func (index indexedColumnIndex) IsAscending() bool { + min, max, _ := index.col.Bounds() + return index.col.typ.Compare(min, max) <= 0 +} +func (index indexedColumnIndex) IsDescending() bool { + min, max, _ := index.col.Bounds() + return index.col.typ.Compare(min, max) > 0 +} + +type indexedOffsetIndex struct{ col *indexedColumnBuffer } + +func (index indexedOffsetIndex) NumPages() int { return 1 } +func (index indexedOffsetIndex) Offset(int) int64 { return 0 } +func (index indexedOffsetIndex) CompressedPageSize(int) int64 { return index.col.Size() } +func (index indexedOffsetIndex) FirstRowIndex(int) int64 { return 0 } diff --git a/vendor/github.com/parquet-go/parquet-go/dictionary_amd64.go b/vendor/github.com/parquet-go/parquet-go/dictionary_amd64.go new file mode 100644 index 0000000000..8d0dedcad4 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/dictionary_amd64.go @@ -0,0 +1,168 @@ +//go:build !purego + +package parquet + +import ( + "unsafe" + + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" +) + +//go:noescape +func dictionaryBoundsInt32(dict []int32, indexes []int32) (min, max int32, err errno) + +//go:noescape +func dictionaryBoundsInt64(dict []int64, indexes []int32) (min, max int64, err errno) + +//go:noescape +func dictionaryBoundsFloat32(dict []float32, indexes []int32) (min, max float32, err errno) + +//go:noescape +func dictionaryBoundsFloat64(dict []float64, indexes []int32) (min, max float64, err errno) + +//go:noescape +func dictionaryBoundsUint32(dict []uint32, indexes []int32) (min, max uint32, err errno) + +//go:noescape +func dictionaryBoundsUint64(dict []uint64, indexes []int32) (min, max uint64, err errno) + +//go:noescape +func dictionaryBoundsBE128(dict [][16]byte, indexes []int32) (min, max *[16]byte, err errno) + +//go:noescape +func dictionaryLookup32(dict []uint32, indexes []int32, rows sparse.Array) errno + +//go:noescape +func dictionaryLookup64(dict []uint64, indexes []int32, rows sparse.Array) errno + +//go:noescape +func dictionaryLookupByteArrayString(dict []uint32, page []byte, indexes []int32, rows sparse.Array) errno + +//go:noescape +func dictionaryLookupFixedLenByteArrayString(dict []byte, len int, indexes []int32, rows sparse.Array) errno + +//go:noescape +func dictionaryLookupFixedLenByteArrayPointer(dict []byte, len int, indexes []int32, rows sparse.Array) errno + +func (d *int32Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + dict := unsafecast.Slice[uint32](d.values) + dictionaryLookup32(dict, indexes, rows).check() +} + +func (d *int64Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + dict := unsafecast.Slice[uint64](d.values) + dictionaryLookup64(dict, indexes, rows).check() +} + +func (d *floatDictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + dict := unsafecast.Slice[uint32](d.values) + dictionaryLookup32(dict, indexes, rows).check() +} + +func (d *doubleDictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + dict := unsafecast.Slice[uint64](d.values) + dictionaryLookup64(dict, indexes, rows).check() +} + +func (d *byteArrayDictionary) lookupString(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + // TODO: this optimization is disabled for now because it appears to race + // with the garbage collector and result in writing pointers to free objects + // to the output. + // + // This command was used to trigger the problem: + // + // GOMAXPROCS=8 go test -run TestIssueSegmentio368 -count 10 + // + // https://github.com/segmentio/parquet-go/issues/368 + // + //dictionaryLookupByteArrayString(d.offsets, d.values, indexes, rows).check() + for i, j := range indexes { + *(*string)(rows.Index(i)) = unsafecast.String(d.index(int(j))) + } +} + +func (d *fixedLenByteArrayDictionary) lookupString(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + //dictionaryLookupFixedLenByteArrayString(d.data, d.size, indexes, rows).check() + for i, j := range indexes { + *(*string)(rows.Index(i)) = unsafecast.String(d.index(j)) + } +} + +func (d *uint32Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + dictionaryLookup32(d.values, indexes, rows).check() +} + +func (d *uint64Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + dictionaryLookup64(d.values, indexes, rows).check() +} + +func (d *be128Dictionary) lookupString(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + //dict := unsafecast.Slice[byte](d.values) + //dictionaryLookupFixedLenByteArrayString(dict, 16, indexes, rows).check() + s := "0123456789ABCDEF" + for i, j := range indexes { + *(**[16]byte)(unsafe.Pointer(&s)) = d.index(j) + *(*string)(rows.Index(i)) = s + } +} + +func (d *be128Dictionary) lookupPointer(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + //dict := unsafecast.Slice[byte](d.values) + //dictionaryLookupFixedLenByteArrayPointer(dict, 16, indexes, rows).check() + for i, j := range indexes { + *(**[16]byte)(rows.Index(i)) = d.index(j) + } +} + +func (d *int32Dictionary) bounds(indexes []int32) (min, max int32) { + min, max, err := dictionaryBoundsInt32(d.values, indexes) + err.check() + return min, max +} + +func (d *int64Dictionary) bounds(indexes []int32) (min, max int64) { + min, max, err := dictionaryBoundsInt64(d.values, indexes) + err.check() + return min, max +} + +func (d *floatDictionary) bounds(indexes []int32) (min, max float32) { + min, max, err := dictionaryBoundsFloat32(d.values, indexes) + err.check() + return min, max +} + +func (d *doubleDictionary) bounds(indexes []int32) (min, max float64) { + min, max, err := dictionaryBoundsFloat64(d.values, indexes) + err.check() + return min, max +} + +func (d *uint32Dictionary) bounds(indexes []int32) (min, max uint32) { + min, max, err := dictionaryBoundsUint32(d.values, indexes) + err.check() + return min, max +} + +func (d *uint64Dictionary) bounds(indexes []int32) (min, max uint64) { + min, max, err := dictionaryBoundsUint64(d.values, indexes) + err.check() + return min, max +} + +func (d *be128Dictionary) bounds(indexes []int32) (min, max *[16]byte) { + min, max, err := dictionaryBoundsBE128(d.values, indexes) + err.check() + return min, max +} diff --git a/vendor/github.com/parquet-go/parquet-go/dictionary_amd64.s b/vendor/github.com/parquet-go/parquet-go/dictionary_amd64.s new file mode 100644 index 0000000000..9372ffbfc0 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/dictionary_amd64.s @@ -0,0 +1,941 @@ +//go:build !purego + +#include "textflag.h" + +#define errnoIndexOutOfBounds 1 + +// func dictionaryBoundsInt32(dict []int32, indexes []int32) (min, max int32, err errno) +TEXT ยทdictionaryBoundsInt32(SB), NOSPLIT, $0-64 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + XORQ R10, R10 // min + XORQ R11, R11 // max + XORQ R12, R12 // err + XORQ SI, SI + + CMPQ DX, $0 + JE return + + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVL (AX)(DI*4), R10 + MOVL R10, R11 + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ $0xFFFF, R8 + KMOVW R8, K1 + + VPBROADCASTD BX, Y2 // [len(dict)...] + VPBROADCASTD R10, Y3 // [min...] + VMOVDQU32 Y3, Y4 // [max...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y2, Y0, K2 + KMOVW K2, R9 + CMPB R9, $0xFF + JNE indexOutOfBounds + VPGATHERDD (AX)(Y0*4), K1, Y1 + VPMINSD Y1, Y3, Y3 + VPMAXSD Y1, Y4, Y4 + KMOVW R8, K1 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + + VPERM2I128 $1, Y3, Y3, Y0 + VPERM2I128 $1, Y4, Y4, Y1 + VPMINSD Y0, Y3, Y3 + VPMAXSD Y1, Y4, Y4 + + VPSHUFD $0b1110, Y3, Y0 + VPSHUFD $0b1110, Y4, Y1 + VPMINSD Y0, Y3, Y3 + VPMAXSD Y1, Y4, Y4 + + VPSHUFD $1, Y3, Y0 + VPSHUFD $1, Y4, Y1 + VPMINSD Y0, Y3, Y3 + VPMAXSD Y1, Y4, Y4 + + MOVQ X3, R10 + MOVQ X4, R11 + ANDQ $0xFFFFFFFF, R10 + ANDQ $0xFFFFFFFF, R11 + + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVL (AX)(DI*4), DI + CMPL DI, R10 + CMOVLLT DI, R10 + CMPL DI, R11 + CMOVLGT DI, R11 + INCQ SI +test: + CMPQ SI, DX + JNE loop +return: + MOVL R10, min+48(FP) + MOVL R11, max+52(FP) + MOVQ R12, err+56(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, R12 + JMP return + +// func dictionaryBoundsInt64(dict []int64, indexes []int32) (min, max int64, err errno) +TEXT ยทdictionaryBoundsInt64(SB), NOSPLIT, $0-72 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + XORQ R10, R10 // min + XORQ R11, R11 // max + XORQ R12, R12 // err + XORQ SI, SI + + CMPQ DX, $0 + JE return + + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVQ (AX)(DI*8), R10 + MOVQ R10, R11 + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ $0xFFFF, R8 + KMOVW R8, K1 + + VPBROADCASTD BX, Y2 // [len(dict)...] + VPBROADCASTQ R10, Z3 // [min...] + VMOVDQU64 Z3, Z4 // [max...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y2, Y0, K2 + KMOVW K2, R9 + CMPB R9, $0xFF + JNE indexOutOfBounds + VPGATHERDQ (AX)(Y0*8), K1, Z1 + VPMINSQ Z1, Z3, Z3 + VPMAXSQ Z1, Z4, Z4 + KMOVW R8, K1 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + + VPERMQ $0b1110, Z3, Z0 + VPERMQ $0b1110, Z4, Z1 + VPMINSQ Z0, Z3, Z3 + VPMAXSQ Z1, Z4, Z4 + + VPERMQ $1, Z3, Z0 + VPERMQ $1, Z4, Z1 + VPMINSQ Z0, Z3, Z3 + VPMAXSQ Z1, Z4, Z4 + + VSHUFF64X2 $2, Z3, Z3, Z0 + VSHUFF64X2 $2, Z4, Z4, Z1 + VPMINSQ Z0, Z3, Z3 + VPMAXSQ Z1, Z4, Z4 + + MOVQ X3, R10 + MOVQ X4, R11 + + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVQ (AX)(DI*8), DI + CMPQ DI, R10 + CMOVQLT DI, R10 + CMPQ DI, R11 + CMOVQGT DI, R11 + INCQ SI +test: + CMPQ SI, DX + JNE loop +return: + MOVQ R10, min+48(FP) + MOVQ R11, max+56(FP) + MOVQ R12, err+64(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, R12 + JMP return + +// func dictionaryBoundsFloat32(dict []float32, indexes []int32) (min, max float32, err errno) +TEXT ยทdictionaryBoundsFloat32(SB), NOSPLIT, $0-64 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + PXOR X3, X3 // min + PXOR X4, X4 // max + XORQ R12, R12 // err + XORQ SI, SI + + CMPQ DX, $0 + JE return + + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVSS (AX)(DI*4), X3 + MOVAPS X3, X4 + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ $0xFFFF, R8 + KMOVW R8, K1 + + VPBROADCASTD BX, Y2 // [len(dict)...] + VPBROADCASTD X3, Y3 // [min...] + VMOVDQU32 Y3, Y4 // [max...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y2, Y0, K2 + KMOVW K2, R9 + CMPB R9, $0xFF + JNE indexOutOfBounds + VPGATHERDD (AX)(Y0*4), K1, Y1 + VMINPS Y1, Y3, Y3 + VMAXPS Y1, Y4, Y4 + KMOVW R8, K1 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + + VPERM2I128 $1, Y3, Y3, Y0 + VPERM2I128 $1, Y4, Y4, Y1 + VMINPS Y0, Y3, Y3 + VMAXPS Y1, Y4, Y4 + + VPSHUFD $0b1110, Y3, Y0 + VPSHUFD $0b1110, Y4, Y1 + VMINPS Y0, Y3, Y3 + VMAXPS Y1, Y4, Y4 + + VPSHUFD $1, Y3, Y0 + VPSHUFD $1, Y4, Y1 + VMINPS Y0, Y3, Y3 + VMAXPS Y1, Y4, Y4 + + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVSS (AX)(DI*4), X1 + UCOMISS X3, X1 + JAE skipAssignMin + MOVAPS X1, X3 +skipAssignMin: + UCOMISS X4, X1 + JBE skipAssignMax + MOVAPS X1, X4 +skipAssignMax: + INCQ SI +test: + CMPQ SI, DX + JNE loop +return: + MOVSS X3, min+48(FP) + MOVSS X4, max+52(FP) + MOVQ R12, err+56(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, R12 + JMP return + +// func dictionaryBoundsFloat64(dict []float64, indexes []int32) (min, max float64, err errno) +TEXT ยทdictionaryBoundsFloat64(SB), NOSPLIT, $0-72 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + PXOR X3, X3 // min + PXOR X4, X4 // max + XORQ R12, R12 // err + XORQ SI, SI + + CMPQ DX, $0 + JE return + + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVSD (AX)(DI*8), X3 + MOVAPS X3, X4 + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ $0xFFFF, R8 + KMOVW R8, K1 + + VPBROADCASTD BX, Y2 // [len(dict)...] + VPBROADCASTQ X3, Z3 // [min...] + VMOVDQU64 Z3, Z4 // [max...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y2, Y0, K2 + KMOVW K2, R9 + CMPB R9, $0xFF + JNE indexOutOfBounds + VPGATHERDQ (AX)(Y0*8), K1, Z1 + VMINPD Z1, Z3, Z3 + VMAXPD Z1, Z4, Z4 + KMOVW R8, K1 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + + VPERMQ $0b1110, Z3, Z0 + VPERMQ $0b1110, Z4, Z1 + VMINPD Z0, Z3, Z3 + VMAXPD Z1, Z4, Z4 + + VPERMQ $1, Z3, Z0 + VPERMQ $1, Z4, Z1 + VMINPD Z0, Z3, Z3 + VMAXPD Z1, Z4, Z4 + + VSHUFF64X2 $2, Z3, Z3, Z0 + VSHUFF64X2 $2, Z4, Z4, Z1 + VMINPD Z0, Z3, Z3 + VMAXPD Z1, Z4, Z4 + + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVSD (AX)(DI*8), X1 + UCOMISD X3, X1 + JAE skipAssignMin + MOVAPD X1, X3 +skipAssignMin: + UCOMISD X4, X1 + JBE skipAssignMax + MOVAPD X1, X4 +skipAssignMax: + INCQ SI +test: + CMPQ SI, DX + JNE loop +return: + MOVSD X3, min+48(FP) + MOVSD X4, max+56(FP) + MOVQ R12, err+64(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, R12 + JMP return + +// func dictionaryBoundsUint32(dict []uint32, indexes []int32) (min, max uint32, err errno) +TEXT ยทdictionaryBoundsUint32(SB), NOSPLIT, $0-64 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + XORQ R10, R10 // min + XORQ R11, R11 // max + XORQ R12, R12 // err + XORQ SI, SI + + CMPQ DX, $0 + JE return + + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVL (AX)(DI*4), R10 + MOVL R10, R11 + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ $0xFFFF, R8 + KMOVW R8, K1 + + VPBROADCASTD BX, Y2 // [len(dict)...] + VPBROADCASTD R10, Y3 // [min...] + VMOVDQU32 Y3, Y4 // [max...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y2, Y0, K2 + KMOVW K2, R9 + CMPB R9, $0xFF + JNE indexOutOfBounds + VPGATHERDD (AX)(Y0*4), K1, Y1 + VPMINUD Y1, Y3, Y3 + VPMAXUD Y1, Y4, Y4 + KMOVW R8, K1 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + + VPERM2I128 $1, Y3, Y3, Y0 + VPERM2I128 $1, Y4, Y4, Y1 + VPMINUD Y0, Y3, Y3 + VPMAXUD Y1, Y4, Y4 + + VPSHUFD $0b1110, Y3, Y0 + VPSHUFD $0b1110, Y4, Y1 + VPMINUD Y0, Y3, Y3 + VPMAXUD Y1, Y4, Y4 + + VPSHUFD $1, Y3, Y0 + VPSHUFD $1, Y4, Y1 + VPMINUD Y0, Y3, Y3 + VPMAXUD Y1, Y4, Y4 + + MOVQ X3, R10 + MOVQ X4, R11 + ANDQ $0xFFFFFFFF, R10 + ANDQ $0xFFFFFFFF, R11 + + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVL (AX)(DI*4), DI + CMPL DI, R10 + CMOVLCS DI, R10 + CMPL DI, R11 + CMOVLHI DI, R11 + INCQ SI +test: + CMPQ SI, DX + JNE loop +return: + MOVL R10, min+48(FP) + MOVL R11, max+52(FP) + MOVQ R12, err+56(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, R12 + JMP return + +// func dictionaryBoundsUint64(dict []uint64, indexes []int32) (min, max uint64, err errno) +TEXT ยทdictionaryBoundsUint64(SB), NOSPLIT, $0-72 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + XORQ R10, R10 // min + XORQ R11, R11 // max + XORQ R12, R12 // err + XORQ SI, SI + + CMPQ DX, $0 + JE return + + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVQ (AX)(DI*8), R10 + MOVQ R10, R11 + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ $0xFFFF, R8 + KMOVW R8, K1 + + VPBROADCASTD BX, Y2 // [len(dict)...] + VPBROADCASTQ R10, Z3 // [min...] + VMOVDQU64 Z3, Z4 // [max...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y2, Y0, K2 + KMOVW K2, R9 + CMPB R9, $0xFF + JNE indexOutOfBounds + VPGATHERDQ (AX)(Y0*8), K1, Z1 + VPMINUQ Z1, Z3, Z3 + VPMAXUQ Z1, Z4, Z4 + KMOVW R8, K1 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + + VPERMQ $0b1110, Z3, Z0 + VPERMQ $0b1110, Z4, Z1 + VPMINUQ Z0, Z3, Z3 + VPMAXUQ Z1, Z4, Z4 + + VPERMQ $1, Z3, Z0 + VPERMQ $1, Z4, Z1 + VPMINUQ Z0, Z3, Z3 + VPMAXUQ Z1, Z4, Z4 + + VSHUFF64X2 $2, Z3, Z3, Z0 + VSHUFF64X2 $2, Z4, Z4, Z1 + VPMINUQ Z0, Z3, Z3 + VPMAXUQ Z1, Z4, Z4 + + MOVQ X3, R10 + MOVQ X4, R11 + + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVQ (AX)(DI*8), DI + CMPQ DI, R10 + CMOVQCS DI, R10 + CMPQ DI, R11 + CMOVQHI DI, R11 + INCQ SI +test: + CMPQ SI, DX + JNE loop +return: + MOVQ R10, min+48(FP) + MOVQ R11, max+56(FP) + MOVQ R12, err+64(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, R12 + JMP return + +// func dictionaryBoundsBE128(dict [][16]byte, indexes []int32) (min, max *[16]byte, err errno) +TEXT ยทdictionaryBoundsBE128(SB), NOSPLIT, $0-72 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + SHLQ $2, DX // x 4 + ADDQ CX, DX // end + + XORQ R8, R8 // min (pointer) + XORQ R9, R9 // max (pointer) + XORQ SI, SI // err + XORQ DI, DI + + CMPQ DX, $0 + JE return + + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + SHLQ $4, DI // the dictionary contains 16 byte words + LEAQ (AX)(DI*1), R8 + MOVQ R8, R9 + MOVQ 0(AX)(DI*1), R10 // min (high) + MOVQ 8(AX)(DI*1), R11 // min (low) + BSWAPQ R10 + BSWAPQ R11 + MOVQ R10, R12 // max (high) + MOVQ R11, R13 // max (low) + + JMP next +loop: + MOVL (CX), DI + CMPL DI, BX + JAE indexOutOfBounds + SHLQ $4, DI + MOVQ 0(AX)(DI*1), R14 + MOVQ 8(AX)(DI*1), R15 + BSWAPQ R14 + BSWAPQ R15 +testLessThan: + CMPQ R14, R10 + JA testGreaterThan + JB lessThan + CMPQ R15, R11 + JAE testGreaterThan +lessThan: + LEAQ (AX)(DI*1), R8 + MOVQ R14, R10 + MOVQ R15, R11 + JMP next +testGreaterThan: + CMPQ R14, R12 + JB next + JA greaterThan + CMPQ R15, R13 + JBE next +greaterThan: + LEAQ (AX)(DI*1), R9 + MOVQ R14, R12 + MOVQ R15, R13 +next: + ADDQ $4, CX + CMPQ CX, DX + JNE loop +return: + MOVQ R8, min+48(FP) + MOVQ R9, max+56(FP) + MOVQ SI, err+64(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, SI + JMP return + +// The lookup functions provide optimized versions of the dictionary index +// lookup logic. +// +// When AVX512 is available, the AVX512 versions of the functions are used +// which use the VPGATHER* instructions to perform 8 parallel lookups of the +// values in the dictionary, then VPSCATTER* to do 8 parallel writes to the +// sparse output buffer. + +// func dictionaryLookup32(dict []uint32, indexes []int32, rows sparse.Array) errno +TEXT ยทdictionaryLookup32(SB), NOSPLIT, $0-80 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + MOVQ rows_array_ptr+48(FP), R8 + MOVQ rows_array_off+64(FP), R9 + + XORQ SI, SI + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ R9, R10 + SHLQ $3, R10 // 8 * size + + MOVW $0xFFFF, R11 + KMOVW R11, K1 + KMOVW R11, K2 + + VPBROADCASTD R9, Y2 // [size...] + VPMULLD ยทrange0n8(SB), Y2, Y2 // [0*size,1*size,...] + VPBROADCASTD BX, Y3 // [len(dict)...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y3, Y0, K3 + KMOVW K3, R11 + CMPB R11, $0xFF + JNE indexOutOfBounds + VPGATHERDD (AX)(Y0*4), K1, Y1 + VPSCATTERDD Y1, K2, (R8)(Y2*1) + KMOVW R11, K1 + KMOVW R11, K2 + ADDQ R10, R8 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVL (AX)(DI*4), DI + MOVL DI, (R8) + ADDQ R9, R8 + INCQ SI +test: + CMPQ SI, DX + JNE loop + XORQ AX, AX +return: + MOVQ AX, ret+72(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, AX + JMP return + +// func dictionaryLookup64(dict []uint64, indexes []int32, rows sparse.Array) errno +TEXT ยทdictionaryLookup64(SB), NOSPLIT, $0-80 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ indexes_base+24(FP), CX + MOVQ indexes_len+32(FP), DX + + MOVQ rows_array_ptr+48(FP), R8 + MOVQ rows_array_off+64(FP), R9 + + XORQ SI, SI + + CMPQ DX, $8 + JB test + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + + MOVQ R9, R10 + SHLQ $3, R10 // 8 * size + + MOVW $0xFFFF, R11 + KMOVW R11, K1 + KMOVW R11, K2 + + VPBROADCASTD R9, Y2 // [size...] + VPMULLD ยทrange0n8(SB), Y2, Y2 // [0*size,1*size,...] + VPBROADCASTD BX, Y3 // [len(dict)...] +loopAVX512: + VMOVDQU32 (CX)(SI*4), Y0 + VPCMPUD $1, Y3, Y0, K3 + KMOVW K3, R11 + CMPB R11, $0xFF + JNE indexOutOfBounds + VPGATHERDQ (AX)(Y0*8), K1, Z1 + VPSCATTERDQ Z1, K2, (R8)(Y2*1) + KMOVW R11, K1 + KMOVW R11, K2 + ADDQ R10, R8 + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX512 + VZEROUPPER + JMP test +loop: + MOVL (CX)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + MOVQ (AX)(DI*8), DI + MOVQ DI, (R8) + ADDQ R9, R8 + INCQ SI +test: + CMPQ SI, DX + JNE loop + XORQ AX, AX +return: + MOVQ AX, ret+72(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, AX + JMP return + +// func dictionaryLookupByteArrayString(dict []uint32, page []byte, indexes []int32, rows sparse.Array) errno +TEXT ยทdictionaryLookupByteArrayString(SB), NOSPLIT, $0-104 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + DECQ BX // the offsets have the total length as last element + + MOVQ page_base+24(FP), CX + + MOVQ indexes_base+48(FP), R8 + MOVQ indexes_len+56(FP), R9 + + MOVQ rows_array_ptr+72(FP), R10 + MOVQ rows_array_off+88(FP), R11 + + XORQ DI, DI + XORQ SI, SI +loop: + // Load the index that we want to read the value from. This may come from + // user input so we must validate that the indexes are within the bounds of + // the dictionary. + MOVL (R8)(SI*4), DI + CMPL DI, BX + JAE indexOutOfBounds + + // Load the offsets within the dictionary page where the value is stored. + // We trust the offsets to be correct since they are generated internally by + // the dictionary code, there is no need to check that they are within the + // bounds of the dictionary page. + MOVL 0(AX)(DI*4), DX + MOVL 4(AX)(DI*4), DI + + // Compute the length of the value (the difference between two consecutive + // offsets), and the pointer to the first byte of the string value. + SUBL DX, DI + LEAQ (CX)(DX*1), DX + + // Store the length and pointer to the value into the output location. + // The memory layout is expected to hold a pointer and length, which are + // both 64 bits words. This is the layout used by parquet.Value and the Go + // string value type. + MOVQ DX, (R10) + MOVQ DI, 8(R10) + + ADDQ R11, R10 + INCQ SI +test: + CMPQ SI, R9 + JNE loop + XORQ AX, AX +return: + MOVQ AX, ret+96(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, AX + JMP return + +// func dictionaryLookupFixedLenByteArrayString(dict []byte, len int, indexes []int32, rows sparse.Array) errno +TEXT ยทdictionaryLookupFixedLenByteArrayString(SB), NOSPLIT, $0-88 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ len+24(FP), CX + + MOVQ indexes_base+32(FP), DX + MOVQ indexes_len+40(FP), R8 + + MOVQ rows_array_ptr+56(FP), R9 + MOVQ rows_array_off+72(FP), R10 + + XORQ DI, DI + XORQ SI, SI +loop: + MOVL (DX)(SI*4), DI + IMULQ CX, DI + CMPL DI, BX + JAE indexOutOfBounds + + ADDQ AX, DI + MOVQ DI, (R9) + MOVQ CX, 8(R9) + + ADDQ R10, R9 + INCQ SI +test: + CMPQ SI, R8 + JNE loop + XORQ AX, AX +return: + MOVQ AX, ret+80(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, AX + JMP return + +// This is the same algorithm as dictionaryLookupFixedLenByteArrayString but we +// only store the pointer to the location holding the value instead of storing +// the pair of pointer and length. Since the length is fixed for this dictionary +// type, the application can assume it at the call site. +// +// func dictionaryLookupFixedLenByteArrayPointer(dict []byte, len int, indexes []int32, rows sparse.Array) errno +TEXT ยทdictionaryLookupFixedLenByteArrayPointer(SB), NOSPLIT, $0-88 + MOVQ dict_base+0(FP), AX + MOVQ dict_len+8(FP), BX + + MOVQ len+24(FP), CX + + MOVQ indexes_base+32(FP), DX + MOVQ indexes_len+40(FP), R8 + + MOVQ rows_array_ptr+56(FP), R9 + MOVQ rows_array_off+72(FP), R10 + + XORQ DI, DI + XORQ SI, SI +loop: + MOVL (DX)(SI*4), DI + IMULQ CX, DI + CMPL DI, BX + JAE indexOutOfBounds + + ADDQ AX, DI + MOVQ DI, (R9) + + ADDQ R10, R9 + INCQ SI +test: + CMPQ SI, R8 + JNE loop + XORQ AX, AX +return: + MOVQ AX, ret+80(FP) + RET +indexOutOfBounds: + MOVQ $errnoIndexOutOfBounds, AX + JMP return + +GLOBL ยทrange0n8(SB), RODATA|NOPTR, $40 +DATA ยทrange0n8+0(SB)/4, $0 +DATA ยทrange0n8+4(SB)/4, $1 +DATA ยทrange0n8+8(SB)/4, $2 +DATA ยทrange0n8+12(SB)/4, $3 +DATA ยทrange0n8+16(SB)/4, $4 +DATA ยทrange0n8+20(SB)/4, $5 +DATA ยทrange0n8+24(SB)/4, $6 +DATA ยทrange0n8+28(SB)/4, $7 +DATA ยทrange0n8+32(SB)/4, $8 diff --git a/vendor/github.com/parquet-go/parquet-go/dictionary_purego.go b/vendor/github.com/parquet-go/parquet-go/dictionary_purego.go new file mode 100644 index 0000000000..4893415250 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/dictionary_purego.go @@ -0,0 +1,210 @@ +//go:build purego || !amd64 + +package parquet + +import ( + "unsafe" + + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" +) + +func (d *int32Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*int32)(rows.Index(i)) = d.index(j) + } +} + +func (d *int64Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*int64)(rows.Index(i)) = d.index(j) + } +} + +func (d *floatDictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*float32)(rows.Index(i)) = d.index(j) + } +} + +func (d *doubleDictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*float64)(rows.Index(i)) = d.index(j) + } +} + +func (d *byteArrayDictionary) lookupString(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*string)(rows.Index(i)) = unsafecast.String(d.index(int(j))) + } +} + +func (d *fixedLenByteArrayDictionary) lookupString(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*string)(rows.Index(i)) = unsafecast.String(d.index(j)) + } +} + +func (d *uint32Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*uint32)(rows.Index(i)) = d.index(j) + } +} + +func (d *uint64Dictionary) lookup(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(*uint64)(rows.Index(i)) = d.index(j) + } +} + +func (d *be128Dictionary) lookupString(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + s := "0123456789ABCDEF" + for i, j := range indexes { + *(**[16]byte)(unsafe.Pointer(&s)) = d.index(j) + *(*string)(rows.Index(i)) = s + } +} + +func (d *be128Dictionary) lookupPointer(indexes []int32, rows sparse.Array) { + checkLookupIndexBounds(indexes, rows) + for i, j := range indexes { + *(**[16]byte)(rows.Index(i)) = d.index(j) + } +} + +func (d *int32Dictionary) bounds(indexes []int32) (min, max int32) { + min = d.index(indexes[0]) + max = min + + for _, i := range indexes[1:] { + value := d.index(i) + if value < min { + min = value + } + if value > max { + max = value + } + } + + return min, max +} + +func (d *int64Dictionary) bounds(indexes []int32) (min, max int64) { + min = d.index(indexes[0]) + max = min + + for _, i := range indexes[1:] { + value := d.index(i) + if value < min { + min = value + } + if value > max { + max = value + } + } + + return min, max +} + +func (d *floatDictionary) bounds(indexes []int32) (min, max float32) { + min = d.index(indexes[0]) + max = min + + for _, i := range indexes[1:] { + value := d.index(i) + if value < min { + min = value + } + if value > max { + max = value + } + } + + return min, max +} + +func (d *doubleDictionary) bounds(indexes []int32) (min, max float64) { + min = d.index(indexes[0]) + max = min + + for _, i := range indexes[1:] { + value := d.index(i) + if value < min { + min = value + } + if value > max { + max = value + } + } + + return min, max +} + +func (d *uint32Dictionary) bounds(indexes []int32) (min, max uint32) { + min = d.index(indexes[0]) + max = min + + for _, i := range indexes[1:] { + value := d.index(i) + if value < min { + min = value + } + if value > max { + max = value + } + } + + return min, max +} + +func (d *uint64Dictionary) bounds(indexes []int32) (min, max uint64) { + min = d.index(indexes[0]) + max = min + + for _, i := range indexes[1:] { + value := d.index(i) + if value < min { + min = value + } + if value > max { + max = value + } + } + + return min, max +} + +func (d *be128Dictionary) bounds(indexes []int32) (min, max *[16]byte) { + values := [64]*[16]byte{} + min = d.index(indexes[0]) + max = min + + for i := 1; i < len(indexes); i += len(values) { + n := len(indexes) - i + if n > len(values) { + n = len(values) + } + j := i + n + d.lookupPointer(indexes[i:j:j], makeArrayBE128(values[:n:n])) + + for _, value := range values[:n:n] { + switch { + case lessBE128(value, min): + min = value + case lessBE128(max, value): + max = value + } + } + } + + return min, max +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding.go b/vendor/github.com/parquet-go/parquet-go/encoding.go new file mode 100644 index 0000000000..65b04ca7a9 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding.go @@ -0,0 +1,160 @@ +package parquet + +import ( + "math/bits" + "sync" + + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/encoding/bitpacked" + "github.com/parquet-go/parquet-go/encoding/bytestreamsplit" + "github.com/parquet-go/parquet-go/encoding/delta" + "github.com/parquet-go/parquet-go/encoding/plain" + "github.com/parquet-go/parquet-go/encoding/rle" + "github.com/parquet-go/parquet-go/format" +) + +var ( + // Plain is the default parquet encoding. + Plain plain.Encoding + + // RLE is the hybrid bit-pack/run-length parquet encoding. + RLE rle.Encoding + + // BitPacked is the deprecated bit-packed encoding for repetition and + // definition levels. + BitPacked bitpacked.Encoding + + // PlainDictionary is the plain dictionary parquet encoding. + // + // This encoding should not be used anymore in parquet 2.0 and later, + // it is implemented for backwards compatibility to support reading + // files that were encoded with older parquet libraries. + PlainDictionary plain.DictionaryEncoding + + // RLEDictionary is the RLE dictionary parquet encoding. + RLEDictionary rle.DictionaryEncoding + + // DeltaBinaryPacked is the delta binary packed parquet encoding. + DeltaBinaryPacked delta.BinaryPackedEncoding + + // DeltaLengthByteArray is the delta length byte array parquet encoding. + DeltaLengthByteArray delta.LengthByteArrayEncoding + + // DeltaByteArray is the delta byte array parquet encoding. + DeltaByteArray delta.ByteArrayEncoding + + // ByteStreamSplit is an encoding for floating-point data. + ByteStreamSplit bytestreamsplit.Encoding + + // Table indexing the encodings supported by this package. + encodings = [...]encoding.Encoding{ + format.Plain: &Plain, + format.PlainDictionary: &PlainDictionary, + format.BitPacked: &BitPacked, + format.RLE: &RLE, + format.RLEDictionary: &RLEDictionary, + format.DeltaBinaryPacked: &DeltaBinaryPacked, + format.DeltaLengthByteArray: &DeltaLengthByteArray, + format.DeltaByteArray: &DeltaByteArray, + format.ByteStreamSplit: &ByteStreamSplit, + } + + // Table indexing RLE encodings for repetition and definition levels of + // all supported bit widths. + levelEncodingsRLE = [...]rle.Encoding{ + 0: {BitWidth: 1}, + 1: {BitWidth: 2}, + 2: {BitWidth: 3}, + 3: {BitWidth: 4}, + 4: {BitWidth: 5}, + 5: {BitWidth: 6}, + 6: {BitWidth: 7}, + 7: {BitWidth: 8}, + } + + levelEncodingsBitPacked = [...]bitpacked.Encoding{ + 0: {BitWidth: 1}, + 1: {BitWidth: 2}, + 2: {BitWidth: 3}, + 3: {BitWidth: 4}, + 4: {BitWidth: 5}, + 5: {BitWidth: 6}, + 6: {BitWidth: 7}, + 7: {BitWidth: 8}, + } +) + +var extraEncodings sync.Map + +func isDictionaryEncoding(encoding encoding.Encoding) bool { + return isDictionaryFormat(encoding.Encoding()) +} + +func isDictionaryFormat(encoding format.Encoding) bool { + return encoding == format.PlainDictionary || encoding == format.RLEDictionary +} + +func RegisterEncoding(enc encoding.Encoding) { + ns := encoding.NotSupported{} + if enc == ns { + panic("cannot register parquet encoding as not-supported") + } + if LookupEncoding(enc.Encoding()) != ns { + panic("cannot register parquet encoding that overrides the standard specification") + } + extraEncodings.Store(enc.Encoding(), enc) +} + +// LookupEncoding returns the parquet encoding associated with the given code. +// +// The function never returns nil. If the encoding is not supported, +// encoding.NotSupported is returned. +func LookupEncoding(enc format.Encoding) encoding.Encoding { + if enc >= 0 && int(enc) < len(encodings) { + if e := encodings[enc]; e != nil { + return e + } + } + if enc, ok := extraEncodings.Load(enc); ok { + return enc.(encoding.Encoding) + } + return encoding.NotSupported{} +} + +func lookupLevelEncoding(enc format.Encoding, max byte) encoding.Encoding { + i := bits.Len8(max) - 1 + switch enc { + case format.RLE: + return &levelEncodingsRLE[i] + case format.BitPacked: + return &levelEncodingsBitPacked[i] + default: + return encoding.NotSupported{} + } +} + +func canEncode(e encoding.Encoding, k Kind) bool { + if isDictionaryEncoding(e) { + return true + } + switch k { + case Boolean: + return encoding.CanEncodeBoolean(e) + case Int32: + return encoding.CanEncodeInt32(e) + case Int64: + return encoding.CanEncodeInt64(e) + case Int96: + return encoding.CanEncodeInt96(e) + case Float: + return encoding.CanEncodeFloat(e) + case Double: + return encoding.CanEncodeDouble(e) + case ByteArray: + return encoding.CanEncodeByteArray(e) + case FixedLenByteArray: + return encoding.CanEncodeFixedLenByteArray(e) + default: + return false + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/bitpacked/bitpacked.go b/vendor/github.com/parquet-go/parquet-go/encoding/bitpacked/bitpacked.go new file mode 100644 index 0000000000..47c3dd0906 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/bitpacked/bitpacked.go @@ -0,0 +1,119 @@ +package bitpacked + +import ( + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" +) + +type Encoding struct { + encoding.NotSupported + BitWidth int +} + +func (e *Encoding) String() string { + return "BIT_PACKED" +} + +func (e *Encoding) Encoding() format.Encoding { + return format.BitPacked +} + +func (e *Encoding) EncodeLevels(dst []byte, src []uint8) ([]byte, error) { + dst, err := encodeLevels(dst[:0], src, uint(e.BitWidth)) + return dst, e.wrap(err) +} + +func (e *Encoding) DecodeLevels(dst []uint8, src []byte) ([]uint8, error) { + dst, err := decodeLevels(dst[:0], src, uint(e.BitWidth)) + return dst, e.wrap(err) +} + +func (e *Encoding) wrap(err error) error { + if err != nil { + err = encoding.Error(e, err) + } + return err +} + +func encodeLevels(dst, src []byte, bitWidth uint) ([]byte, error) { + if bitWidth == 0 || len(src) == 0 { + return append(dst[:0], 0), nil + } + + n := ((int(bitWidth) * len(src)) + 7) / 8 + c := n + 1 + + if cap(dst) < c { + dst = make([]byte, c, 2*c) + } else { + dst = dst[:c] + for i := range dst { + dst[i] = 0 + } + } + + bitMask := byte(1<> bitShift + i := bitOffset / 8 + j := bitOffset % 8 + dst[i+0] |= (v & bitMask) << j + dst[i+1] |= (v >> (8 - j)) + bitOffset += bitWidth + } + + return dst[:n], nil +} + +func decodeLevels(dst, src []byte, bitWidth uint) ([]byte, error) { + if bitWidth == 0 || len(src) == 0 { + return append(dst[:0], 0), nil + } + + numBits := 8 * uint(len(src)) + numValues := int(numBits / bitWidth) + if (numBits % bitWidth) != 0 { + numValues++ + } + + if cap(dst) < numValues { + dst = make([]byte, numValues, 2*numValues) + } else { + dst = dst[:numValues] + for i := range dst { + dst[i] = 0 + } + } + + bitMask := byte(1<> j) + if int(i+1) < len(src) { + v |= (src[i+1] << (8 - j)) + } + v &= bitMask + dst[k] = bitFlip(v) >> bitShift + bitOffset += bitWidth + } + + return dst, nil +} + +func bitFlip(b byte) byte { + return (((b >> 0) & 1) << 7) | + (((b >> 1) & 1) << 6) | + (((b >> 2) & 1) << 5) | + (((b >> 3) & 1) << 4) | + (((b >> 4) & 1) << 3) | + (((b >> 5) & 1) << 2) | + (((b >> 6) & 1) << 1) | + (((b >> 7) & 1) << 0) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit.go b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit.go new file mode 100644 index 0000000000..23b0202d7b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit.go @@ -0,0 +1,60 @@ +package bytestreamsplit + +import ( + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// This encoder implements a version of the Byte Stream Split encoding as described +// in https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 +type Encoding struct { + encoding.NotSupported +} + +func (e *Encoding) String() string { + return "BYTE_STREAM_SPLIT" +} + +func (e *Encoding) Encoding() format.Encoding { + return format.ByteStreamSplit +} + +func (e *Encoding) EncodeFloat(dst []byte, src []float32) ([]byte, error) { + dst = resize(dst, 4*len(src)) + encodeFloat(dst, unsafecast.Slice[byte](src)) + return dst, nil +} + +func (e *Encoding) EncodeDouble(dst []byte, src []float64) ([]byte, error) { + dst = resize(dst, 8*len(src)) + encodeDouble(dst, unsafecast.Slice[byte](src)) + return dst, nil +} + +func (e *Encoding) DecodeFloat(dst []float32, src []byte) ([]float32, error) { + if (len(src) % 4) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "FLOAT", len(src)) + } + buf := resize(unsafecast.Slice[byte](dst), len(src)) + decodeFloat(buf, src) + return unsafecast.Slice[float32](buf), nil +} + +func (e *Encoding) DecodeDouble(dst []float64, src []byte) ([]float64, error) { + if (len(src) % 8) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "DOUBLE", len(src)) + } + buf := resize(unsafecast.Slice[byte](dst), len(src)) + decodeDouble(buf, src) + return unsafecast.Slice[float64](buf), nil +} + +func resize(buf []byte, size int) []byte { + if cap(buf) < size { + buf = make([]byte, size, 2*size) + } else { + buf = buf[:size] + } + return buf +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.go b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.go new file mode 100644 index 0000000000..1798c8916c --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.go @@ -0,0 +1,35 @@ +//go:build !purego + +package bytestreamsplit + +import ( + "golang.org/x/sys/cpu" +) + +var encodeFloatHasAVX512 = cpu.X86.HasAVX512 && + cpu.X86.HasAVX512F && + cpu.X86.HasAVX512VL + +var encodeDoubleHasAVX512 = cpu.X86.HasAVX512 && + cpu.X86.HasAVX512F && + cpu.X86.HasAVX512VL && + cpu.X86.HasAVX512VBMI // VPERMB + +var decodeFloatHasAVX2 = cpu.X86.HasAVX2 + +var decodeDoubleHasAVX512 = cpu.X86.HasAVX512 && + cpu.X86.HasAVX512F && + cpu.X86.HasAVX512VL && + cpu.X86.HasAVX512VBMI // VPERMB + +//go:noescape +func encodeFloat(dst, src []byte) + +//go:noescape +func encodeDouble(dst, src []byte) + +//go:noescape +func decodeFloat(dst, src []byte) + +//go:noescape +func decodeDouble(dst, src []byte) diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.s b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.s new file mode 100644 index 0000000000..b0c7622dbc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_amd64.s @@ -0,0 +1,426 @@ + //go:build !purego + +#include "textflag.h" + +// This file contains optimizations of the BYTE_STREAM_SPLIT encoding using AVX2 +// and AVX512 (when available). +// +// The AVX2/512 instruction set comes with instructions to load memory from, or +// store memory at sparse locations called VPGATHER and VPSCATTER. VPGATHER was +// available in the AVX2 instruction set, VPSCATTER was introduced in AVX512 +// (when the AVX512_VBMI extension is supported). Gathering bytes are sparse +// memory locations is useful during the decoding process since we are +// recomposing 32 or 64 bit floating point values from 4 or 8 bytes dispatched +// in the input byte array. +// +// To either deconstruct or reconstruct floating point values, we need to +// reorder the bytes of each value. If we have 4 32 bit floats, we can permute +// their bytes so that the first one contains all the first bytes, the second +// contains all the second bytes, etc... The VPSHUFB instruction is used to +// perform the byte permutation, or the VPERMB instruction for 64 bit floats. +// +// We use different instructions because the VPSHUFB instruction works on two +// lanes of 16 bytes when used on YMM registers. 4 32 bit floats take 16 bytes, +// so a a YMM register can hold two lanes of 4 32 bit floats and the VPSHUFB +// can permute the two sets of values in a single invocation. For 64 bit floats +// we need to permute 8 values, which take 64 bytes and therefore need to be +// held in a ZMM register and apply permutations across the entire register, +// which is only possible using VPERMB. +// +// Technically we could use ZMM registers when working on 32 bit floats to work +// on 16 values per iteration. However, measurements indicated that the latency +// of VPGATHERDD/VPSCATTERDD on ZMM registers did not provide any improvements +// to the throughput of the algorithms, but working on more values increased the +// code complexity. Using YMM registers offered the best balance between +// performance and maintainability. +// +// At a high level the vectorized algorithms are the following: +// +// encoding +// -------- +// * Load a vector of data from the input buffer +// * Permute bytes, grouping bytes by index +// * Scatter bytes of the register to the output buffer +// +// decoding +// -------- +// * Gather sparse bytes from the input buffer +// * Permute bytes, reconstructing the original values +// * Store the vector in the output buffer +// +// When AVX instructions are not available, the functions fallback to scalar +// implementations of the algorithms. These yield much lower throughput, but +// performed 20-30% better than the code generated by the Go compiler. + +// func encodeFloat(dst, src []byte) +TEXT ยทencodeFloat(SB), NOSPLIT, $0-48 + MOVQ src_base+24(FP), AX + MOVQ src_len+32(FP), BX + MOVQ dst_base+0(FP), DX + + MOVQ AX, CX + ADDQ BX, CX // end + SHRQ $2, BX // len + + CMPQ BX, $0 + JE done + + CMPB ยทencodeFloatHasAVX512(SB), $0 + JE loop1x4 + + CMPQ BX, $8 + JB loop1x4 + + MOVQ CX, DI + SUBQ AX, DI + SHRQ $5, DI + SHLQ $5, DI + ADDQ AX, DI + + VMOVDQU32 shuffle8x4<>(SB), Y0 + VPBROADCASTD BX, Y2 + VPMULLD scale8x4<>(SB), Y2, Y2 + VPADDD offset8x4<>(SB), Y2, Y2 +loop8x4: + KXORQ K1, K1, K1 + KNOTQ K1, K1 + + VMOVDQU32 (AX), Y1 + VPSHUFB Y0, Y1, Y1 + VPSCATTERDD Y1, K1, (DX)(Y2*1) + + ADDQ $32, AX + ADDQ $8, DX + CMPQ AX, DI + JNE loop8x4 + VZEROUPPER + + CMPQ AX, CX + JE done +loop1x4: + MOVL (AX), SI + MOVQ DX, DI + + MOVB SI, (DI) + SHRL $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRL $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRL $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + + ADDQ $4, AX + INCQ DX + CMPQ AX, CX + JB loop1x4 +done: + RET + +// func encodeDouble(dst, src []byte) +TEXT ยทencodeDouble(SB), NOSPLIT, $0-48 + MOVQ src_base+24(FP), AX + MOVQ src_len+32(FP), BX + MOVQ dst_base+0(FP), DX + + MOVQ AX, CX + ADDQ BX, CX + SHRQ $3, BX + + CMPQ BX, $0 + JE done + + CMPB ยทencodeDoubleHasAVX512(SB), $0 + JE loop1x8 + + CMPQ BX, $8 + JB loop1x8 + + MOVQ CX, DI + SUBQ AX, DI + SHRQ $6, DI + SHLQ $6, DI + ADDQ AX, DI + + VMOVDQU64 shuffle8x8<>(SB), Z0 + VPBROADCASTQ BX, Z2 + VPMULLQ scale8x8<>(SB), Z2, Z2 +loop8x8: + KXORQ K1, K1, K1 + KNOTQ K1, K1 + + VMOVDQU64 (AX), Z1 + VPERMB Z1, Z0, Z1 + VPSCATTERQQ Z1, K1, (DX)(Z2*1) + + ADDQ $64, AX + ADDQ $8, DX + CMPQ AX, DI + JNE loop8x8 + VZEROUPPER + + CMPQ AX, CX + JE done +loop1x8: + MOVQ (AX), SI + MOVQ DX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + SHRQ $8, SI + ADDQ BX, DI + + MOVB SI, (DI) + + ADDQ $8, AX + INCQ DX + CMPQ AX, CX + JB loop1x8 +done: + RET + +// func decodeFloat(dst, src []byte) +TEXT ยทdecodeFloat(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVQ src_base+24(FP), DX + + MOVQ AX, CX + ADDQ BX, CX // end + SHRQ $2, BX // len + + CMPQ BX, $0 + JE done + + CMPB ยทdecodeFloatHasAVX2(SB), $0 + JE loop1x4 + + CMPQ BX, $8 + JB loop1x4 + + MOVQ CX, DI + SUBQ AX, DI + SHRQ $5, DI + SHLQ $5, DI + ADDQ AX, DI + + MOVQ $0xFFFFFFFF, SI + MOVQ BX, X5 + MOVQ SI, X6 + VMOVDQU shuffle8x4<>(SB), Y0 + VPBROADCASTD X5, Y2 + VPBROADCASTD X6, Y3 + VPMULLD scale8x4<>(SB), Y2, Y2 + VPADDD offset8x4<>(SB), Y2, Y2 + VMOVDQU Y3, Y4 +loop8x4: + VPGATHERDD Y4, (DX)(Y2*1), Y1 + VPSHUFB Y0, Y1, Y1 + VMOVDQU Y1, (AX) + VMOVDQU Y3, Y4 + + ADDQ $32, AX + ADDQ $8, DX + CMPQ AX, DI + JNE loop8x4 + VZEROUPPER + + CMPQ AX, CX + JE done +loop1x4: + MOVQ DX, DI + MOVBLZX (DI), R8 + ADDQ BX, DI + MOVBLZX (DI), R9 + ADDQ BX, DI + MOVBLZX (DI), R10 + ADDQ BX, DI + MOVBLZX (DI), R11 + + SHLL $8, R9 + SHLL $16, R10 + SHLL $24, R11 + + ORL R9, R8 + ORL R10, R8 + ORL R11, R8 + + MOVL R8, (AX) + + ADDQ $4, AX + INCQ DX + CMPQ AX, CX + JB loop1x4 +done: + RET + +// func decodeDouble(dst, src []byte) +TEXT ยทdecodeDouble(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVQ src_base+24(FP), DX + + MOVQ AX, CX + ADDQ BX, CX + SHRQ $3, BX + + CMPQ BX, $0 + JE done + + CMPB ยทdecodeDoubleHasAVX512(SB), $0 + JE loop1x8 + + CMPQ BX, $8 + JB loop1x8 + + MOVQ CX, DI + SUBQ AX, DI + SHRQ $6, DI + SHLQ $6, DI + ADDQ AX, DI + + VMOVDQU64 shuffle8x8<>(SB), Z0 + VPBROADCASTQ BX, Z2 + VPMULLQ scale8x8<>(SB), Z2, Z2 +loop8x8: + KXORQ K1, K1, K1 + KNOTQ K1, K1 + + VPGATHERQQ (DX)(Z2*1), K1, Z1 + VPERMB Z1, Z0, Z1 + VMOVDQU64 Z1, (AX) + + ADDQ $64, AX + ADDQ $8, DX + CMPQ AX, DI + JNE loop8x8 + VZEROUPPER + + CMPQ AX, CX + JE done +loop1x8: + MOVQ DX, DI + XORQ R12, R12 + + MOVBQZX (DI), R8 + ADDQ BX, DI + MOVBQZX (DI), R9 + ADDQ BX, DI + MOVBQZX (DI), R10 + ADDQ BX, DI + MOVBQZX (DI), R11 + ADDQ BX, DI + + SHLQ $8, R9 + SHLQ $16, R10 + SHLQ $24, R11 + + ORQ R8, R12 + ORQ R9, R12 + ORQ R10, R12 + ORQ R11, R12 + + MOVBQZX (DI), R8 + ADDQ BX, DI + MOVBQZX (DI), R9 + ADDQ BX, DI + MOVBQZX (DI), R10 + ADDQ BX, DI + MOVBQZX (DI), R11 + + SHLQ $32, R8 + SHLQ $40, R9 + SHLQ $48, R10 + SHLQ $56, R11 + + ORQ R8, R12 + ORQ R9, R12 + ORQ R10, R12 + ORQ R11, R12 + + MOVQ R12, (AX) + + ADDQ $8, AX + INCQ DX + CMPQ AX, CX + JB loop1x8 +done: + RET + +GLOBL scale8x4<>(SB), RODATA|NOPTR, $32 +DATA scale8x4<>+0(SB)/4, $0 +DATA scale8x4<>+4(SB)/4, $1 +DATA scale8x4<>+8(SB)/4, $2 +DATA scale8x4<>+12(SB)/4, $3 +DATA scale8x4<>+16(SB)/4, $0 +DATA scale8x4<>+20(SB)/4, $1 +DATA scale8x4<>+24(SB)/4, $2 +DATA scale8x4<>+28(SB)/4, $3 + +GLOBL offset8x4<>(SB), RODATA|NOPTR, $32 +DATA offset8x4<>+0(SB)/4, $0 +DATA offset8x4<>+4(SB)/4, $0 +DATA offset8x4<>+8(SB)/4, $0 +DATA offset8x4<>+12(SB)/4, $0 +DATA offset8x4<>+16(SB)/4, $4 +DATA offset8x4<>+20(SB)/4, $4 +DATA offset8x4<>+24(SB)/4, $4 +DATA offset8x4<>+28(SB)/4, $4 + +GLOBL shuffle8x4<>(SB), RODATA|NOPTR, $32 +DATA shuffle8x4<>+0(SB)/4, $0x0C080400 +DATA shuffle8x4<>+4(SB)/4, $0x0D090501 +DATA shuffle8x4<>+8(SB)/4, $0x0E0A0602 +DATA shuffle8x4<>+12(SB)/4, $0x0F0B0703 +DATA shuffle8x4<>+16(SB)/4, $0x0C080400 +DATA shuffle8x4<>+20(SB)/4, $0x0D090501 +DATA shuffle8x4<>+24(SB)/4, $0x0E0A0602 +DATA shuffle8x4<>+28(SB)/4, $0x0F0B0703 + +GLOBL scale8x8<>(SB), RODATA|NOPTR, $64 +DATA scale8x8<>+0(SB)/8, $0 +DATA scale8x8<>+8(SB)/8, $1 +DATA scale8x8<>+16(SB)/8, $2 +DATA scale8x8<>+24(SB)/8, $3 +DATA scale8x8<>+32(SB)/8, $4 +DATA scale8x8<>+40(SB)/8, $5 +DATA scale8x8<>+48(SB)/8, $6 +DATA scale8x8<>+56(SB)/8, $7 + +GLOBL shuffle8x8<>(SB), RODATA|NOPTR, $64 +DATA shuffle8x8<>+0(SB)/8, $0x3830282018100800 +DATA shuffle8x8<>+8(SB)/8, $0x3931292119110901 +DATA shuffle8x8<>+16(SB)/8, $0x3A322A221A120A02 +DATA shuffle8x8<>+24(SB)/8, $0x3B332B231B130B03 +DATA shuffle8x8<>+32(SB)/8, $0x3C342C241C140C04 +DATA shuffle8x8<>+40(SB)/8, $0x3D352D251D150D05 +DATA shuffle8x8<>+48(SB)/8, $0x3E362E261E160E06 +DATA shuffle8x8<>+56(SB)/8, $0x3F372F271F170F07 diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_purego.go b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_purego.go new file mode 100644 index 0000000000..6f5bf15c79 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/bytestreamsplit/bytestreamsplit_purego.go @@ -0,0 +1,83 @@ +//go:build purego || !amd64 + +package bytestreamsplit + +import "github.com/parquet-go/parquet-go/internal/unsafecast" + +func encodeFloat(dst, src []byte) { + n := len(src) / 4 + b0 := dst[0*n : 1*n] + b1 := dst[1*n : 2*n] + b2 := dst[2*n : 3*n] + b3 := dst[3*n : 4*n] + + for i, v := range unsafecast.Slice[uint32](src) { + b0[i] = byte(v >> 0) + b1[i] = byte(v >> 8) + b2[i] = byte(v >> 16) + b3[i] = byte(v >> 24) + } +} + +func encodeDouble(dst, src []byte) { + n := len(src) / 8 + b0 := dst[0*n : 1*n] + b1 := dst[1*n : 2*n] + b2 := dst[2*n : 3*n] + b3 := dst[3*n : 4*n] + b4 := dst[4*n : 5*n] + b5 := dst[5*n : 6*n] + b6 := dst[6*n : 7*n] + b7 := dst[7*n : 8*n] + + for i, v := range unsafecast.Slice[uint64](src) { + b0[i] = byte(v >> 0) + b1[i] = byte(v >> 8) + b2[i] = byte(v >> 16) + b3[i] = byte(v >> 24) + b4[i] = byte(v >> 32) + b5[i] = byte(v >> 40) + b6[i] = byte(v >> 48) + b7[i] = byte(v >> 56) + } +} + +func decodeFloat(dst, src []byte) { + n := len(src) / 4 + b0 := src[0*n : 1*n] + b1 := src[1*n : 2*n] + b2 := src[2*n : 3*n] + b3 := src[3*n : 4*n] + + dst32 := unsafecast.Slice[uint32](dst) + for i := range dst32 { + dst32[i] = uint32(b0[i]) | + uint32(b1[i])<<8 | + uint32(b2[i])<<16 | + uint32(b3[i])<<24 + } +} + +func decodeDouble(dst, src []byte) { + n := len(src) / 8 + b0 := src[0*n : 1*n] + b1 := src[1*n : 2*n] + b2 := src[2*n : 3*n] + b3 := src[3*n : 4*n] + b4 := src[4*n : 5*n] + b5 := src[5*n : 6*n] + b6 := src[6*n : 7*n] + b7 := src[7*n : 8*n] + + dst64 := unsafecast.Slice[uint64](dst) + for i := range dst64 { + dst64[i] = uint64(b0[i]) | + uint64(b1[i])<<8 | + uint64(b2[i])<<16 | + uint64(b3[i])<<24 | + uint64(b4[i])<<32 | + uint64(b5[i])<<40 | + uint64(b6[i])<<48 | + uint64(b7[i])<<56 + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed.go new file mode 100644 index 0000000000..cf9d4cfc9e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed.go @@ -0,0 +1,488 @@ +package delta + +import ( + "encoding/binary" + "fmt" + "io" + "math" + "math/bits" + + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/bitpack" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type BinaryPackedEncoding struct { + encoding.NotSupported +} + +func (e *BinaryPackedEncoding) String() string { + return "DELTA_BINARY_PACKED" +} + +func (e *BinaryPackedEncoding) Encoding() format.Encoding { + return format.DeltaBinaryPacked +} + +func (e *BinaryPackedEncoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + return encodeInt32(dst[:0], src), nil +} + +func (e *BinaryPackedEncoding) EncodeInt64(dst []byte, src []int64) ([]byte, error) { + return encodeInt64(dst[:0], src), nil +} + +func (e *BinaryPackedEncoding) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + buf := unsafecast.Slice[byte](dst) + buf, _, err := decodeInt32(buf[:0], src) + return unsafecast.Slice[int32](buf), e.wrap(err) +} + +func (e *BinaryPackedEncoding) DecodeInt64(dst []int64, src []byte) ([]int64, error) { + buf := unsafecast.Slice[byte](dst) + buf, _, err := decodeInt64(buf[:0], src) + return unsafecast.Slice[int64](buf), e.wrap(err) +} + +func (e *BinaryPackedEncoding) wrap(err error) error { + if err != nil { + err = encoding.Error(e, err) + } + return err +} + +const ( + blockSize = 128 + numMiniBlocks = 4 + miniBlockSize = blockSize / numMiniBlocks + // The parquet spec does not enforce a limit to the block size, but we need + // one otherwise invalid inputs may result in unbounded memory allocations. + // + // 65K+ values should be enough for any valid use case. + maxSupportedBlockSize = 65536 + + maxHeaderLength32 = 4 * binary.MaxVarintLen64 + maxMiniBlockLength32 = binary.MaxVarintLen64 + numMiniBlocks + (4 * blockSize) + + maxHeaderLength64 = 8 * binary.MaxVarintLen64 + maxMiniBlockLength64 = binary.MaxVarintLen64 + numMiniBlocks + (8 * blockSize) +) + +var ( + encodeInt32 = encodeInt32Default + encodeInt64 = encodeInt64Default +) + +func encodeInt32Default(dst []byte, src []int32) []byte { + totalValues := len(src) + firstValue := int32(0) + if totalValues > 0 { + firstValue = src[0] + } + + n := len(dst) + dst = resize(dst, n+maxHeaderLength32) + dst = dst[:n+encodeBinaryPackedHeader(dst[n:], blockSize, numMiniBlocks, totalValues, int64(firstValue))] + + if totalValues < 2 { + return dst + } + + lastValue := firstValue + for i := 1; i < len(src); i += blockSize { + block := [blockSize]int32{} + blockLength := copy(block[:], src[i:]) + + lastValue = blockDeltaInt32(&block, lastValue) + minDelta := blockMinInt32(&block) + blockSubInt32(&block, minDelta) + blockClearInt32(&block, blockLength) + + bitWidths := [numMiniBlocks]byte{} + blockBitWidthsInt32(&bitWidths, &block) + + n := len(dst) + dst = resize(dst, n+maxMiniBlockLength32+4) + n += encodeBlockHeader(dst[n:], int64(minDelta), bitWidths) + + for i, bitWidth := range bitWidths { + if bitWidth != 0 { + miniBlock := (*[miniBlockSize]int32)(block[i*miniBlockSize:]) + encodeMiniBlockInt32(dst[n:], miniBlock, uint(bitWidth)) + n += (miniBlockSize * int(bitWidth)) / 8 + } + } + + dst = dst[:n] + } + + return dst +} + +func encodeInt64Default(dst []byte, src []int64) []byte { + totalValues := len(src) + firstValue := int64(0) + if totalValues > 0 { + firstValue = src[0] + } + + n := len(dst) + dst = resize(dst, n+maxHeaderLength64) + dst = dst[:n+encodeBinaryPackedHeader(dst[n:], blockSize, numMiniBlocks, totalValues, firstValue)] + + if totalValues < 2 { + return dst + } + + lastValue := firstValue + for i := 1; i < len(src); i += blockSize { + block := [blockSize]int64{} + blockLength := copy(block[:], src[i:]) + + lastValue = blockDeltaInt64(&block, lastValue) + minDelta := blockMinInt64(&block) + blockSubInt64(&block, minDelta) + blockClearInt64(&block, blockLength) + + bitWidths := [numMiniBlocks]byte{} + blockBitWidthsInt64(&bitWidths, &block) + + n := len(dst) + dst = resize(dst, n+maxMiniBlockLength64+8) + n += encodeBlockHeader(dst[n:], minDelta, bitWidths) + + for i, bitWidth := range bitWidths { + if bitWidth != 0 { + miniBlock := (*[miniBlockSize]int64)(block[i*miniBlockSize:]) + encodeMiniBlockInt64(dst[n:], miniBlock, uint(bitWidth)) + n += (miniBlockSize * int(bitWidth)) / 8 + } + } + + dst = dst[:n] + } + + return dst +} + +func encodeBinaryPackedHeader(dst []byte, blockSize, numMiniBlocks, totalValues int, firstValue int64) (n int) { + n += binary.PutUvarint(dst[n:], uint64(blockSize)) + n += binary.PutUvarint(dst[n:], uint64(numMiniBlocks)) + n += binary.PutUvarint(dst[n:], uint64(totalValues)) + n += binary.PutVarint(dst[n:], firstValue) + return n +} + +func encodeBlockHeader(dst []byte, minDelta int64, bitWidths [numMiniBlocks]byte) (n int) { + n += binary.PutVarint(dst, int64(minDelta)) + n += copy(dst[n:], bitWidths[:]) + return n +} + +func blockClearInt32(block *[blockSize]int32, blockLength int) { + if blockLength < blockSize { + clear := block[blockLength:] + for i := range clear { + clear[i] = 0 + } + } +} + +func blockDeltaInt32(block *[blockSize]int32, lastValue int32) int32 { + for i, v := range block { + block[i], lastValue = v-lastValue, v + } + return lastValue +} + +func blockMinInt32(block *[blockSize]int32) int32 { + min := block[0] + for _, v := range block[1:] { + if v < min { + min = v + } + } + return min +} + +func blockSubInt32(block *[blockSize]int32, value int32) { + for i := range block { + block[i] -= value + } +} + +func blockBitWidthsInt32(bitWidths *[numMiniBlocks]byte, block *[blockSize]int32) { + for i := range bitWidths { + j := (i + 0) * miniBlockSize + k := (i + 1) * miniBlockSize + bitWidth := 0 + + for _, v := range block[j:k] { + if n := bits.Len32(uint32(v)); n > bitWidth { + bitWidth = n + } + } + + bitWidths[i] = byte(bitWidth) + } +} + +func blockClearInt64(block *[blockSize]int64, blockLength int) { + if blockLength < blockSize { + clear := block[blockLength:] + for i := range clear { + clear[i] = 0 + } + } +} + +func blockDeltaInt64(block *[blockSize]int64, lastValue int64) int64 { + for i, v := range block { + block[i], lastValue = v-lastValue, v + } + return lastValue +} + +func blockMinInt64(block *[blockSize]int64) int64 { + min := block[0] + for _, v := range block[1:] { + if v < min { + min = v + } + } + return min +} + +func blockSubInt64(block *[blockSize]int64, value int64) { + for i := range block { + block[i] -= value + } +} + +func blockBitWidthsInt64(bitWidths *[numMiniBlocks]byte, block *[blockSize]int64) { + for i := range bitWidths { + j := (i + 0) * miniBlockSize + k := (i + 1) * miniBlockSize + bitWidth := 0 + + for _, v := range block[j:k] { + if n := bits.Len64(uint64(v)); n > bitWidth { + bitWidth = n + } + } + + bitWidths[i] = byte(bitWidth) + } +} + +func decodeInt32(dst, src []byte) ([]byte, []byte, error) { + blockSize, numMiniBlocks, totalValues, firstValue, src, err := decodeBinaryPackedHeader(src) + if err != nil { + return dst, src, err + } + if totalValues == 0 { + return dst, src, nil + } + if firstValue < math.MinInt32 || firstValue > math.MaxInt32 { + return dst, src, fmt.Errorf("first value out of range: %d", firstValue) + } + + writeOffset := len(dst) + dst = resize(dst, len(dst)+4*totalValues) + out := unsafecast.Slice[int32](dst) + out[writeOffset] = int32(firstValue) + writeOffset++ + totalValues-- + lastValue := int32(firstValue) + numValuesInMiniBlock := blockSize / numMiniBlocks + + const padding = 16 + miniBlockTemp := make([]byte, 256+padding) + + for totalValues > 0 && len(src) > 0 { + var minDelta int64 + var bitWidths []byte + minDelta, bitWidths, src, err = decodeBinaryPackedBlock(src, numMiniBlocks) + if err != nil { + return dst, src, err + } + + blockOffset := writeOffset + + for _, bitWidth := range bitWidths { + n := min(numValuesInMiniBlock, totalValues) + if bitWidth != 0 { + miniBlockSize := (numValuesInMiniBlock * int(bitWidth)) / 8 + miniBlockData := src + if miniBlockSize <= len(src) { + miniBlockData = miniBlockData[:miniBlockSize] + } + src = src[len(miniBlockData):] + if cap(miniBlockData) < miniBlockSize+bitpack.PaddingInt32 { + miniBlockTemp = resize(miniBlockTemp[:0], miniBlockSize+bitpack.PaddingInt32) + miniBlockData = miniBlockTemp[:copy(miniBlockTemp, miniBlockData)] + } + miniBlockData = miniBlockData[:miniBlockSize] + bitpack.UnpackInt32(out[writeOffset:writeOffset+n], miniBlockData, uint(bitWidth)) + } + writeOffset += n + totalValues -= n + if totalValues == 0 { + break + } + } + + lastValue = decodeBlockInt32(out[blockOffset:writeOffset], int32(minDelta), lastValue) + } + + if totalValues > 0 { + return dst, src, fmt.Errorf("%d missing values: %w", totalValues, io.ErrUnexpectedEOF) + } + + return dst, src, nil +} + +func decodeInt64(dst, src []byte) ([]byte, []byte, error) { + blockSize, numMiniBlocks, totalValues, firstValue, src, err := decodeBinaryPackedHeader(src) + if err != nil { + return dst, src, err + } + if totalValues == 0 { + return dst, src, nil + } + + writeOffset := len(dst) + dst = resize(dst, len(dst)+8*totalValues) + out := unsafecast.Slice[int64](dst) + out[writeOffset] = firstValue + writeOffset++ + totalValues-- + lastValue := firstValue + numValuesInMiniBlock := blockSize / numMiniBlocks + + const padding = 16 + miniBlockTemp := make([]byte, 512+padding) + + for totalValues > 0 && len(src) > 0 { + var minDelta int64 + var bitWidths []byte + minDelta, bitWidths, src, err = decodeBinaryPackedBlock(src, numMiniBlocks) + if err != nil { + return dst, src, err + } + blockOffset := writeOffset + + for _, bitWidth := range bitWidths { + n := min(numValuesInMiniBlock, totalValues) + if bitWidth != 0 { + miniBlockSize := (numValuesInMiniBlock * int(bitWidth)) / 8 + miniBlockData := src + if miniBlockSize <= len(src) { + miniBlockData = src[:miniBlockSize] + } + src = src[len(miniBlockData):] + if len(miniBlockData) < miniBlockSize+bitpack.PaddingInt64 { + miniBlockTemp = resize(miniBlockTemp[:0], miniBlockSize+bitpack.PaddingInt64) + miniBlockData = miniBlockTemp[:copy(miniBlockTemp, miniBlockData)] + } + miniBlockData = miniBlockData[:miniBlockSize] + bitpack.UnpackInt64(out[writeOffset:writeOffset+n], miniBlockData, uint(bitWidth)) + } + writeOffset += n + totalValues -= n + if totalValues == 0 { + break + } + } + + lastValue = decodeBlockInt64(out[blockOffset:writeOffset], minDelta, lastValue) + } + + if totalValues > 0 { + return dst, src, fmt.Errorf("%d missing values: %w", totalValues, io.ErrUnexpectedEOF) + } + + return dst, src, nil +} + +func decodeBinaryPackedHeader(src []byte) (blockSize, numMiniBlocks, totalValues int, firstValue int64, next []byte, err error) { + u := uint64(0) + n := 0 + i := 0 + + if u, n, err = decodeUvarint(src[i:], "block size"); err != nil { + return + } + i += n + blockSize = int(u) + + if u, n, err = decodeUvarint(src[i:], "number of mini-blocks"); err != nil { + return + } + i += n + numMiniBlocks = int(u) + + if u, n, err = decodeUvarint(src[i:], "total values"); err != nil { + return + } + i += n + totalValues = int(u) + + if firstValue, n, err = decodeVarint(src[i:], "first value"); err != nil { + return + } + i += n + + if numMiniBlocks == 0 { + err = fmt.Errorf("invalid number of mini block (%d)", numMiniBlocks) + } else if (blockSize <= 0) || (blockSize%128) != 0 { + err = fmt.Errorf("invalid block size is not a multiple of 128 (%d)", blockSize) + } else if blockSize > maxSupportedBlockSize { + err = fmt.Errorf("invalid block size is too large (%d)", blockSize) + } else if miniBlockSize := blockSize / numMiniBlocks; (numMiniBlocks <= 0) || (miniBlockSize%32) != 0 { + err = fmt.Errorf("invalid mini block size is not a multiple of 32 (%d)", miniBlockSize) + } else if totalValues < 0 { + err = fmt.Errorf("invalid total number of values is negative (%d)", totalValues) + } else if totalValues > math.MaxInt32 { + err = fmt.Errorf("too many values: %d", totalValues) + } + + return blockSize, numMiniBlocks, totalValues, firstValue, src[i:], err +} + +func decodeBinaryPackedBlock(src []byte, numMiniBlocks int) (minDelta int64, bitWidths, next []byte, err error) { + minDelta, n, err := decodeVarint(src, "min delta") + if err != nil { + return 0, nil, src, err + } + src = src[n:] + if len(src) < numMiniBlocks { + bitWidths, next = src, nil + } else { + bitWidths, next = src[:numMiniBlocks], src[numMiniBlocks:] + } + return minDelta, bitWidths, next, nil +} + +func decodeUvarint(buf []byte, what string) (u uint64, n int, err error) { + u, n = binary.Uvarint(buf) + if n == 0 { + return 0, 0, fmt.Errorf("decoding %s: %w", what, io.ErrUnexpectedEOF) + } + if n < 0 { + return 0, 0, fmt.Errorf("overflow decoding %s (read %d/%d bytes)", what, -n, len(buf)) + } + return u, n, nil +} + +func decodeVarint(buf []byte, what string) (v int64, n int, err error) { + v, n = binary.Varint(buf) + if n == 0 { + return 0, 0, fmt.Errorf("decoding %s: %w", what, io.ErrUnexpectedEOF) + } + if n < 0 { + return 0, 0, fmt.Errorf("overflow decoding %s (read %d/%d bytes)", what, -n, len(buf)) + } + return v, n, nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.go new file mode 100644 index 0000000000..11a5a538b1 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.go @@ -0,0 +1,256 @@ +//go:build !purego + +package delta + +import ( + "github.com/parquet-go/parquet-go/internal/unsafecast" + "golang.org/x/sys/cpu" +) + +func init() { + if cpu.X86.HasAVX2 { + encodeInt32 = encodeInt32AVX2 + encodeInt64 = encodeInt64AVX2 + } +} + +//go:noescape +func blockDeltaInt32AVX2(block *[blockSize]int32, lastValue int32) int32 + +//go:noescape +func blockMinInt32AVX2(block *[blockSize]int32) int32 + +//go:noescape +func blockSubInt32AVX2(block *[blockSize]int32, value int32) + +//go:noescape +func blockBitWidthsInt32AVX2(bitWidths *[numMiniBlocks]byte, block *[blockSize]int32) + +//go:noescape +func encodeMiniBlockInt32Default(dst *byte, src *[miniBlockSize]int32, bitWidth uint) + +//go:noescape +func encodeMiniBlockInt32x1bitAVX2(dst *byte, src *[miniBlockSize]int32) + +//go:noescape +func encodeMiniBlockInt32x2bitsAVX2(dst *byte, src *[miniBlockSize]int32) + +//go:noescape +func encodeMiniBlockInt32x3to16bitsAVX2(dst *byte, src *[miniBlockSize]int32, bitWidth uint) + +//go:noescape +func encodeMiniBlockInt32x32bitsAVX2(dst *byte, src *[miniBlockSize]int32) + +func encodeMiniBlockInt32(dst []byte, src *[miniBlockSize]int32, bitWidth uint) { + encodeMiniBlockInt32Default(&dst[0], src, bitWidth) +} + +func encodeMiniBlockInt32AVX2(dst *byte, src *[miniBlockSize]int32, bitWidth uint) { + switch { + case bitWidth == 1: + encodeMiniBlockInt32x1bitAVX2(dst, src) + case bitWidth == 2: + encodeMiniBlockInt32x2bitsAVX2(dst, src) + case bitWidth == 32: + encodeMiniBlockInt32x32bitsAVX2(dst, src) + case bitWidth <= 16: + encodeMiniBlockInt32x3to16bitsAVX2(dst, src, bitWidth) + default: + encodeMiniBlockInt32Default(dst, src, bitWidth) + } +} + +func encodeInt32AVX2(dst []byte, src []int32) []byte { + totalValues := len(src) + firstValue := int32(0) + if totalValues > 0 { + firstValue = src[0] + } + + n := len(dst) + dst = resize(dst, n+maxHeaderLength32) + dst = dst[:n+encodeBinaryPackedHeader(dst[n:], blockSize, numMiniBlocks, totalValues, int64(firstValue))] + + if totalValues < 2 { + return dst + } + + lastValue := firstValue + for i := 1; i < len(src); i += blockSize { + block := [blockSize]int32{} + blockLength := copy(block[:], src[i:]) + + lastValue = blockDeltaInt32AVX2(&block, lastValue) + minDelta := blockMinInt32AVX2(&block) + blockSubInt32AVX2(&block, minDelta) + blockClearInt32(&block, blockLength) + + bitWidths := [numMiniBlocks]byte{} + blockBitWidthsInt32AVX2(&bitWidths, &block) + + n := len(dst) + dst = resize(dst, n+maxMiniBlockLength32+16) + n += encodeBlockHeader(dst[n:], int64(minDelta), bitWidths) + + for i, bitWidth := range bitWidths { + if bitWidth != 0 { + miniBlock := (*[miniBlockSize]int32)(block[i*miniBlockSize:]) + encodeMiniBlockInt32AVX2(&dst[n], miniBlock, uint(bitWidth)) + n += (miniBlockSize * int(bitWidth)) / 8 + } + } + + dst = dst[:n] + } + + return dst +} + +//go:noescape +func blockDeltaInt64AVX2(block *[blockSize]int64, lastValue int64) int64 + +//go:noescape +func blockMinInt64AVX2(block *[blockSize]int64) int64 + +//go:noescape +func blockSubInt64AVX2(block *[blockSize]int64, value int64) + +//go:noescape +func blockBitWidthsInt64AVX2(bitWidths *[numMiniBlocks]byte, block *[blockSize]int64) + +//go:noescape +func encodeMiniBlockInt64Default(dst *byte, src *[miniBlockSize]int64, bitWidth uint) + +//go:noescape +func encodeMiniBlockInt64x1bitAVX2(dst *byte, src *[miniBlockSize]int64) + +//go:noescape +func encodeMiniBlockInt64x2bitsAVX2(dst *byte, src *[miniBlockSize]int64) + +//go:noescape +func encodeMiniBlockInt64x64bitsAVX2(dst *byte, src *[miniBlockSize]int64) + +func encodeMiniBlockInt64(dst []byte, src *[miniBlockSize]int64, bitWidth uint) { + encodeMiniBlockInt64Default(&dst[0], src, bitWidth) +} + +func encodeMiniBlockInt64AVX2(dst *byte, src *[miniBlockSize]int64, bitWidth uint) { + switch { + case bitWidth == 1: + encodeMiniBlockInt64x1bitAVX2(dst, src) + case bitWidth == 2: + encodeMiniBlockInt64x2bitsAVX2(dst, src) + case bitWidth == 64: + encodeMiniBlockInt64x64bitsAVX2(dst, src) + default: + encodeMiniBlockInt64Default(dst, src, bitWidth) + } +} + +func encodeInt64AVX2(dst []byte, src []int64) []byte { + totalValues := len(src) + firstValue := int64(0) + if totalValues > 0 { + firstValue = src[0] + } + + n := len(dst) + dst = resize(dst, n+maxHeaderLength64) + dst = dst[:n+encodeBinaryPackedHeader(dst[n:], blockSize, numMiniBlocks, totalValues, int64(firstValue))] + + if totalValues < 2 { + return dst + } + + lastValue := firstValue + for i := 1; i < len(src); i += blockSize { + block := [blockSize]int64{} + blockLength := copy(block[:], src[i:]) + + lastValue = blockDeltaInt64AVX2(&block, lastValue) + minDelta := blockMinInt64AVX2(&block) + blockSubInt64AVX2(&block, minDelta) + blockClearInt64(&block, blockLength) + + bitWidths := [numMiniBlocks]byte{} + blockBitWidthsInt64AVX2(&bitWidths, &block) + + n := len(dst) + dst = resize(dst, n+maxMiniBlockLength64+16) + n += encodeBlockHeader(dst[n:], int64(minDelta), bitWidths) + + for i, bitWidth := range bitWidths { + if bitWidth != 0 { + miniBlock := (*[miniBlockSize]int64)(block[i*miniBlockSize:]) + encodeMiniBlockInt64AVX2(&dst[n], miniBlock, uint(bitWidth)) + n += (miniBlockSize * int(bitWidth)) / 8 + } + } + + dst = dst[:n] + } + + return dst +} + +//go:noescape +func decodeBlockInt32Default(dst []int32, minDelta, lastValue int32) int32 + +//go:noescape +func decodeBlockInt32AVX2(dst []int32, minDelta, lastValue int32) int32 + +func decodeBlockInt32(dst []int32, minDelta, lastValue int32) int32 { + switch { + case cpu.X86.HasAVX2: + return decodeBlockInt32AVX2(dst, minDelta, lastValue) + default: + return decodeBlockInt32Default(dst, minDelta, lastValue) + } +} + +//go:noescape +func decodeMiniBlockInt32Default(dst []int32, src []uint32, bitWidth uint) + +//go:noescape +func decodeMiniBlockInt32x1to16bitsAVX2(dst []int32, src []uint32, bitWidth uint) + +//go:noescape +func decodeMiniBlockInt32x17to26bitsAVX2(dst []int32, src []uint32, bitWidth uint) + +//go:noescape +func decodeMiniBlockInt32x27to31bitsAVX2(dst []int32, src []uint32, bitWidth uint) + +func decodeMiniBlockInt32(dst []int32, src []uint32, bitWidth uint) { + hasAVX2 := cpu.X86.HasAVX2 + switch { + case hasAVX2 && bitWidth <= 16: + decodeMiniBlockInt32x1to16bitsAVX2(dst, src, bitWidth) + case hasAVX2 && bitWidth <= 26: + decodeMiniBlockInt32x17to26bitsAVX2(dst, src, bitWidth) + case hasAVX2 && bitWidth <= 31: + decodeMiniBlockInt32x27to31bitsAVX2(dst, src, bitWidth) + case bitWidth == 32: + copy(dst, unsafecast.Slice[int32](src)) + default: + decodeMiniBlockInt32Default(dst, src, bitWidth) + } +} + +//go:noescape +func decodeBlockInt64Default(dst []int64, minDelta, lastValue int64) int64 + +func decodeBlockInt64(dst []int64, minDelta, lastValue int64) int64 { + return decodeBlockInt64Default(dst, minDelta, lastValue) +} + +//go:noescape +func decodeMiniBlockInt64Default(dst []int64, src []uint32, bitWidth uint) + +func decodeMiniBlockInt64(dst []int64, src []uint32, bitWidth uint) { + switch { + case bitWidth == 64: + copy(dst, unsafecast.Slice[int64](src)) + default: + decodeMiniBlockInt64Default(dst, src, bitWidth) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.s b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.s new file mode 100644 index 0000000000..08f80f0900 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_amd64.s @@ -0,0 +1,920 @@ +//go:build !purego + +#include "textflag.h" + +#define blockSize 128 +#define numMiniBlocks 4 +#define miniBlockSize 32 + +// ----------------------------------------------------------------------------- +// 32 bits +// ----------------------------------------------------------------------------- + +#define deltaInt32AVX2x8(baseAddr) \ + VMOVDQU baseAddr, Y1 \ // [0,1,2,3,4,5,6,7] + VPERMD Y1, Y3, Y2 \ // [7,0,1,2,3,4,5,6] + VPBLENDD $1, Y0, Y2, Y2 \ // [x,0,1,2,3,4,5,6] + VPSUBD Y2, Y1, Y2 \ // [0,1,2,...] - [x,0,1,...] + VMOVDQU Y2, baseAddr \ + VPERMD Y1, Y3, Y0 + +// func blockDeltaInt32AVX2(block *[blockSize]int32, lastValue int32) int32 +TEXT ยทblockDeltaInt32AVX2(SB), NOSPLIT, $0-20 + MOVQ block+0(FP), AX + MOVL 4*blockSize-4(AX), CX + MOVL CX, ret+16(FP) + + VPBROADCASTD lastValue+8(FP), Y0 + VMOVDQU ยทrotateLeft32(SB), Y3 + + XORQ SI, SI +loop: + deltaInt32AVX2x8(0(AX)(SI*4)) + deltaInt32AVX2x8(32(AX)(SI*4)) + deltaInt32AVX2x8(64(AX)(SI*4)) + deltaInt32AVX2x8(96(AX)(SI*4)) + ADDQ $32, SI + CMPQ SI, $blockSize + JNE loop + VZEROUPPER + RET + +// func blockMinInt32AVX2(block *[blockSize]int32) int32 +TEXT ยทblockMinInt32AVX2(SB), NOSPLIT, $0-12 + MOVQ block+0(FP), AX + VPBROADCASTD (AX), Y15 + + VPMINSD 0(AX), Y15, Y0 + VPMINSD 32(AX), Y15, Y1 + VPMINSD 64(AX), Y15, Y2 + VPMINSD 96(AX), Y15, Y3 + VPMINSD 128(AX), Y15, Y4 + VPMINSD 160(AX), Y15, Y5 + VPMINSD 192(AX), Y15, Y6 + VPMINSD 224(AX), Y15, Y7 + VPMINSD 256(AX), Y15, Y8 + VPMINSD 288(AX), Y15, Y9 + VPMINSD 320(AX), Y15, Y10 + VPMINSD 352(AX), Y15, Y11 + VPMINSD 384(AX), Y15, Y12 + VPMINSD 416(AX), Y15, Y13 + VPMINSD 448(AX), Y15, Y14 + VPMINSD 480(AX), Y15, Y15 + + VPMINSD Y1, Y0, Y0 + VPMINSD Y3, Y2, Y2 + VPMINSD Y5, Y4, Y4 + VPMINSD Y7, Y6, Y6 + VPMINSD Y9, Y8, Y8 + VPMINSD Y11, Y10, Y10 + VPMINSD Y13, Y12, Y12 + VPMINSD Y15, Y14, Y14 + + VPMINSD Y2, Y0, Y0 + VPMINSD Y6, Y4, Y4 + VPMINSD Y10, Y8, Y8 + VPMINSD Y14, Y12, Y12 + + VPMINSD Y4, Y0, Y0 + VPMINSD Y12, Y8, Y8 + + VPMINSD Y8, Y0, Y0 + + VPERM2I128 $1, Y0, Y0, Y1 + VPMINSD Y1, Y0, Y0 + + VPSHUFD $0b00011011, Y0, Y1 + VPMINSD Y1, Y0, Y0 + VZEROUPPER + + MOVQ X0, CX + MOVL CX, BX + SHRQ $32, CX + CMPL CX, BX + CMOVLLT CX, BX + MOVL BX, ret+8(FP) + RET + +#define subInt32AVX2x32(baseAddr, offset) \ + VMOVDQU offset+0(baseAddr), Y1 \ + VMOVDQU offset+32(baseAddr), Y2 \ + VMOVDQU offset+64(baseAddr), Y3 \ + VMOVDQU offset+96(baseAddr), Y4 \ + VPSUBD Y0, Y1, Y1 \ + VPSUBD Y0, Y2, Y2 \ + VPSUBD Y0, Y3, Y3 \ + VPSUBD Y0, Y4, Y4 \ + VMOVDQU Y1, offset+0(baseAddr) \ + VMOVDQU Y2, offset+32(baseAddr) \ + VMOVDQU Y3, offset+64(baseAddr) \ + VMOVDQU Y4, offset+96(baseAddr) + +// func blockSubInt32AVX2(block *[blockSize]int32, value int32) +TEXT ยทblockSubInt32AVX2(SB), NOSPLIT, $0-12 + MOVQ block+0(FP), AX + VPBROADCASTD value+8(FP), Y0 + subInt32AVX2x32(AX, 0) + subInt32AVX2x32(AX, 128) + subInt32AVX2x32(AX, 256) + subInt32AVX2x32(AX, 384) + VZEROUPPER + RET + +// func blockBitWidthsInt32AVX2(bitWidths *[numMiniBlocks]byte, block *[blockSize]int32) +TEXT ยทblockBitWidthsInt32AVX2(SB), NOSPLIT, $0-16 + MOVQ bitWidths+0(FP), AX + MOVQ block+8(FP), BX + + // AVX2 only has signed comparisons (and min/max), we emulate working on + // unsigned values by adding -2^31 to the values. Y5 is a vector of -2^31 + // used to offset 8 packed 32 bits integers in other YMM registers where + // the block data are loaded. + VPCMPEQD Y5, Y5, Y5 + VPSLLD $31, Y5, Y5 + + XORQ DI, DI +loop: + VPBROADCASTD (BX), Y0 // max + VPADDD Y5, Y0, Y0 + + VMOVDQU (BX), Y1 + VMOVDQU 32(BX), Y2 + VMOVDQU 64(BX), Y3 + VMOVDQU 96(BX), Y4 + + VPADDD Y5, Y1, Y1 + VPADDD Y5, Y2, Y2 + VPADDD Y5, Y3, Y3 + VPADDD Y5, Y4, Y4 + + VPMAXSD Y2, Y1, Y1 + VPMAXSD Y4, Y3, Y3 + VPMAXSD Y3, Y1, Y1 + VPMAXSD Y1, Y0, Y0 + + VPERM2I128 $1, Y0, Y0, Y1 + VPMAXSD Y1, Y0, Y0 + + VPSHUFD $0b00011011, Y0, Y1 + VPMAXSD Y1, Y0, Y0 + VPSUBD Y5, Y0, Y0 + + MOVQ X0, CX + MOVL CX, DX + SHRQ $32, CX + CMPL CX, DX + CMOVLHI CX, DX + + LZCNTL DX, DX + NEGL DX + ADDL $32, DX + MOVB DX, (AX)(DI*1) + + ADDQ $128, BX + INCQ DI + CMPQ DI, $numMiniBlocks + JNE loop + VZEROUPPER + RET + +// encodeMiniBlockInt32Default is the generic implementation of the algorithm to +// pack 32 bit integers into values of a given bit width (<=32). +// +// This algorithm is much slower than the vectorized versions, but is useful +// as a reference implementation to run the tests against, and as fallback when +// the code runs on a CPU which does not support the AVX2 instruction set. +// +// func encodeMiniBlockInt32Default(dst *byte, src *[miniBlockSize]int32, bitWidth uint) +TEXT ยทencodeMiniBlockInt32Default(SB), NOSPLIT, $0-24 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + MOVQ bitWidth+16(FP), R9 + + XORQ DI, DI // bitOffset + XORQ SI, SI +loop: + MOVQ DI, CX + MOVQ DI, DX + + ANDQ $0b11111, CX // bitOffset % 32 + SHRQ $5, DX // bitOffset / 32 + + MOVLQZX (BX)(SI*4), R8 + SHLQ CX, R8 + ORQ R8, (AX)(DX*4) + + ADDQ R9, DI + INCQ SI + CMPQ SI, $miniBlockSize + JNE loop + RET + +// encodeMiniBlockInt32x1bitAVX2 packs 32 bit integers into 1 bit values in the +// the output buffer. +// +// The algorithm uses MOVMSKPS to extract the 8 relevant bits from the 8 values +// packed in YMM registers, then combines 4 of these into a 32 bit word which +// then gets written to the output. The result is 32 bits because each mini +// block has 32 values (the block size is 128 and there are 4 mini blocks per +// block). +// +// func encodeMiniBlockInt32x1bitAVX2(dst *byte, src *[miniBlockSize]int32) +TEXT ยทencodeMiniBlockInt32x1bitAVX2(SB), NOSPLIT, $0-16 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + + VMOVDQU 0(BX), Y0 + VMOVDQU 32(BX), Y1 + VMOVDQU 64(BX), Y2 + VMOVDQU 96(BX), Y3 + + VPSLLD $31, Y0, Y0 + VPSLLD $31, Y1, Y1 + VPSLLD $31, Y2, Y2 + VPSLLD $31, Y3, Y3 + + VMOVMSKPS Y0, R8 + VMOVMSKPS Y1, R9 + VMOVMSKPS Y2, R10 + VMOVMSKPS Y3, R11 + + SHLL $8, R9 + SHLL $16, R10 + SHLL $24, R11 + + ORL R9, R8 + ORL R10, R8 + ORL R11, R8 + MOVL R8, (AX) + VZEROUPPER + RET + +// encodeMiniBlockInt32x2bitsAVX2 implements an algorithm for packing 32 bit +// integers into 2 bit values. +// +// The algorithm is derived from the one employed in encodeMiniBlockInt32x1bitAVX2 +// but needs to perform a bit extra work since MOVMSKPS can only extract one bit +// per packed integer of each YMM vector. We run two passes to extract the two +// bits needed to compose each item of the result, and merge the values by +// interleaving the first and second bits with PDEP. +// +// func encodeMiniBlockInt32x2bitsAVX2(dst *byte, src *[miniBlockSize]int32) +TEXT ยทencodeMiniBlockInt32x2bitsAVX2(SB), NOSPLIT, $0-16 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + + VMOVDQU 0(BX), Y0 + VMOVDQU 32(BX), Y1 + VMOVDQU 64(BX), Y2 + VMOVDQU 96(BX), Y3 + + VPSLLD $31, Y0, Y4 + VPSLLD $31, Y1, Y5 + VPSLLD $31, Y2, Y6 + VPSLLD $31, Y3, Y7 + + VMOVMSKPS Y4, R8 + VMOVMSKPS Y5, R9 + VMOVMSKPS Y6, R10 + VMOVMSKPS Y7, R11 + + SHLQ $8, R9 + SHLQ $16, R10 + SHLQ $24, R11 + ORQ R9, R8 + ORQ R10, R8 + ORQ R11, R8 + + MOVQ $0x5555555555555555, DX // 0b010101... + PDEPQ DX, R8, R8 + + VPSLLD $30, Y0, Y8 + VPSLLD $30, Y1, Y9 + VPSLLD $30, Y2, Y10 + VPSLLD $30, Y3, Y11 + + VMOVMSKPS Y8, R12 + VMOVMSKPS Y9, R13 + VMOVMSKPS Y10, R14 + VMOVMSKPS Y11, R15 + + SHLQ $8, R13 + SHLQ $16, R14 + SHLQ $24, R15 + ORQ R13, R12 + ORQ R14, R12 + ORQ R15, R12 + + MOVQ $0xAAAAAAAAAAAAAAAA, DI // 0b101010... + PDEPQ DI, R12, R12 + + ORQ R12, R8 + MOVQ R8, (AX) + VZEROUPPER + RET + +// encodeMiniBlockInt32x32bitsAVX2 is a specialization of the bit packing logic +// for 32 bit integers when the output bit width is also 32, in which case a +// simple copy of the mini block to the output buffer produces the result. +// +// func encodeMiniBlockInt32x32bitsAVX2(dst *byte, src *[miniBlockSize]int32) +TEXT ยทencodeMiniBlockInt32x32bitsAVX2(SB), NOSPLIT, $0-16 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + VMOVDQU 0(BX), Y0 + VMOVDQU 32(BX), Y1 + VMOVDQU 64(BX), Y2 + VMOVDQU 96(BX), Y3 + VMOVDQU Y0, 0(AX) + VMOVDQU Y1, 32(AX) + VMOVDQU Y2, 64(AX) + VMOVDQU Y3, 96(AX) + VZEROUPPER + RET + +// encodeMiniBlockInt32x3to16bitsAVX2 is the algorithm used to bit-pack 32 bit +// integers into values of width 3 to 16 bits. +// +// This function is a small overhead due to having to initialize registers with +// values that depend on the bit width. We measured this cost at ~10% throughput +// in synthetic benchmarks compared to generating constant shifts and offsets +// using a macro. Using a single function rather than generating one for each +// bit width has the benefit of reducing the code size, which in practice can +// also yield benefits like reducing CPU cache misses. Not using a macro also +// has other advantages like providing accurate line number of stack traces and +// enabling the use of breakpoints when debugging. Overall, this approach seemed +// to be the right trade off between performance and maintainability. +// +// The algorithm treats chunks of 8 values in 4 iterations to process all 32 +// values of the mini block. Writes to the output buffer are aligned on 128 bits +// since we may write up to 128 bits (8 x 16 bits). Padding is therefore +// required in the output buffer to avoid triggering a segfault. +// The encodeInt32AVX2 method adds enough padding when sizing the output buffer +// to account for this requirement. +// +// We leverage the two lanes of YMM registers to work on two sets of 4 values +// (in the sequence of VMOVDQU/VPSHUFD, VPAND, VPSLLQ, VPOR), resulting in having +// two sets of bit-packed values in the lower 64 bits of each YMM lane. +// The upper lane is then permuted into a lower lane to merge the two results, +// which may not be aligned on byte boundaries so we shift the lower and upper +// bits and compose two sets of 128 bits sequences (VPSLLQ, VPSRLQ, VBLENDPD), +// merge them and write the 16 bytes result to the output buffer. +TEXT ยทencodeMiniBlockInt32x3to16bitsAVX2(SB), NOSPLIT, $0-24 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + MOVQ bitWidth+16(FP), CX + + VPBROADCASTQ bitWidth+16(FP), Y6 // [1*bitWidth...] + VPSLLQ $1, Y6, Y7 // [2*bitWidth...] + VPADDQ Y6, Y7, Y8 // [3*bitWidth...] + VPSLLQ $2, Y6, Y9 // [4*bitWidth...] + + VPBROADCASTQ sixtyfour<>(SB), Y10 + VPSUBQ Y6, Y10, Y11 // [64-1*bitWidth...] + VPSUBQ Y9, Y10, Y12 // [64-4*bitWidth...] + VPCMPEQQ Y4, Y4, Y4 + VPSRLVQ Y11, Y4, Y4 + + VPXOR Y5, Y5, Y5 + XORQ SI, SI +loop: + VMOVDQU (BX)(SI*4), Y0 + VPSHUFD $0b01010101, Y0, Y1 + VPSHUFD $0b10101010, Y0, Y2 + VPSHUFD $0b11111111, Y0, Y3 + + VPAND Y4, Y0, Y0 + VPAND Y4, Y1, Y1 + VPAND Y4, Y2, Y2 + VPAND Y4, Y3, Y3 + + VPSLLVQ Y6, Y1, Y1 + VPSLLVQ Y7, Y2, Y2 + VPSLLVQ Y8, Y3, Y3 + + VPOR Y1, Y0, Y0 + VPOR Y3, Y2, Y2 + VPOR Y2, Y0, Y0 + + VPERMQ $0b00001010, Y0, Y1 + + VPSLLVQ X9, X1, X2 + VPSRLQ X12, X1, X3 + VBLENDPD $0b10, X3, X2, X1 + VBLENDPD $0b10, X5, X0, X0 + VPOR X1, X0, X0 + + VMOVDQU X0, (AX) + + ADDQ CX, AX + ADDQ $8, SI + CMPQ SI, $miniBlockSize + JNE loop + VZEROUPPER + RET + +GLOBL sixtyfour<>(SB), RODATA|NOPTR, $32 +DATA sixtyfour<>+0(SB)/8, $64 +DATA sixtyfour<>+8(SB)/8, $64 +DATA sixtyfour<>+16(SB)/8, $64 +DATA sixtyfour<>+24(SB)/8, $64 + +// func decodeBlockInt32Default(dst []int32, minDelta, lastValue int32) int32 +TEXT ยทdecodeBlockInt32Default(SB), NOSPLIT, $0-36 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVLQZX minDelta+24(FP), CX + MOVLQZX lastValue+28(FP), DX + XORQ SI, SI + JMP test +loop: + MOVL (AX)(SI*4), DI + ADDL CX, DI + ADDL DI, DX + MOVL DX, (AX)(SI*4) + INCQ SI +test: + CMPQ SI, BX + JNE loop +done: + MOVL DX, ret+32(FP) + RET + +// func decodeBlockInt32AVX2(dst []int32, minDelta, lastValue int32) int32 +TEXT ยทdecodeBlockInt32AVX2(SB), NOSPLIT, $0-36 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVLQZX minDelta+24(FP), CX + MOVLQZX lastValue+28(FP), DX + XORQ SI, SI + + CMPQ BX, $8 + JB test + + MOVQ BX, DI + SHRQ $3, DI + SHLQ $3, DI + + VPXOR X1, X1, X1 + MOVQ CX, X0 + MOVQ DX, X1 + VPBROADCASTD X0, Y0 +loopAVX2: + VMOVDQU (AX)(SI*4), Y2 + VPADDD Y0, Y2, Y2 // Y2[:] += minDelta + VPADDD Y1, Y2, Y2 // Y2[0] += lastValue + + VPSLLDQ $4, Y2, Y3 + VPADDD Y3, Y2, Y2 + + VPSLLDQ $8, Y2, Y3 + VPADDD Y3, Y2, Y2 + + VPSHUFD $0xFF, X2, X1 + VPERM2I128 $1, Y2, Y2, Y3 + VPADDD X1, X3, X3 + + VMOVDQU X2, (AX)(SI*4) + VMOVDQU X3, 16(AX)(SI*4) + VPSRLDQ $12, X3, X1 // lastValue + + ADDQ $8, SI + CMPQ SI, DI + JNE loopAVX2 + VZEROUPPER + MOVQ X1, DX + JMP test +loop: + MOVL (AX)(SI*4), DI + ADDL CX, DI + ADDL DI, DX + MOVL DX, (AX)(SI*4) + INCQ SI +test: + CMPQ SI, BX + JNE loop +done: + MOVL DX, ret+32(FP) + RET + +// ----------------------------------------------------------------------------- +// 64 bits +// ----------------------------------------------------------------------------- + +#define deltaInt64AVX2x4(baseAddr) \ + VMOVDQU baseAddr, Y1 \ // [0,1,2,3] + VPERMQ $0b10010011, Y1, Y2 \ // [3,0,1,2] + VPBLENDD $3, Y0, Y2, Y2 \ // [x,0,1,2] + VPSUBQ Y2, Y1, Y2 \ // [0,1,2,3] - [x,0,1,2] + VMOVDQU Y2, baseAddr \ + VPERMQ $0b10010011, Y1, Y0 + +// func blockDeltaInt64AVX2(block *[blockSize]int64, lastValue int64) int64 +TEXT ยทblockDeltaInt64AVX2(SB), NOSPLIT, $0-24 + MOVQ block+0(FP), AX + MOVQ 8*blockSize-8(AX), CX + MOVQ CX, ret+16(FP) + + VPBROADCASTQ lastValue+8(FP), Y0 + XORQ SI, SI +loop: + deltaInt64AVX2x4((AX)(SI*8)) + deltaInt64AVX2x4(32(AX)(SI*8)) + deltaInt64AVX2x4(64(AX)(SI*8)) + deltaInt64AVX2x4(96(AX)(SI*8)) + ADDQ $16, SI + CMPQ SI, $blockSize + JNE loop + VZEROUPPER + RET + +// vpminsq is an emulation of the AVX-512 VPMINSQ instruction with AVX2. +#define vpminsq(ones, tmp, arg2, arg1, ret) \ + VPCMPGTQ arg1, arg2, tmp \ + VPBLENDVB tmp, arg1, arg2, ret + +// func blockMinInt64AVX2(block *[blockSize]int64) int64 +TEXT ยทblockMinInt64AVX2(SB), NOSPLIT, $0-16 + MOVQ block+0(FP), AX + XORQ SI, SI + VPCMPEQQ Y9, Y9, Y9 // ones + VPBROADCASTQ (AX), Y0 +loop: + VMOVDQU 0(AX)(SI*8), Y1 + VMOVDQU 32(AX)(SI*8), Y2 + VMOVDQU 64(AX)(SI*8), Y3 + VMOVDQU 96(AX)(SI*8), Y4 + VMOVDQU 128(AX)(SI*8), Y5 + VMOVDQU 160(AX)(SI*8), Y6 + VMOVDQU 192(AX)(SI*8), Y7 + VMOVDQU 224(AX)(SI*8), Y8 + + vpminsq(Y9, Y10, Y0, Y1, Y1) + vpminsq(Y9, Y11, Y0, Y2, Y2) + vpminsq(Y9, Y12, Y0, Y3, Y3) + vpminsq(Y9, Y13, Y0, Y4, Y4) + vpminsq(Y9, Y14, Y0, Y5, Y5) + vpminsq(Y9, Y15, Y0, Y6, Y6) + vpminsq(Y9, Y10, Y0, Y7, Y7) + vpminsq(Y9, Y11, Y0, Y8, Y8) + + vpminsq(Y9, Y12, Y2, Y1, Y1) + vpminsq(Y9, Y13, Y4, Y3, Y3) + vpminsq(Y9, Y14, Y6, Y5, Y5) + vpminsq(Y9, Y15, Y8, Y7, Y7) + + vpminsq(Y9, Y10, Y3, Y1, Y1) + vpminsq(Y9, Y11, Y7, Y5, Y5) + vpminsq(Y9, Y12, Y5, Y1, Y0) + + ADDQ $32, SI + CMPQ SI, $blockSize + JNE loop + + VPERM2I128 $1, Y0, Y0, Y1 + vpminsq(Y9, Y10, Y1, Y0, Y0) + + MOVQ X0, CX + VPEXTRQ $1, X0, BX + CMPQ CX, BX + CMOVQLT CX, BX + MOVQ BX, ret+8(FP) + VZEROUPPER + RET + +#define subInt64AVX2x32(baseAddr, offset) \ + VMOVDQU offset+0(baseAddr), Y1 \ + VMOVDQU offset+32(baseAddr), Y2 \ + VMOVDQU offset+64(baseAddr), Y3 \ + VMOVDQU offset+96(baseAddr), Y4 \ + VMOVDQU offset+128(baseAddr), Y5 \ + VMOVDQU offset+160(baseAddr), Y6 \ + VMOVDQU offset+192(baseAddr), Y7 \ + VMOVDQU offset+224(baseAddr), Y8 \ + VPSUBQ Y0, Y1, Y1 \ + VPSUBQ Y0, Y2, Y2 \ + VPSUBQ Y0, Y3, Y3 \ + VPSUBQ Y0, Y4, Y4 \ + VPSUBQ Y0, Y5, Y5 \ + VPSUBQ Y0, Y6, Y6 \ + VPSUBQ Y0, Y7, Y7 \ + VPSUBQ Y0, Y8, Y8 \ + VMOVDQU Y1, offset+0(baseAddr) \ + VMOVDQU Y2, offset+32(baseAddr) \ + VMOVDQU Y3, offset+64(baseAddr) \ + VMOVDQU Y4, offset+96(baseAddr) \ + VMOVDQU Y5, offset+128(baseAddr) \ + VMOVDQU Y6, offset+160(baseAddr) \ + VMOVDQU Y7, offset+192(baseAddr) \ + VMOVDQU Y8, offset+224(baseAddr) + +// func blockSubInt64AVX2(block *[blockSize]int64, value int64) +TEXT ยทblockSubInt64AVX2(SB), NOSPLIT, $0-16 + MOVQ block+0(FP), AX + VPBROADCASTQ value+8(FP), Y0 + subInt64AVX2x32(AX, 0) + subInt64AVX2x32(AX, 256) + subInt64AVX2x32(AX, 512) + subInt64AVX2x32(AX, 768) + VZEROUPPER + RET + +// vpmaxsq is an emulation of the AVX-512 VPMAXSQ instruction with AVX2. +#define vpmaxsq(tmp, arg2, arg1, ret) \ + VPCMPGTQ arg2, arg1, tmp \ + VPBLENDVB tmp, arg1, arg2, ret + +// func blockBitWidthsInt64AVX2(bitWidths *[numMiniBlocks]byte, block *[blockSize]int64) +TEXT ยทblockBitWidthsInt64AVX2(SB), NOSPLIT, $0-16 + MOVQ bitWidths+0(FP), AX + MOVQ block+8(FP), BX + + // AVX2 only has signed comparisons (and min/max), we emulate working on + // unsigned values by adding -2^64 to the values. Y9 is a vector of -2^64 + // used to offset 4 packed 64 bits integers in other YMM registers where + // the block data are loaded. + VPCMPEQQ Y9, Y9, Y9 + VPSLLQ $63, Y9, Y9 + + XORQ DI, DI +loop: + VPBROADCASTQ (BX), Y0 // max + VPADDQ Y9, Y0, Y0 + + VMOVDQU (BX), Y1 + VMOVDQU 32(BX), Y2 + VMOVDQU 64(BX), Y3 + VMOVDQU 96(BX), Y4 + VMOVDQU 128(BX), Y5 + VMOVDQU 160(BX), Y6 + VMOVDQU 192(BX), Y7 + VMOVDQU 224(BX), Y8 + + VPADDQ Y9, Y1, Y1 + VPADDQ Y9, Y2, Y2 + VPADDQ Y9, Y3, Y3 + VPADDQ Y9, Y4, Y4 + VPADDQ Y9, Y5, Y5 + VPADDQ Y9, Y6, Y6 + VPADDQ Y9, Y7, Y7 + VPADDQ Y9, Y8, Y8 + + vpmaxsq(Y10, Y2, Y1, Y1) + vpmaxsq(Y11, Y4, Y3, Y3) + vpmaxsq(Y12, Y6, Y5, Y5) + vpmaxsq(Y13, Y8, Y7, Y7) + + vpmaxsq(Y10, Y3, Y1, Y1) + vpmaxsq(Y11, Y7, Y5, Y5) + vpmaxsq(Y12, Y5, Y1, Y1) + vpmaxsq(Y13, Y1, Y0, Y0) + + VPERM2I128 $1, Y0, Y0, Y1 + vpmaxsq(Y10, Y1, Y0, Y0) + VPSUBQ Y9, Y0, Y0 + + MOVQ X0, CX + VPEXTRQ $1, X0, DX + CMPQ CX, DX + CMOVQHI CX, DX + + LZCNTQ DX, DX + NEGQ DX + ADDQ $64, DX + MOVB DX, (AX)(DI*1) + + ADDQ $256, BX + INCQ DI + CMPQ DI, $numMiniBlocks + JNE loop + VZEROUPPER + RET + +// encodeMiniBlockInt64Default is the generic implementation of the algorithm to +// pack 64 bit integers into values of a given bit width (<=64). +// +// This algorithm is much slower than the vectorized versions, but is useful +// as a reference implementation to run the tests against, and as fallback when +// the code runs on a CPU which does not support the AVX2 instruction set. +// +// func encodeMiniBlockInt64Default(dst *byte, src *[miniBlockSize]int64, bitWidth uint) +TEXT ยทencodeMiniBlockInt64Default(SB), NOSPLIT, $0-24 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + MOVQ bitWidth+16(FP), R10 + + XORQ R11, R11 // zero + XORQ DI, DI // bitOffset + XORQ SI, SI +loop: + MOVQ DI, CX + MOVQ DI, DX + + ANDQ $0b111111, CX // bitOffset % 64 + SHRQ $6, DX // bitOffset / 64 + + MOVQ (BX)(SI*8), R8 + MOVQ R8, R9 + SHLQ CX, R8 + NEGQ CX + ADDQ $64, CX + SHRQ CX, R9 + CMPQ CX, $64 + CMOVQEQ R11, R9 // needed because shifting by more than 63 is undefined + + ORQ R8, 0(AX)(DX*8) + ORQ R9, 8(AX)(DX*8) + + ADDQ R10, DI + INCQ SI + CMPQ SI, $miniBlockSize + JNE loop + RET + +// func encodeMiniBlockInt64x1bitAVX2(dst *byte, src *[miniBlockSize]int64) +TEXT ยทencodeMiniBlockInt64x1bitAVX2(SB), NOSPLIT, $0-16 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + + VMOVDQU 0(BX), Y0 + VMOVDQU 32(BX), Y1 + VMOVDQU 64(BX), Y2 + VMOVDQU 96(BX), Y3 + VMOVDQU 128(BX), Y4 + VMOVDQU 160(BX), Y5 + VMOVDQU 192(BX), Y6 + VMOVDQU 224(BX), Y7 + + VPSLLQ $63, Y0, Y0 + VPSLLQ $63, Y1, Y1 + VPSLLQ $63, Y2, Y2 + VPSLLQ $63, Y3, Y3 + VPSLLQ $63, Y4, Y4 + VPSLLQ $63, Y5, Y5 + VPSLLQ $63, Y6, Y6 + VPSLLQ $63, Y7, Y7 + + VMOVMSKPD Y0, R8 + VMOVMSKPD Y1, R9 + VMOVMSKPD Y2, R10 + VMOVMSKPD Y3, R11 + VMOVMSKPD Y4, R12 + VMOVMSKPD Y5, R13 + VMOVMSKPD Y6, R14 + VMOVMSKPD Y7, R15 + + SHLL $4, R9 + SHLL $8, R10 + SHLL $12, R11 + SHLL $16, R12 + SHLL $20, R13 + SHLL $24, R14 + SHLL $28, R15 + + ORL R9, R8 + ORL R11, R10 + ORL R13, R12 + ORL R15, R14 + ORL R10, R8 + ORL R14, R12 + ORL R12, R8 + + MOVL R8, (AX) + VZEROUPPER + RET + +// func encodeMiniBlockInt64x2bitsAVX2(dst *byte, src *[miniBlockSize]int64) +TEXT ยทencodeMiniBlockInt64x2bitsAVX2(SB), NOSPLIT, $0-16 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + + VMOVDQU 0(BX), Y8 + VMOVDQU 32(BX), Y9 + VMOVDQU 64(BX), Y10 + VMOVDQU 96(BX), Y11 + VMOVDQU 128(BX), Y12 + VMOVDQU 160(BX), Y13 + VMOVDQU 192(BX), Y14 + VMOVDQU 224(BX), Y15 + + VPSLLQ $63, Y8, Y0 + VPSLLQ $63, Y9, Y1 + VPSLLQ $63, Y10, Y2 + VPSLLQ $63, Y11, Y3 + VPSLLQ $63, Y12, Y4 + VPSLLQ $63, Y13, Y5 + VPSLLQ $63, Y14, Y6 + VPSLLQ $63, Y15, Y7 + + VMOVMSKPD Y0, R8 + VMOVMSKPD Y1, R9 + VMOVMSKPD Y2, R10 + VMOVMSKPD Y3, R11 + VMOVMSKPD Y4, R12 + VMOVMSKPD Y5, R13 + VMOVMSKPD Y6, R14 + VMOVMSKPD Y7, R15 + + SHLQ $4, R9 + SHLQ $8, R10 + SHLQ $12, R11 + SHLQ $16, R12 + SHLQ $20, R13 + SHLQ $24, R14 + SHLQ $28, R15 + + ORQ R9, R8 + ORQ R11, R10 + ORQ R13, R12 + ORQ R15, R14 + ORQ R10, R8 + ORQ R14, R12 + ORQ R12, R8 + + MOVQ $0x5555555555555555, CX // 0b010101... + PDEPQ CX, R8, CX + + VPSLLQ $62, Y8, Y8 + VPSLLQ $62, Y9, Y9 + VPSLLQ $62, Y10, Y10 + VPSLLQ $62, Y11, Y11 + VPSLLQ $62, Y12, Y12 + VPSLLQ $62, Y13, Y13 + VPSLLQ $62, Y14, Y14 + VPSLLQ $62, Y15, Y15 + + VMOVMSKPD Y8, R8 + VMOVMSKPD Y9, R9 + VMOVMSKPD Y10, R10 + VMOVMSKPD Y11, R11 + VMOVMSKPD Y12, R12 + VMOVMSKPD Y13, R13 + VMOVMSKPD Y14, R14 + VMOVMSKPD Y15, R15 + + SHLQ $4, R9 + SHLQ $8, R10 + SHLQ $12, R11 + SHLQ $16, R12 + SHLQ $20, R13 + SHLQ $24, R14 + SHLQ $28, R15 + + ORQ R9, R8 + ORQ R11, R10 + ORQ R13, R12 + ORQ R15, R14 + ORQ R10, R8 + ORQ R14, R12 + ORQ R12, R8 + + MOVQ $0xAAAAAAAAAAAAAAAA, DX // 0b101010... + PDEPQ DX, R8, DX + ORQ DX, CX + MOVQ CX, (AX) + VZEROUPPER + RET + +// func encodeMiniBlockInt64x64bitsAVX2(dst *byte, src *[miniBlockSize]int64) +TEXT ยทencodeMiniBlockInt64x64bitsAVX2(SB), NOSPLIT, $0-16 + MOVQ dst+0(FP), AX + MOVQ src+8(FP), BX + VMOVDQU 0(BX), Y0 + VMOVDQU 32(BX), Y1 + VMOVDQU 64(BX), Y2 + VMOVDQU 96(BX), Y3 + VMOVDQU 128(BX), Y4 + VMOVDQU 160(BX), Y5 + VMOVDQU 192(BX), Y6 + VMOVDQU 224(BX), Y7 + VMOVDQU Y0, 0(AX) + VMOVDQU Y1, 32(AX) + VMOVDQU Y2, 64(AX) + VMOVDQU Y3, 96(AX) + VMOVDQU Y4, 128(AX) + VMOVDQU Y5, 160(AX) + VMOVDQU Y6, 192(AX) + VMOVDQU Y7, 224(AX) + VZEROUPPER + RET + +// func decodeBlockInt64Default(dst []int64, minDelta, lastValue int64) int64 +TEXT ยทdecodeBlockInt64Default(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVQ minDelta+24(FP), CX + MOVQ lastValue+32(FP), DX + XORQ SI, SI + JMP test +loop: + MOVQ (AX)(SI*8), DI + ADDQ CX, DI + ADDQ DI, DX + MOVQ DX, (AX)(SI*8) + INCQ SI +test: + CMPQ SI, BX + JNE loop +done: + MOVQ DX, ret+40(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_purego.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_purego.go new file mode 100644 index 0000000000..aa1f62367b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/binary_packed_purego.go @@ -0,0 +1,105 @@ +//go:build purego || !amd64 + +package delta + +import ( + "encoding/binary" +) + +func encodeMiniBlockInt32(dst []byte, src *[miniBlockSize]int32, bitWidth uint) { + bitMask := uint32(1<> (32 - j)) + + binary.LittleEndian.PutUint32(dst[(i+0)*4:], lo) + binary.LittleEndian.PutUint32(dst[(i+1)*4:], hi) + + bitOffset += bitWidth + } +} + +func encodeMiniBlockInt64(dst []byte, src *[miniBlockSize]int64, bitWidth uint) { + bitMask := uint64(1<> (64 - j)) + + binary.LittleEndian.PutUint64(dst[(i+0)*8:], lo) + binary.LittleEndian.PutUint64(dst[(i+1)*8:], hi) + + bitOffset += bitWidth + } +} + +func decodeBlockInt32(block []int32, minDelta, lastValue int32) int32 { + for i := range block { + block[i] += minDelta + block[i] += lastValue + lastValue = block[i] + } + return lastValue +} + +func decodeBlockInt64(block []int64, minDelta, lastValue int64) int64 { + for i := range block { + block[i] += minDelta + block[i] += lastValue + lastValue = block[i] + } + return lastValue +} + +func decodeMiniBlockInt32(dst []int32, src []uint32, bitWidth uint) { + bitMask := uint32(1<> j + if j+bitWidth > 32 { + k := 32 - j + d |= (src[i+1] & (bitMask >> k)) << k + } + dst[n] = int32(d) + bitOffset += bitWidth + } +} + +func decodeMiniBlockInt64(dst []int64, src []uint32, bitWidth uint) { + bitMask := uint64(1<> j + if j+bitWidth > 32 { + k := 32 - j + d |= (uint64(src[i+1]) & (bitMask >> k)) << k + if j+bitWidth > 64 { + k := 64 - j + d |= (uint64(src[i+2]) & (bitMask >> k)) << k + } + } + dst[n] = int64(d) + bitOffset += bitWidth + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array.go new file mode 100644 index 0000000000..7033dc95b3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array.go @@ -0,0 +1,212 @@ +package delta + +import ( + "bytes" + "sort" + + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" +) + +const ( + maxLinearSearchPrefixLength = 64 // arbitrary +) + +type ByteArrayEncoding struct { + encoding.NotSupported +} + +func (e *ByteArrayEncoding) String() string { + return "DELTA_BYTE_ARRAY" +} + +func (e *ByteArrayEncoding) Encoding() format.Encoding { + return format.DeltaByteArray +} + +func (e *ByteArrayEncoding) EncodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, error) { + prefix := getInt32Buffer() + defer putInt32Buffer(prefix) + + length := getInt32Buffer() + defer putInt32Buffer(length) + + totalSize := 0 + if len(offsets) > 0 { + lastValue := ([]byte)(nil) + baseOffset := offsets[0] + + for _, endOffset := range offsets[1:] { + v := src[baseOffset:endOffset:endOffset] + n := int(endOffset - baseOffset) + p := 0 + baseOffset = endOffset + + if len(v) <= maxLinearSearchPrefixLength { + p = linearSearchPrefixLength(lastValue, v) + } else { + p = binarySearchPrefixLength(lastValue, v) + } + + prefix.values = append(prefix.values, int32(p)) + length.values = append(length.values, int32(n-p)) + lastValue = v + totalSize += n - p + } + } + + dst = dst[:0] + dst = encodeInt32(dst, prefix.values) + dst = encodeInt32(dst, length.values) + dst = resize(dst, len(dst)+totalSize) + + if len(offsets) > 0 { + b := dst[len(dst)-totalSize:] + i := int(offsets[0]) + j := 0 + + _ = length.values[:len(prefix.values)] + + for k, p := range prefix.values { + n := p + length.values[k] + j += copy(b[j:], src[i+int(p):i+int(n)]) + i += int(n) + } + } + + return dst, nil +} + +func (e *ByteArrayEncoding) EncodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + // The parquet specs say that this encoding is only supported for BYTE_ARRAY + // values, but the reference Java implementation appears to support + // FIXED_LEN_BYTE_ARRAY as well: + // https://github.com/apache/parquet-java/blob/5608695f5777de1eb0899d9075ec9411cfdf31d3/parquet-column/src/main/java/org/apache/parquet/column/Encoding.java#L211 + if size < 0 || size > encoding.MaxFixedLenByteArraySize { + return dst[:0], encoding.Error(e, encoding.ErrInvalidArgument) + } + if (len(src) % size) != 0 { + return dst[:0], encoding.ErrEncodeInvalidInputSize(e, "FIXED_LEN_BYTE_ARRAY", len(src)) + } + + prefix := getInt32Buffer() + defer putInt32Buffer(prefix) + + length := getInt32Buffer() + defer putInt32Buffer(length) + + totalSize := 0 + lastValue := ([]byte)(nil) + + for i := size; i <= len(src); i += size { + v := src[i-size : i : i] + p := linearSearchPrefixLength(lastValue, v) + n := size - p + prefix.values = append(prefix.values, int32(p)) + length.values = append(length.values, int32(n)) + lastValue = v + totalSize += n + } + + dst = dst[:0] + dst = encodeInt32(dst, prefix.values) + dst = encodeInt32(dst, length.values) + dst = resize(dst, len(dst)+totalSize) + + b := dst[len(dst)-totalSize:] + i := 0 + j := 0 + + for _, p := range prefix.values { + j += copy(b[j:], src[i+int(p):i+size]) + i += size + } + + return dst, nil +} + +func (e *ByteArrayEncoding) DecodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, []uint32, error) { + dst, offsets = dst[:0], offsets[:0] + + prefix := getInt32Buffer() + defer putInt32Buffer(prefix) + + suffix := getInt32Buffer() + defer putInt32Buffer(suffix) + + var err error + src, err = prefix.decode(src) + if err != nil { + return dst, offsets, e.wrapf("decoding prefix lengths: %w", err) + } + src, err = suffix.decode(src) + if err != nil { + return dst, offsets, e.wrapf("decoding suffix lengths: %w", err) + } + if len(prefix.values) != len(suffix.values) { + return dst, offsets, e.wrap(errPrefixAndSuffixLengthMismatch(len(prefix.values), len(suffix.values))) + } + return decodeByteArray(dst, src, prefix.values, suffix.values, offsets) +} + +func (e *ByteArrayEncoding) DecodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + dst = dst[:0] + + if size < 0 || size > encoding.MaxFixedLenByteArraySize { + return dst, e.wrap(encoding.ErrInvalidArgument) + } + + prefix := getInt32Buffer() + defer putInt32Buffer(prefix) + + suffix := getInt32Buffer() + defer putInt32Buffer(suffix) + + var err error + src, err = prefix.decode(src) + if err != nil { + return dst, e.wrapf("decoding prefix lengths: %w", err) + } + src, err = suffix.decode(src) + if err != nil { + return dst, e.wrapf("decoding suffix lengths: %w", err) + } + if len(prefix.values) != len(suffix.values) { + return dst, e.wrap(errPrefixAndSuffixLengthMismatch(len(prefix.values), len(suffix.values))) + } + return decodeFixedLenByteArray(dst[:0], src, size, prefix.values, suffix.values) +} + +func (e *ByteArrayEncoding) EstimateDecodeByteArraySize(src []byte) int { + length := getInt32Buffer() + defer putInt32Buffer(length) + src, _ = length.decode(src) + sum := int(length.sum()) + length.decode(src) + return sum + int(length.sum()) +} + +func (e *ByteArrayEncoding) wrap(err error) error { + if err != nil { + err = encoding.Error(e, err) + } + return err +} + +func (e *ByteArrayEncoding) wrapf(msg string, args ...any) error { + return encoding.Errorf(e, msg, args...) +} + +func linearSearchPrefixLength(base, data []byte) (n int) { + for n < len(base) && n < len(data) && base[n] == data[n] { + n++ + } + return n +} + +func binarySearchPrefixLength(base, data []byte) int { + n := min(len(base), len(data)) + return sort.Search(n, func(i int) bool { + return !bytes.Equal(base[:i+1], data[:i+1]) + }) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.go new file mode 100644 index 0000000000..71a077ee88 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.go @@ -0,0 +1,164 @@ +//go:build !purego + +package delta + +import ( + "golang.org/x/sys/cpu" +) + +//go:noescape +func validatePrefixAndSuffixLengthValuesAVX2(prefix, suffix []int32, maxLength int) (totalPrefixLength, totalSuffixLength int, ok bool) + +func validatePrefixAndSuffixLengthValues(prefix, suffix []int32, maxLength int) (totalPrefixLength, totalSuffixLength int, err error) { + if cpu.X86.HasAVX2 { + totalPrefixLength, totalSuffixLength, ok := validatePrefixAndSuffixLengthValuesAVX2(prefix, suffix, maxLength) + if ok { + return totalPrefixLength, totalSuffixLength, nil + } + } + + lastValueLength := 0 + + for i := range prefix { + p := int(prefix[i]) + n := int(suffix[i]) + if p < 0 { + err = errInvalidNegativePrefixLength(p) + return + } + if n < 0 { + err = errInvalidNegativeValueLength(n) + return + } + if p > lastValueLength { + err = errPrefixLengthOutOfBounds(p, lastValueLength) + return + } + totalPrefixLength += p + totalSuffixLength += n + lastValueLength = p + n + } + + if totalSuffixLength > maxLength { + err = errValueLengthOutOfBounds(totalSuffixLength, maxLength) + return + } + + return totalPrefixLength, totalSuffixLength, nil +} + +//go:noescape +func decodeByteArrayOffsets(offsets []uint32, prefix, suffix []int32) + +//go:noescape +func decodeByteArrayAVX2(dst, src []byte, prefix, suffix []int32) int + +func decodeByteArray(dst, src []byte, prefix, suffix []int32, offsets []uint32) ([]byte, []uint32, error) { + totalPrefixLength, totalSuffixLength, err := validatePrefixAndSuffixLengthValues(prefix, suffix, len(src)) + if err != nil { + return dst, offsets, err + } + + totalLength := totalPrefixLength + totalSuffixLength + dst = resizeNoMemclr(dst, totalLength+padding) + + if size := len(prefix) + 1; cap(offsets) < size { + offsets = make([]uint32, size) + } else { + offsets = offsets[:size] + } + + _ = prefix[:len(suffix)] + _ = suffix[:len(prefix)] + decodeByteArrayOffsets(offsets, prefix, suffix) + + var lastValue []byte + var i int + var j int + + if cpu.X86.HasAVX2 && len(src) > padding { + k := len(suffix) + n := 0 + + for k > 0 && n < padding { + k-- + n += int(suffix[k]) + } + + if k > 0 && n >= padding { + i = decodeByteArrayAVX2(dst, src, prefix[:k], suffix[:k]) + j = len(src) - n + lastValue = dst[i-(int(prefix[k-1])+int(suffix[k-1])):] + prefix = prefix[k:] + suffix = suffix[k:] + } + } + + for k := range prefix { + p := int(prefix[k]) + n := int(suffix[k]) + lastValueOffset := i + i += copy(dst[i:], lastValue[:p]) + i += copy(dst[i:], src[j:j+n]) + j += n + lastValue = dst[lastValueOffset:] + } + + return dst[:totalLength], offsets, nil +} + +//go:noescape +func decodeByteArrayAVX2x128bits(dst, src []byte, prefix, suffix []int32) int + +func decodeFixedLenByteArray(dst, src []byte, size int, prefix, suffix []int32) ([]byte, error) { + totalPrefixLength, totalSuffixLength, err := validatePrefixAndSuffixLengthValues(prefix, suffix, len(src)) + if err != nil { + return dst, err + } + + totalLength := totalPrefixLength + totalSuffixLength + dst = resizeNoMemclr(dst, totalLength+padding) + + _ = prefix[:len(suffix)] + _ = suffix[:len(prefix)] + + var lastValue []byte + var i int + var j int + + if cpu.X86.HasAVX2 && len(src) > padding { + k := len(suffix) + n := 0 + + for k > 0 && n < padding { + k-- + n += int(suffix[k]) + } + + if k > 0 && n >= padding { + if size == 16 { + i = decodeByteArrayAVX2x128bits(dst, src, prefix[:k], suffix[:k]) + } else { + i = decodeByteArrayAVX2(dst, src, prefix[:k], suffix[:k]) + } + j = len(src) - n + prefix = prefix[k:] + suffix = suffix[k:] + if i >= size { + lastValue = dst[i-size:] + } + } + } + + for k := range prefix { + p := int(prefix[k]) + n := int(suffix[k]) + k := i + i += copy(dst[i:], lastValue[:p]) + i += copy(dst[i:], src[j:j+n]) + j += n + lastValue = dst[k:] + } + + return dst[:totalLength], nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.s b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.s new file mode 100644 index 0000000000..b8b7098321 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_amd64.s @@ -0,0 +1,243 @@ +//go:build !purego + +#include "funcdata.h" +#include "textflag.h" + +// func validatePrefixAndSuffixLengthValuesAVX2(prefix, suffix []int32, maxLength int) (totalPrefixLength, totalSuffixLength int, ok bool) +TEXT ยทvalidatePrefixAndSuffixLengthValuesAVX2(SB), NOSPLIT, $0-73 + MOVQ prefix_base+0(FP), AX + MOVQ suffix_base+24(FP), BX + MOVQ suffix_len+32(FP), CX + MOVQ maxLength+48(FP), DX + + XORQ SI, SI + XORQ DI, DI // lastValueLength + XORQ R8, R8 + XORQ R9, R9 + XORQ R10, R10 // totalPrefixLength + XORQ R11, R11 // totalSuffixLength + XORQ R12, R12 // ok + + CMPQ CX, $8 + JB test + + MOVQ CX, R13 + SHRQ $3, R13 + SHLQ $3, R13 + + VPXOR X0, X0, X0 // lastValueLengths + VPXOR X1, X1, X1 // totalPrefixLengths + VPXOR X2, X2, X2 // totalSuffixLengths + VPXOR X3, X3, X3 // negative prefix length sentinels + VPXOR X4, X4, X4 // negative suffix length sentinels + VPXOR X5, X5, X5 // prefix length overflow sentinels + VMOVDQU ยทrotateLeft32(SB), Y6 + +loopAVX2: + VMOVDQU (AX)(SI*4), Y7 // p + VMOVDQU (BX)(SI*4), Y8 // n + + VPADDD Y7, Y1, Y1 + VPADDD Y8, Y2, Y2 + + VPOR Y7, Y3, Y3 + VPOR Y8, Y4, Y4 + + VPADDD Y7, Y8, Y9 // p + n + VPERMD Y0, Y6, Y10 + VPBLENDD $1, Y10, Y9, Y10 + VPCMPGTD Y10, Y7, Y10 + VPOR Y10, Y5, Y5 + + VMOVDQU Y9, Y0 + ADDQ $8, SI + CMPQ SI, R13 + JNE loopAVX2 + + // If any of the sentinel values has its most significant bit set then one + // of the values was negative or one of the prefixes was greater than the + // length of the previous value, return false. + VPOR Y4, Y3, Y3 + VPOR Y5, Y3, Y3 + VMOVMSKPS Y3, R13 + CMPQ R13, $0 + JNE done + + // We computed 8 sums in parallel for the prefix and suffix arrays, they + // need to be accumulated into single values, which is what these reduction + // steps do. + VPSRLDQ $4, Y1, Y5 + VPSRLDQ $8, Y1, Y6 + VPSRLDQ $12, Y1, Y7 + VPADDD Y5, Y1, Y1 + VPADDD Y6, Y1, Y1 + VPADDD Y7, Y1, Y1 + VPERM2I128 $1, Y1, Y1, Y0 + VPADDD Y0, Y1, Y1 + MOVQ X1, R10 + ANDQ $0x7FFFFFFF, R10 + + VPSRLDQ $4, Y2, Y5 + VPSRLDQ $8, Y2, Y6 + VPSRLDQ $12, Y2, Y7 + VPADDD Y5, Y2, Y2 + VPADDD Y6, Y2, Y2 + VPADDD Y7, Y2, Y2 + VPERM2I128 $1, Y2, Y2, Y0 + VPADDD Y0, Y2, Y2 + MOVQ X2, R11 + ANDQ $0x7FFFFFFF, R11 + + JMP test +loop: + MOVLQSX (AX)(SI*4), R8 + MOVLQSX (BX)(SI*4), R9 + + CMPQ R8, $0 // p < 0 ? + JL done + + CMPQ R9, $0 // n < 0 ? + JL done + + CMPQ R8, DI // p > lastValueLength ? + JG done + + ADDQ R8, R10 + ADDQ R9, R11 + ADDQ R8, DI + ADDQ R9, DI + + INCQ SI +test: + CMPQ SI, CX + JNE loop + + CMPQ R11, DX // totalSuffixLength > maxLength ? + JG done + + MOVB $1, R12 +done: + MOVQ R10, totalPrefixLength+56(FP) + MOVQ R11, totalSuffixLength+64(FP) + MOVB R12, ok+72(FP) + RET + +// func decodeByteArrayOffsets(offsets []uint32, prefix, suffix []int32) +TEXT ยทdecodeByteArrayOffsets(SB), NOSPLIT, $0-72 + MOVQ offsets_base+0(FP), AX + MOVQ prefix_base+24(FP), BX + MOVQ suffix_base+48(FP), CX + MOVQ suffix_len+56(FP), DX + + XORQ SI, SI + XORQ R10, R10 + JMP test +loop: + MOVL (BX)(SI*4), R8 + MOVL (CX)(SI*4), R9 + MOVL R10, (AX)(SI*4) + ADDL R8, R10 + ADDL R9, R10 + INCQ SI +test: + CMPQ SI, DX + JNE loop + MOVL R10, (AX)(SI*4) + RET + +// func decodeByteArrayAVX2(dst, src []byte, prefix, suffix []int32) int +TEXT ยทdecodeByteArrayAVX2(SB), NOSPLIT, $0-104 + MOVQ dst_base+0(FP), AX + MOVQ src_base+24(FP), BX + MOVQ prefix_base+48(FP), CX + MOVQ suffix_base+72(FP), DX + MOVQ suffix_len+80(FP), DI + + XORQ SI, SI + XORQ R8, R8 + XORQ R9, R9 + MOVQ AX, R10 // last value + + JMP test +loop: + MOVLQZX (CX)(SI*4), R8 // prefix length + MOVLQZX (DX)(SI*4), R9 // suffix length +prefix: + VMOVDQU (R10), Y0 + VMOVDQU Y0, (AX) + CMPQ R8, $32 + JA copyPrefix +suffix: + VMOVDQU (BX), Y1 + VMOVDQU Y1, (AX)(R8*1) + CMPQ R9, $32 + JA copySuffix +next: + MOVQ AX, R10 + ADDQ R9, R8 + LEAQ (AX)(R8*1), AX + LEAQ (BX)(R9*1), BX + INCQ SI +test: + CMPQ SI, DI + JNE loop + MOVQ dst_base+0(FP), BX + SUBQ BX, AX + MOVQ AX, ret+96(FP) + VZEROUPPER + RET +copyPrefix: + MOVQ $32, R12 +copyPrefixLoop: + VMOVDQU (R10)(R12*1), Y0 + VMOVDQU Y0, (AX)(R12*1) + ADDQ $32, R12 + CMPQ R12, R8 + JB copyPrefixLoop + JMP suffix +copySuffix: + MOVQ $32, R12 + LEAQ (AX)(R8*1), R13 +copySuffixLoop: + VMOVDQU (BX)(R12*1), Y1 + VMOVDQU Y1, (R13)(R12*1) + ADDQ $32, R12 + CMPQ R12, R9 + JB copySuffixLoop + JMP next + +// func decodeByteArrayAVX2x128bits(dst, src []byte, prefix, suffix []int32) int +TEXT ยทdecodeByteArrayAVX2x128bits(SB), NOSPLIT, $0-104 + MOVQ dst_base+0(FP), AX + MOVQ src_base+24(FP), BX + MOVQ prefix_base+48(FP), CX + MOVQ suffix_base+72(FP), DX + MOVQ suffix_len+80(FP), DI + + XORQ SI, SI + XORQ R8, R8 + XORQ R9, R9 + VPXOR X0, X0, X0 + + JMP test +loop: + MOVLQZX (CX)(SI*4), R8 // prefix length + MOVLQZX (DX)(SI*4), R9 // suffix length + + VMOVDQU (BX), X1 + VMOVDQU X0, (AX) + VMOVDQU X1, (AX)(R8*1) + VMOVDQU (AX), X0 + + ADDQ R9, R8 + LEAQ (AX)(R8*1), AX + LEAQ (BX)(R9*1), BX + INCQ SI +test: + CMPQ SI, DI + JNE loop + MOVQ dst_base+0(FP), BX + SUBQ BX, AX + MOVQ AX, ret+96(FP) + VZEROUPPER + RET diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_purego.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_purego.go new file mode 100644 index 0000000000..972c1feca2 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/byte_array_purego.go @@ -0,0 +1,63 @@ +//go:build purego || !amd64 + +package delta + +func decodeByteArray(dst, src []byte, prefix, suffix []int32, offsets []uint32) ([]byte, []uint32, error) { + _ = prefix[:len(suffix)] + _ = suffix[:len(prefix)] + + var lastValue []byte + for i := range suffix { + n := int(suffix[i]) + p := int(prefix[i]) + if n < 0 { + return dst, offsets, errInvalidNegativeValueLength(n) + } + if n > len(src) { + return dst, offsets, errValueLengthOutOfBounds(n, len(src)) + } + if p < 0 { + return dst, offsets, errInvalidNegativePrefixLength(p) + } + if p > len(lastValue) { + return dst, offsets, errPrefixLengthOutOfBounds(p, len(lastValue)) + } + j := len(dst) + offsets = append(offsets, uint32(j)) + dst = append(dst, lastValue[:p]...) + dst = append(dst, src[:n]...) + lastValue = dst[j:] + src = src[n:] + } + + return dst, append(offsets, uint32(len(dst))), nil +} + +func decodeFixedLenByteArray(dst, src []byte, size int, prefix, suffix []int32) ([]byte, error) { + _ = prefix[:len(suffix)] + _ = suffix[:len(prefix)] + + var lastValue []byte + for i := range suffix { + n := int(suffix[i]) + p := int(prefix[i]) + if n < 0 { + return dst, errInvalidNegativeValueLength(n) + } + if n > len(src) { + return dst, errValueLengthOutOfBounds(n, len(src)) + } + if p < 0 { + return dst, errInvalidNegativePrefixLength(p) + } + if p > len(lastValue) { + return dst, errPrefixLengthOutOfBounds(p, len(lastValue)) + } + j := len(dst) + dst = append(dst, lastValue[:p]...) + dst = append(dst, src[:n]...) + lastValue = dst[j:] + src = src[n:] + } + return dst, nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta.go new file mode 100644 index 0000000000..3549a3270f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta.go @@ -0,0 +1,100 @@ +package delta + +import ( + "fmt" + "sync" + + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type int32Buffer struct { + values []int32 +} + +func (buf *int32Buffer) resize(size int) { + if cap(buf.values) < size { + buf.values = make([]int32, size, 2*size) + } else { + buf.values = buf.values[:size] + } +} + +func (buf *int32Buffer) decode(src []byte) ([]byte, error) { + values, remain, err := decodeInt32(unsafecast.Slice[byte](buf.values[:0]), src) + buf.values = unsafecast.Slice[int32](values) + return remain, err +} + +func (buf *int32Buffer) sum() (sum int32) { + for _, v := range buf.values { + sum += v + } + return sum +} + +var ( + int32BufferPool sync.Pool // *int32Buffer +) + +func getInt32Buffer() *int32Buffer { + b, _ := int32BufferPool.Get().(*int32Buffer) + if b != nil { + b.values = b.values[:0] + } else { + b = &int32Buffer{ + values: make([]int32, 0, 1024), + } + } + return b +} + +func putInt32Buffer(b *int32Buffer) { + int32BufferPool.Put(b) +} + +func resizeNoMemclr(buf []byte, size int) []byte { + if cap(buf) < size { + return grow(buf, size) + } + return buf[:size] +} + +func resize(buf []byte, size int) []byte { + if cap(buf) < size { + return grow(buf, size) + } + if size > len(buf) { + clear := buf[len(buf):size] + for i := range clear { + clear[i] = 0 + } + } + return buf[:size] +} + +func grow(buf []byte, size int) []byte { + newCap := max(2*cap(buf), size) + newBuf := make([]byte, size, newCap) + copy(newBuf, buf) + return newBuf +} + +func errPrefixAndSuffixLengthMismatch(prefixLength, suffixLength int) error { + return fmt.Errorf("length of prefix and suffix mismatch: %d != %d", prefixLength, suffixLength) +} + +func errInvalidNegativeValueLength(length int) error { + return fmt.Errorf("invalid negative value length: %d", length) +} + +func errInvalidNegativePrefixLength(length int) error { + return fmt.Errorf("invalid negative prefix length: %d", length) +} + +func errValueLengthOutOfBounds(length, maxLength int) error { + return fmt.Errorf("value length is larger than the input size: %d > %d", length, maxLength) +} + +func errPrefixLengthOutOfBounds(length, maxLength int) error { + return fmt.Errorf("prefix length %d is larger than the last value of size %d", length, maxLength) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.go new file mode 100644 index 0000000000..864aeac136 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.go @@ -0,0 +1,16 @@ +//go:build !purego + +package delta + +const ( + padding = 64 +) + +func findNegativeLength(lengths []int32) int { + for _, n := range lengths { + if n < 0 { + return int(n) + } + } + return -1 +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.s b/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.s new file mode 100644 index 0000000000..e8748a263d --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/delta_amd64.s @@ -0,0 +1,13 @@ +//go:build !purego + +#include "textflag.h" + +GLOBL ยทrotateLeft32(SB), RODATA|NOPTR, $32 +DATA ยทrotateLeft32+0(SB)/4, $7 +DATA ยทrotateLeft32+4(SB)/4, $0 +DATA ยทrotateLeft32+8(SB)/4, $1 +DATA ยทrotateLeft32+12(SB)/4, $2 +DATA ยทrotateLeft32+16(SB)/4, $3 +DATA ยทrotateLeft32+20(SB)/4, $4 +DATA ยทrotateLeft32+24(SB)/4, $5 +DATA ยทrotateLeft32+28(SB)/4, $6 diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array.go new file mode 100644 index 0000000000..65ed6f79be --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array.go @@ -0,0 +1,81 @@ +package delta + +import ( + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" +) + +type LengthByteArrayEncoding struct { + encoding.NotSupported +} + +func (e *LengthByteArrayEncoding) String() string { + return "DELTA_LENGTH_BYTE_ARRAY" +} + +func (e *LengthByteArrayEncoding) Encoding() format.Encoding { + return format.DeltaLengthByteArray +} + +func (e *LengthByteArrayEncoding) EncodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, error) { + if len(offsets) == 0 { + return dst[:0], nil + } + + length := getInt32Buffer() + defer putInt32Buffer(length) + + length.resize(len(offsets) - 1) + encodeByteArrayLengths(length.values, offsets) + + dst = dst[:0] + dst = encodeInt32(dst, length.values) + dst = append(dst, src...) + return dst, nil +} + +func (e *LengthByteArrayEncoding) DecodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, []uint32, error) { + dst, offsets = dst[:0], offsets[:0] + + length := getInt32Buffer() + defer putInt32Buffer(length) + + src, err := length.decode(src) + if err != nil { + return dst, offsets, e.wrap(err) + } + + if size := len(length.values) + 1; cap(offsets) < size { + offsets = make([]uint32, size, 2*size) + } else { + offsets = offsets[:size] + } + + lastOffset, invalidLength := decodeByteArrayLengths(offsets, length.values) + if invalidLength != 0 { + return dst, offsets, e.wrap(errInvalidNegativeValueLength(int(invalidLength))) + } + if int(lastOffset) > len(src) { + return dst, offsets, e.wrap(errValueLengthOutOfBounds(int(lastOffset), len(src))) + } + + return append(dst, src[:lastOffset]...), offsets, nil +} + +func (e *LengthByteArrayEncoding) EstimateDecodeByteArraySize(src []byte) int { + length := getInt32Buffer() + defer putInt32Buffer(length) + length.decode(src) + return int(length.sum()) +} + +func (e *LengthByteArrayEncoding) CanDecodeInPlace() bool { + return true +} + +func (e *LengthByteArrayEncoding) wrap(err error) error { + if err != nil { + err = encoding.Error(e, err) + } + return err +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.go new file mode 100644 index 0000000000..905e8516ee --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.go @@ -0,0 +1,9 @@ +//go:build !purego + +package delta + +//go:noescape +func encodeByteArrayLengths(lengths []int32, offsets []uint32) + +//go:noescape +func decodeByteArrayLengths(offsets []uint32, lengths []int32) (lastOffset uint32, invalidLength int32) diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.s b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.s new file mode 100644 index 0000000000..bc6292e2a1 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_amd64.s @@ -0,0 +1,122 @@ +//go:build !purego + +#include "textflag.h" + +// func encodeByteArrayLengths(lengths []int32, offsets []uint32) +TEXT ยทencodeByteArrayLengths(SB), NOSPLIT, $0-48 + MOVQ lengths_base+0(FP), AX + MOVQ lengths_len+8(FP), CX + MOVQ offsets_base+24(FP), BX + XORQ SI, SI + + CMPQ CX, $4 + JB test + + MOVQ CX, DX + SHRQ $2, DX + SHLQ $2, DX + +loopSSE2: + MOVOU 0(BX)(SI*4), X0 + MOVOU 4(BX)(SI*4), X1 + PSUBL X0, X1 + MOVOU X1, (AX)(SI*4) + ADDQ $4, SI + CMPQ SI, DX + JNE loopSSE2 + JMP test +loop: + MOVL 0(BX)(SI*4), R8 + MOVL 4(BX)(SI*4), R9 + SUBL R8, R9 + MOVL R9, (AX)(SI*4) + INCQ SI +test: + CMPQ SI, CX + JNE loop + RET + +// func decodeByteArrayLengths(offsets []uint32, length []int32) (lastOffset uint32, invalidLength int32) +TEXT ยทdecodeByteArrayLengths(SB), NOSPLIT, $0-56 + MOVQ offsets_base+0(FP), AX + MOVQ lengths_base+24(FP), BX + MOVQ lengths_len+32(FP), CX + + XORQ DX, DX // lastOffset + XORQ DI, DI // invalidLength + XORQ SI, SI + + CMPQ CX, $4 + JL test + + MOVQ CX, R8 + SHRQ $2, R8 + SHLQ $2, R8 + + MOVL $0, (AX) + PXOR X0, X0 + PXOR X3, X3 + // This loop computes the prefix sum of the lengths array in order to + // generate values of the offsets array. + // + // We stick to SSE2 to keep the code simple (the Go compiler appears to + // assume that SSE2 must be supported on AMD64) which already yields most + // of the performance that we could get on this subroutine if we were using + // AVX2. + // + // The X3 register also accumulates a mask of all length values, which is + // checked after the loop to determine whether any of the lengths were + // negative. + // + // The following article contains a description of the prefix sum algorithm + // used in this function: https://en.algorithmica.org/hpc/algorithms/prefix/ +loopSSE2: + MOVOU (BX)(SI*4), X1 + POR X1, X3 + + MOVOA X1, X2 + PSLLDQ $4, X2 + PADDD X2, X1 + + MOVOA X1, X2 + PSLLDQ $8, X2 + PADDD X2, X1 + + PADDD X1, X0 + MOVOU X0, 4(AX)(SI*4) + + PSHUFD $0b11111111, X0, X0 + + ADDQ $4, SI + CMPQ SI, R8 + JNE loopSSE2 + + // If any of the most significant bits of double words in the X3 register + // are set to 1, it indicates that one of the lengths was negative and + // therefore the prefix sum is invalid. + // + // TODO: we report the invalid length as -1, effectively losing the original + // value due to the aggregation within X3. This is something that we might + // want to address in the future to provide better error reporting. + MOVMSKPS X3, R8 + MOVL $-1, R9 + CMPL R8, $0 + CMOVLNE R9, DI + + MOVQ X0, DX + JMP test +loop: + MOVL (BX)(SI*4), R8 + MOVL DX, (AX)(SI*4) + ADDL R8, DX + CMPL R8, $0 + CMOVLLT R8, DI + INCQ SI +test: + CMPQ SI, CX + JNE loop + + MOVL DX, (AX)(SI*4) + MOVL DX, lastOffset+48(FP) + MOVL DI, invalidLength+52(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_purego.go b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_purego.go new file mode 100644 index 0000000000..0c0fb6baeb --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/delta/length_byte_array_purego.go @@ -0,0 +1,24 @@ +//go:build purego || !amd64 + +package delta + +func encodeByteArrayLengths(lengths []int32, offsets []uint32) { + for i := range lengths { + lengths[i] = int32(offsets[i+1] - offsets[i]) + } +} + +func decodeByteArrayLengths(offsets []uint32, lengths []int32) (uint32, int32) { + lastOffset := uint32(0) + + for i, n := range lengths { + if n < 0 { + return lastOffset, n + } + offsets[i] = lastOffset + lastOffset += uint32(n) + } + + offsets[len(lengths)] = lastOffset + return lastOffset, 0 +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/encoding.go b/vendor/github.com/parquet-go/parquet-go/encoding/encoding.go new file mode 100644 index 0000000000..a919f59172 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/encoding.go @@ -0,0 +1,72 @@ +// Package encoding provides the generic APIs implemented by parquet encodings +// in its sub-packages. +package encoding + +import ( + "math" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/format" +) + +const ( + MaxFixedLenByteArraySize = math.MaxInt16 +) + +// The Encoding interface is implemented by types representing parquet column +// encodings. +// +// Encoding instances must be safe to use concurrently from multiple goroutines. +type Encoding interface { + // Returns a human-readable name for the encoding. + String() string + + // Returns the parquet code representing the encoding. + Encoding() format.Encoding + + // Encode methods serialize the source sequence of values into the + // destination buffer, potentially reallocating it if it was too short to + // contain the output. + // + // The methods panic if the type of src values differ from the type of + // values being encoded. + EncodeLevels(dst []byte, src []uint8) ([]byte, error) + EncodeBoolean(dst []byte, src []byte) ([]byte, error) + EncodeInt32(dst []byte, src []int32) ([]byte, error) + EncodeInt64(dst []byte, src []int64) ([]byte, error) + EncodeInt96(dst []byte, src []deprecated.Int96) ([]byte, error) + EncodeFloat(dst []byte, src []float32) ([]byte, error) + EncodeDouble(dst []byte, src []float64) ([]byte, error) + EncodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, error) + EncodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) + + // Decode methods deserialize from the source buffer into the destination + // slice, potentially growing it if it was too short to contain the result. + // + // The methods panic if the type of dst values differ from the type of + // values being decoded. + DecodeLevels(dst []uint8, src []byte) ([]uint8, error) + DecodeBoolean(dst []byte, src []byte) ([]byte, error) + DecodeInt32(dst []int32, src []byte) ([]int32, error) + DecodeInt64(dst []int64, src []byte) ([]int64, error) + DecodeInt96(dst []deprecated.Int96, src []byte) ([]deprecated.Int96, error) + DecodeFloat(dst []float32, src []byte) ([]float32, error) + DecodeDouble(dst []float64, src []byte) ([]float64, error) + DecodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, []uint32, error) + DecodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) + + // Computes an estimation of the output size of decoding the encoded page + // of values passed as argument. + // + // Note that this is an estimate, it is useful to preallocate the output + // buffer that will be passed to the decode method, but the actual output + // size may be different. + // + // The estimate never errors since it is not intended to be used as an + // input validation method. + EstimateDecodeByteArraySize(src []byte) int + + // When this method returns true, the encoding supports receiving the same + // buffer as source and destination. + CanDecodeInPlace() bool +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/notsupported.go b/vendor/github.com/parquet-go/parquet-go/encoding/notsupported.go new file mode 100644 index 0000000000..59215330d1 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/notsupported.go @@ -0,0 +1,213 @@ +package encoding + +import ( + "errors" + "fmt" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/format" +) + +var ( + // ErrNotSupported is an error returned when the underlying encoding does + // not support the type of values being encoded or decoded. + // + // This error may be wrapped with type information, applications must use + // errors.Is rather than equality comparisons to test the error values + // returned by encoders and decoders. + ErrNotSupported = errors.New("encoding not supported") + + // ErrInvalidArgument is an error returned one or more arguments passed to + // the encoding functions are incorrect. + // + // As with ErrNotSupported, this error may be wrapped with specific + // information about the problem and applications are expected to use + // errors.Is for comparisons. + ErrInvalidArgument = errors.New("invalid argument") +) + +// Error constructs an error which wraps err and indicates that it originated +// from the given encoding. +func Error(e Encoding, err error) error { + return fmt.Errorf("%s: %w", e, err) +} + +// Errorf is like Error but constructs the error message from the given format +// and arguments. +func Errorf(e Encoding, msg string, args ...any) error { + return Error(e, fmt.Errorf(msg, args...)) +} + +// ErrEncodeInvalidInputSize constructs an error indicating that encoding failed +// due to the size of the input. +func ErrEncodeInvalidInputSize(e Encoding, typ string, size int) error { + return errInvalidInputSize(e, "encode", typ, size) +} + +// ErrDecodeInvalidInputSize constructs an error indicating that decoding failed +// due to the size of the input. +func ErrDecodeInvalidInputSize(e Encoding, typ string, size int) error { + return errInvalidInputSize(e, "decode", typ, size) +} + +func errInvalidInputSize(e Encoding, op, typ string, size int) error { + return Errorf(e, "cannot %s %s from input of size %d: %w", op, typ, size, ErrInvalidArgument) +} + +// CanEncodeInt8 reports whether e can encode LEVELS values. +func CanEncodeLevels(e Encoding) bool { + _, err := e.EncodeLevels(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeBoolean reports whether e can encode BOOLEAN values. +func CanEncodeBoolean(e Encoding) bool { + _, err := e.EncodeBoolean(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeInt32 reports whether e can encode INT32 values. +func CanEncodeInt32(e Encoding) bool { + _, err := e.EncodeInt32(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeInt64 reports whether e can encode INT64 values. +func CanEncodeInt64(e Encoding) bool { + _, err := e.EncodeInt64(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeInt96 reports whether e can encode INT96 values. +func CanEncodeInt96(e Encoding) bool { + _, err := e.EncodeInt96(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeFloat reports whether e can encode FLOAT values. +func CanEncodeFloat(e Encoding) bool { + _, err := e.EncodeFloat(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeDouble reports whether e can encode DOUBLE values. +func CanEncodeDouble(e Encoding) bool { + _, err := e.EncodeDouble(nil, nil) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeByteArray reports whether e can encode BYTE_ARRAY values. +func CanEncodeByteArray(e Encoding) bool { + _, err := e.EncodeByteArray(nil, nil, zeroOffsets[:]) + return !errors.Is(err, ErrNotSupported) +} + +// CanEncodeFixedLenByteArray reports whether e can encode +// FIXED_LEN_BYTE_ARRAY values. +func CanEncodeFixedLenByteArray(e Encoding) bool { + _, err := e.EncodeFixedLenByteArray(nil, nil, 1) + return !errors.Is(err, ErrNotSupported) +} + +var zeroOffsets [1]uint32 + +// NotSupported is a type satisfying the Encoding interface which does not +// support encoding nor decoding any value types. +type NotSupported struct { +} + +func (NotSupported) String() string { + return "NOT_SUPPORTED" +} + +func (NotSupported) Encoding() format.Encoding { + return -1 +} + +func (NotSupported) EncodeLevels(dst []byte, src []uint8) ([]byte, error) { + return dst[:0], errNotSupported("LEVELS") +} + +func (NotSupported) EncodeBoolean(dst []byte, src []byte) ([]byte, error) { + return dst[:0], errNotSupported("BOOLEAN") +} + +func (NotSupported) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + return dst[:0], errNotSupported("INT32") +} + +func (NotSupported) EncodeInt64(dst []byte, src []int64) ([]byte, error) { + return dst[:0], errNotSupported("INT64") +} + +func (NotSupported) EncodeInt96(dst []byte, src []deprecated.Int96) ([]byte, error) { + return dst[:0], errNotSupported("INT96") +} + +func (NotSupported) EncodeFloat(dst []byte, src []float32) ([]byte, error) { + return dst[:0], errNotSupported("FLOAT") +} + +func (NotSupported) EncodeDouble(dst []byte, src []float64) ([]byte, error) { + return dst[:0], errNotSupported("DOUBLE") +} + +func (NotSupported) EncodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, error) { + return dst[:0], errNotSupported("BYTE_ARRAY") +} + +func (NotSupported) EncodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + return dst[:0], errNotSupported("FIXED_LEN_BYTE_ARRAY") +} + +func (NotSupported) DecodeLevels(dst []uint8, src []byte) ([]uint8, error) { + return dst, errNotSupported("LEVELS") +} + +func (NotSupported) DecodeBoolean(dst []byte, src []byte) ([]byte, error) { + return dst, errNotSupported("BOOLEAN") +} + +func (NotSupported) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + return dst, errNotSupported("INT32") +} + +func (NotSupported) DecodeInt64(dst []int64, src []byte) ([]int64, error) { + return dst, errNotSupported("INT64") +} + +func (NotSupported) DecodeInt96(dst []deprecated.Int96, src []byte) ([]deprecated.Int96, error) { + return dst, errNotSupported("INT96") +} + +func (NotSupported) DecodeFloat(dst []float32, src []byte) ([]float32, error) { + return dst, errNotSupported("FLOAT") +} + +func (NotSupported) DecodeDouble(dst []float64, src []byte) ([]float64, error) { + return dst, errNotSupported("DOUBLE") +} + +func (NotSupported) DecodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, []uint32, error) { + return dst, offsets, errNotSupported("BYTE_ARRAY") +} + +func (NotSupported) DecodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + return dst, errNotSupported("FIXED_LEN_BYTE_ARRAY") +} + +func (NotSupported) EstimateDecodeByteArraySize(src []byte) int { + return 0 +} + +func (NotSupported) CanDecodeInPlace() bool { + return false +} + +func errNotSupported(typ string) error { + return fmt.Errorf("%w for type %s", ErrNotSupported, typ) +} + +var ( + _ Encoding = NotSupported{} +) diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/plain/dictionary.go b/vendor/github.com/parquet-go/parquet-go/encoding/plain/dictionary.go new file mode 100644 index 0000000000..4946a79293 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/plain/dictionary.go @@ -0,0 +1,27 @@ +package plain + +import ( + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" +) + +type DictionaryEncoding struct { + encoding.NotSupported + plain Encoding +} + +func (e *DictionaryEncoding) String() string { + return "PLAIN_DICTIONARY" +} + +func (e *DictionaryEncoding) Encoding() format.Encoding { + return format.PlainDictionary +} + +func (e *DictionaryEncoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + return e.plain.EncodeInt32(dst, src) +} + +func (e *DictionaryEncoding) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + return e.plain.DecodeInt32(dst, src) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain.go b/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain.go new file mode 100644 index 0000000000..690bc81555 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain.go @@ -0,0 +1,246 @@ +// Package plain implements the PLAIN parquet encoding. +// +// https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 +package plain + +import ( + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +const ( + ByteArrayLengthSize = 4 + MaxByteArrayLength = math.MaxInt32 +) + +type Encoding struct { + encoding.NotSupported +} + +func (e *Encoding) String() string { + return "PLAIN" +} + +func (e *Encoding) Encoding() format.Encoding { + return format.Plain +} + +func (e *Encoding) EncodeBoolean(dst []byte, src []byte) ([]byte, error) { + return append(dst[:0], src...), nil +} + +func (e *Encoding) EncodeInt96(dst []byte, src []deprecated.Int96) ([]byte, error) { + return append(dst[:0], unsafecast.Slice[byte](src)...), nil +} + +func (e *Encoding) EncodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, error) { + dst = dst[:0] + + if len(offsets) > 0 { + baseOffset := offsets[0] + + for _, endOffset := range offsets[1:] { + dst = AppendByteArray(dst, src[baseOffset:endOffset:endOffset]) + baseOffset = endOffset + } + } + + return dst, nil +} + +func (e *Encoding) EncodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + if size < 0 || size > encoding.MaxFixedLenByteArraySize { + return dst[:0], encoding.Error(e, encoding.ErrInvalidArgument) + } + return append(dst[:0], src...), nil +} + +func (e *Encoding) DecodeBoolean(dst []byte, src []byte) ([]byte, error) { + return append(dst[:0], src...), nil +} + +func (e *Encoding) DecodeInt96(dst []deprecated.Int96, src []byte) ([]deprecated.Int96, error) { + if (len(src) % 12) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "INT96", len(src)) + } + return append(dst[:0], unsafecast.Slice[deprecated.Int96](src)...), nil +} + +func (e *Encoding) DecodeByteArray(dst []byte, src []byte, offsets []uint32) ([]byte, []uint32, error) { + dst, offsets = dst[:0], offsets[:0] + + for i := 0; i < len(src); { + if (len(src) - i) < ByteArrayLengthSize { + return dst, offsets, ErrTooShort(len(src)) + } + n := ByteArrayLength(src[i:]) + if n > (len(src) - ByteArrayLengthSize) { + return dst, offsets, ErrTooShort(len(src)) + } + i += ByteArrayLengthSize + offsets = append(offsets, uint32(len(dst))) + dst = append(dst, src[i:i+n]...) + i += n + } + + return dst, append(offsets, uint32(len(dst))), nil +} + +func (e *Encoding) DecodeFixedLenByteArray(dst []byte, src []byte, size int) ([]byte, error) { + if size < 0 || size > encoding.MaxFixedLenByteArraySize { + return dst, encoding.Error(e, encoding.ErrInvalidArgument) + } + if (len(src) % size) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "FIXED_LEN_BYTE_ARRAY", len(src)) + } + return append(dst[:0], src...), nil +} + +func (e *Encoding) EstimateDecodeByteArraySize(src []byte) int { + return len(src) +} + +func (e *Encoding) CanDecodeInPlace() bool { + return true +} + +func Boolean(v bool) []byte { return AppendBoolean(nil, 0, v) } + +func Int32(v int32) []byte { return AppendInt32(nil, v) } + +func Int64(v int64) []byte { return AppendInt64(nil, v) } + +func Int96(v deprecated.Int96) []byte { return AppendInt96(nil, v) } + +func Float(v float32) []byte { return AppendFloat(nil, v) } + +func Double(v float64) []byte { return AppendDouble(nil, v) } + +func ByteArray(v []byte) []byte { return AppendByteArray(nil, v) } + +func AppendBoolean(b []byte, n int, v bool) []byte { + i := n / 8 + j := n % 8 + + if cap(b) > i { + b = b[:i+1] + } else { + tmp := make([]byte, i+1, 2*(i+1)) + copy(tmp, b) + b = tmp + } + + k := uint(j) + x := byte(0) + if v { + x = 1 + } + + b[i] = (b[i] & ^(1 << k)) | (x << k) + return b +} + +func AppendInt32(b []byte, v int32) []byte { + x := [4]byte{} + binary.LittleEndian.PutUint32(x[:], uint32(v)) + return append(b, x[:]...) +} + +func AppendInt64(b []byte, v int64) []byte { + x := [8]byte{} + binary.LittleEndian.PutUint64(x[:], uint64(v)) + return append(b, x[:]...) +} + +func AppendInt96(b []byte, v deprecated.Int96) []byte { + x := [12]byte{} + binary.LittleEndian.PutUint32(x[0:4], v[0]) + binary.LittleEndian.PutUint32(x[4:8], v[1]) + binary.LittleEndian.PutUint32(x[8:12], v[2]) + return append(b, x[:]...) +} + +func AppendFloat(b []byte, v float32) []byte { + x := [4]byte{} + binary.LittleEndian.PutUint32(x[:], math.Float32bits(v)) + return append(b, x[:]...) +} + +func AppendDouble(b []byte, v float64) []byte { + x := [8]byte{} + binary.LittleEndian.PutUint64(x[:], math.Float64bits(v)) + return append(b, x[:]...) +} + +func AppendByteArray(b, v []byte) []byte { + length := [ByteArrayLengthSize]byte{} + PutByteArrayLength(length[:], len(v)) + b = append(b, length[:]...) + b = append(b, v...) + return b +} + +func AppendByteArrayString(b []byte, v string) []byte { + length := [ByteArrayLengthSize]byte{} + PutByteArrayLength(length[:], len(v)) + b = append(b, length[:]...) + b = append(b, v...) + return b +} + +func AppendByteArrayLength(b []byte, n int) []byte { + length := [ByteArrayLengthSize]byte{} + PutByteArrayLength(length[:], n) + return append(b, length[:]...) +} + +func ByteArrayLength(b []byte) int { + return int(binary.LittleEndian.Uint32(b)) +} + +func PutByteArrayLength(b []byte, n int) { + binary.LittleEndian.PutUint32(b, uint32(n)) +} + +func RangeByteArray(b []byte, do func([]byte) error) (err error) { + for len(b) > 0 { + var v []byte + if v, b, err = NextByteArray(b); err != nil { + return err + } + if err = do(v); err != nil { + return err + } + } + return nil +} + +func NextByteArray(b []byte) (v, r []byte, err error) { + if len(b) < ByteArrayLengthSize { + return nil, b, ErrTooShort(len(b)) + } + n := ByteArrayLength(b) + if n > (len(b) - ByteArrayLengthSize) { + return nil, b, ErrTooShort(len(b)) + } + if n > MaxByteArrayLength { + return nil, b, ErrTooLarge(n) + } + n += ByteArrayLengthSize + return b[ByteArrayLengthSize:n:n], b[n:len(b):len(b)], nil +} + +func ErrTooShort(length int) error { + return fmt.Errorf("input of length %d is too short to contain a PLAIN encoded byte array value: %w", length, io.ErrUnexpectedEOF) +} + +func ErrTooLarge(length int) error { + return fmt.Errorf("byte array of length %d is too large to be encoded", length) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_be.go b/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_be.go new file mode 100644 index 0000000000..6c8c9000b5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_be.go @@ -0,0 +1,113 @@ +//go:build s390x + +package plain + +import ( + "encoding/binary" + "math" + + "github.com/parquet-go/parquet-go/encoding" +) + +// TODO: optimize by doing the byte swap in the output slice instead of +// allocating a temporay buffer. + +func (e *Encoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + srcLen := len(src) + byteEnc := make([]byte, (srcLen * 4)) + idx := 0 + for k := range srcLen { + binary.LittleEndian.PutUint32(byteEnc[idx:(4+idx)], uint32((src)[k])) + idx += 4 + } + return append(dst[:0], (byteEnc)...), nil +} + +func (e *Encoding) EncodeInt64(dst []byte, src []int64) ([]byte, error) { + srcLen := len(src) + byteEnc := make([]byte, (srcLen * 8)) + idx := 0 + for k := range srcLen { + binary.LittleEndian.PutUint64(byteEnc[idx:(8+idx)], uint64((src)[k])) + idx += 8 + } + return append(dst[:0], (byteEnc)...), nil +} + +func (e *Encoding) EncodeFloat(dst []byte, src []float32) ([]byte, error) { + srcLen := len(src) + byteEnc := make([]byte, (srcLen * 4)) + idx := 0 + for k := range srcLen { + binary.LittleEndian.PutUint32(byteEnc[idx:(4+idx)], math.Float32bits((src)[k])) + idx += 4 + } + return append(dst[:0], (byteEnc)...), nil +} + +func (e *Encoding) EncodeDouble(dst []byte, src []float64) ([]byte, error) { + srcLen := len(src) + byteEnc := make([]byte, (srcLen * 8)) + idx := 0 + for k := range srcLen { + binary.LittleEndian.PutUint64(byteEnc[idx:(8+idx)], math.Float64bits((src)[k])) + idx += 8 + } + return append(dst[:0], (byteEnc)...), nil +} + +func (e *Encoding) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + if (len(src) % 4) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "INT32", len(src)) + } + srcLen := (len(src) / 4) + byteDec := make([]int32, srcLen) + idx := 0 + for k := range srcLen { + byteDec[k] = int32(binary.LittleEndian.Uint32((src)[idx:(4 + idx)])) + idx += 4 + } + return append(dst[:0], (byteDec)...), nil +} + +func (e *Encoding) DecodeInt64(dst []int64, src []byte) ([]int64, error) { + if (len(src) % 8) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "INT64", len(src)) + } + srcLen := (len(src) / 8) + byteDec := make([]int64, srcLen) + idx := 0 + for k := range srcLen { + byteDec[k] = int64(binary.LittleEndian.Uint64((src)[idx:(8 + idx)])) + idx += 8 + } + return append(dst[:0], (byteDec)...), nil +} + +func (e *Encoding) DecodeFloat(dst []float32, src []byte) ([]float32, error) { + if (len(src) % 4) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "FLOAT", len(src)) + } + srcLen := (len(src) / 4) + byteDec := make([]float32, srcLen) + idx := 0 + for k := range srcLen { + byteDec[k] = float32(math.Float32frombits(binary.LittleEndian.Uint32((src)[idx:(4 + idx)]))) + idx += 4 + } + return append(dst[:0], (byteDec)...), nil +} + +func (e *Encoding) DecodeDouble(dst []float64, src []byte) ([]float64, error) { + if (len(src) % 8) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "DOUBLE", len(src)) + } + srcLen := (len(src) / 8) + byteDec := make([]float64, srcLen) + idx := 0 + for k := range srcLen { + byteDec[k] = float64(math.Float64frombits(binary.LittleEndian.Uint64((src)[idx:(8 + idx)]))) + idx += 8 + } + return append(dst[:0], (byteDec)...), nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_le.go b/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_le.go new file mode 100644 index 0000000000..bd1eadf6a0 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/plain/plain_le.go @@ -0,0 +1,52 @@ +//go:build !s390x + +package plain + +import ( + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +func (e *Encoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + return append(dst[:0], unsafecast.Slice[byte](src)...), nil +} + +func (e *Encoding) EncodeInt64(dst []byte, src []int64) ([]byte, error) { + return append(dst[:0], unsafecast.Slice[byte](src)...), nil +} + +func (e *Encoding) EncodeFloat(dst []byte, src []float32) ([]byte, error) { + return append(dst[:0], unsafecast.Slice[byte](src)...), nil +} + +func (e *Encoding) EncodeDouble(dst []byte, src []float64) ([]byte, error) { + return append(dst[:0], unsafecast.Slice[byte](src)...), nil +} + +func (e *Encoding) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + if (len(src) % 4) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "INT32", len(src)) + } + return append(dst[:0], unsafecast.Slice[int32](src)...), nil +} + +func (e *Encoding) DecodeInt64(dst []int64, src []byte) ([]int64, error) { + if (len(src) % 8) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "INT64", len(src)) + } + return append(dst[:0], unsafecast.Slice[int64](src)...), nil +} + +func (e *Encoding) DecodeFloat(dst []float32, src []byte) ([]float32, error) { + if (len(src) % 4) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "FLOAT", len(src)) + } + return append(dst[:0], unsafecast.Slice[float32](src)...), nil +} + +func (e *Encoding) DecodeDouble(dst []float64, src []byte) ([]float64, error) { + if (len(src) % 8) != 0 { + return dst, encoding.ErrDecodeInvalidInputSize(e, "DOUBLE", len(src)) + } + return append(dst[:0], unsafecast.Slice[float64](src)...), nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/rle/dictionary.go b/vendor/github.com/parquet-go/parquet-go/encoding/rle/dictionary.go new file mode 100644 index 0000000000..8304afc018 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/rle/dictionary.go @@ -0,0 +1,59 @@ +package rle + +import ( + "math/bits" + + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type DictionaryEncoding struct { + encoding.NotSupported +} + +func (e *DictionaryEncoding) String() string { + return "RLE_DICTIONARY" +} + +func (e *DictionaryEncoding) Encoding() format.Encoding { + return format.RLEDictionary +} + +func (e *DictionaryEncoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + bitWidth := maxLenInt32(src) + dst = append(dst[:0], byte(bitWidth)) + dst, err := encodeInt32(dst, src, uint(bitWidth)) + return dst, e.wrap(err) +} + +func (e *DictionaryEncoding) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + if len(src) == 0 { + return dst[:0], nil + } + buf := unsafecast.Slice[byte](dst) + buf, err := decodeInt32(buf[:0], src[1:], uint(src[0])) + return unsafecast.Slice[int32](buf), e.wrap(err) +} + +func (e *DictionaryEncoding) wrap(err error) error { + if err != nil { + err = encoding.Error(e, err) + } + return err +} + +func clearInt32(data []int32) { + for i := range data { + data[i] = 0 + } +} + +func maxLenInt32(data []int32) (max int) { + for _, v := range data { + if n := bits.Len32(uint32(v)); n > max { + max = n + } + } + return max +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle.go b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle.go new file mode 100644 index 0000000000..4f17d3c4b8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle.go @@ -0,0 +1,570 @@ +// Package rle implements the hybrid RLE/Bit-Packed encoding employed in +// repetition and definition levels, dictionary indexed data pages, and +// boolean values in the PLAIN encoding. +// +// https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 +package rle + +import ( + "encoding/binary" + "fmt" + "io" + "unsafe" + + "golang.org/x/sys/cpu" + + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" + "github.com/parquet-go/parquet-go/internal/bitpack" + "github.com/parquet-go/parquet-go/internal/bytealg" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +const ( + // This limit is intended to prevent unbounded memory allocations when + // decoding runs. + // + // We use a generous limit which allows for over 16 million values per page + // if there is only one run to encode the repetition or definition levels + // (this should be uncommon). + maxSupportedValueCount = 16 * 1024 * 1024 +) + +type Encoding struct { + encoding.NotSupported + BitWidth int +} + +func (e *Encoding) String() string { + return "RLE" +} + +func (e *Encoding) Encoding() format.Encoding { + return format.RLE +} + +func (e *Encoding) EncodeLevels(dst []byte, src []uint8) ([]byte, error) { + dst, err := encodeBytes(dst[:0], src, uint(e.BitWidth)) + return dst, e.wrap(err) +} + +func (e *Encoding) EncodeBoolean(dst []byte, src []byte) ([]byte, error) { + // In the case of encoding a boolean values, the 4 bytes length of the + // output is expected by the parquet format. We add the bytes as placeholder + // before appending the encoded data. + dst = append(dst[:0], 0, 0, 0, 0) + dst, err := encodeBits(dst, src) + binary.LittleEndian.PutUint32(dst, uint32(len(dst))-4) + return dst, e.wrap(err) +} + +func (e *Encoding) EncodeInt32(dst []byte, src []int32) ([]byte, error) { + dst, err := encodeInt32(dst[:0], src, uint(e.BitWidth)) + return dst, e.wrap(err) +} + +func (e *Encoding) DecodeLevels(dst []uint8, src []byte) ([]uint8, error) { + dst, err := decodeBytes(dst[:0], src, uint(e.BitWidth)) + return dst, e.wrap(err) +} + +func (e *Encoding) DecodeBoolean(dst []byte, src []byte) ([]byte, error) { + if len(src) == 4 { + return dst[:0], nil + } + if len(src) < 4 { + return dst[:0], fmt.Errorf("input shorter than 4 bytes: %w", io.ErrUnexpectedEOF) + } + n := int(binary.LittleEndian.Uint32(src)) + src = src[4:] + if n > len(src) { + return dst[:0], fmt.Errorf("input shorter than length prefix: %d < %d: %w", len(src), n, io.ErrUnexpectedEOF) + } + dst, err := decodeBits(dst[:0], src[:n]) + return dst, e.wrap(err) +} + +func (e *Encoding) DecodeInt32(dst []int32, src []byte) ([]int32, error) { + buf := unsafecast.Slice[byte](dst) + buf, err := decodeInt32(buf[:0], src, uint(e.BitWidth)) + return unsafecast.Slice[int32](buf), e.wrap(err) +} + +func (e *Encoding) wrap(err error) error { + if err != nil { + err = encoding.Error(e, err) + } + return err +} + +func encodeBits(dst, src []byte) ([]byte, error) { + if len(src) == 0 || isZero(src) || isOnes(src) { + dst = appendUvarint(dst, uint64(8*len(src))<<1) + if len(src) > 0 { + dst = append(dst, src[0]) + } + return dst, nil + } + + for i := 0; i < len(src); { + j := i + 1 + + // Look for contiguous sections of 8 bits, all zeros or ones; these + // are run-length encoded as it only takes 2 or 3 bytes to store these + // sequences. + if src[i] == 0 || src[i] == 0xFF { + for j < len(src) && src[i] == src[j] { + j++ + } + + if n := j - i; n > 1 { + dst = appendRunLengthBits(dst, 8*n, src[i]) + i = j + continue + } + } + + // Sequences of bits that are neither all zeroes or ones are bit-packed, + // which is a simple copy of the input to the output preceded with the + // bit-pack header. + for j < len(src) && (src[j-1] != src[j] || (src[j] != 0 && src[j] == 0xFF)) { + j++ + } + + if (j-i) > 1 && j < len(src) { + j-- + } + + dst = appendBitPackedBits(dst, src[i:j]) + i = j + } + return dst, nil +} + +func encodeBytes(dst, src []byte, bitWidth uint) ([]byte, error) { + if bitWidth > 8 { + return dst, errEncodeInvalidBitWidth("INT8", bitWidth) + } + if bitWidth == 0 { + if !isZero(src) { + return dst, errEncodeInvalidBitWidth("INT8", bitWidth) + } + return appendUvarint(dst, uint64(len(src))<<1), nil + } + + if len(src) >= 8 { + words := unsafecast.Slice[uint64](src) + if cpu.IsBigEndian { + srcLen := (len(src) / 8) + idx := 0 + for k := range srcLen { + words[k] = binary.LittleEndian.Uint64((src)[idx:(8 + idx)]) + idx += 8 + } + } else { + words = unsafe.Slice((*uint64)(unsafe.Pointer(&src[0])), len(src)/8) + } + + for i := 0; i < len(words); { + j := i + pattern := broadcast8x1(words[i]) + + for j < len(words) && words[j] == pattern { + j++ + } + + if i < j { + dst = appendRunLengthBytes(dst, 8*(j-i), byte(pattern)) + } else { + j++ + + for j < len(words) && words[j] != broadcast8x1(words[j-1]) { + j++ + } + + dst = appendBitPackedBytes(dst, words[i:j], bitWidth) + } + + i = j + } + } + + for i := (len(src) / 8) * 8; i < len(src); { + j := i + 1 + + for j < len(src) && src[i] == src[j] { + j++ + } + + dst = appendRunLengthBytes(dst, j-i, src[i]) + i = j + } + + return dst, nil +} + +func encodeInt32(dst []byte, src []int32, bitWidth uint) ([]byte, error) { + if bitWidth > 32 { + return dst, errEncodeInvalidBitWidth("INT32", bitWidth) + } + if bitWidth == 0 { + if !isZero(unsafecast.Slice[byte](src)) { + return dst, errEncodeInvalidBitWidth("INT32", bitWidth) + } + return appendUvarint(dst, uint64(len(src))<<1), nil + } + + if len(src) >= 8 { + words := unsafecast.Slice[[8]int32](src) + + for i := 0; i < len(words); { + j := i + pattern := broadcast8x4(words[i][0]) + + for j < len(words) && words[j] == pattern { + j++ + } + + if i < j { + dst = appendRunLengthInt32(dst, 8*(j-i), pattern[0], bitWidth) + } else { + j += 1 + j += encodeInt32IndexEqual8Contiguous(words[j:]) + dst = appendBitPackedInt32(dst, words[i:j], bitWidth) + } + + i = j + } + } + + for i := (len(src) / 8) * 8; i < len(src); { + j := i + 1 + + for j < len(src) && src[i] == src[j] { + j++ + } + + dst = appendRunLengthInt32(dst, j-i, src[i], bitWidth) + i = j + } + + return dst, nil +} + +func decodeBits(dst, src []byte) ([]byte, error) { + for i := 0; i < len(src); { + u, n := binary.Uvarint(src[i:]) + if n == 0 { + return dst, fmt.Errorf("decoding run-length block header: %w", io.ErrUnexpectedEOF) + } + if n < 0 { + return dst, fmt.Errorf("overflow after decoding %d/%d bytes of run-length block header", -n+i, len(src)) + } + i += n + + count, bitpacked := uint(u>>1), (u&1) != 0 + if count > maxSupportedValueCount { + return dst, fmt.Errorf("decoded run-length block cannot have more than %d values", maxSupportedValueCount) + } + if bitpacked { + n := int(count) + j := i + n + + if j > len(src) { + return dst, fmt.Errorf("decoding bit-packed block of %d values: %w", n, io.ErrUnexpectedEOF) + } + + dst = append(dst, src[i:j]...) + i = j + } else { + word := byte(0) + if i < len(src) { + word = src[i] + i++ + } + + offset := len(dst) + length := bitpack.ByteCount(count) + dst = resize(dst, offset+length) + bytealg.Broadcast(dst[offset:], word) + } + } + return dst, nil +} + +func decodeBytes(dst, src []byte, bitWidth uint) ([]byte, error) { + if bitWidth > 8 { + return dst, errDecodeInvalidBitWidth("INT8", bitWidth) + } + + for i := 0; i < len(src); { + u, n := binary.Uvarint(src[i:]) + if n == 0 { + return dst, fmt.Errorf("decoding run-length block header: %w", io.ErrUnexpectedEOF) + } + if n < 0 { + return dst, fmt.Errorf("overflow after decoding %d/%d bytes of run-length block header", -n+i, len(src)) + } + i += n + + count, bitpacked := uint(u>>1), (u&1) != 0 + if count > maxSupportedValueCount { + return dst, fmt.Errorf("decoded run-length block cannot have more than %d values", maxSupportedValueCount) + } + if bitpacked { + count *= 8 + j := i + bitpack.ByteCount(count*bitWidth) + + if j > len(src) { + return dst, fmt.Errorf("decoding bit-packed block of %d values: %w", 8*count, io.ErrUnexpectedEOF) + } + + offset := len(dst) + length := int(count) + dst = resize(dst, offset+length) + decodeBytesBitpack(dst[offset:], src[i:j], count, bitWidth) + + i = j + } else { + if bitWidth != 0 && (i+1) > len(src) { + return dst, fmt.Errorf("decoding run-length block of %d values: %w", count, io.ErrUnexpectedEOF) + } + + word := byte(0) + if bitWidth != 0 { + word = src[i] + i++ + } + + offset := len(dst) + length := int(count) + dst = resize(dst, offset+length) + bytealg.Broadcast(dst[offset:], word) + } + } + + return dst, nil +} + +func decodeInt32(dst, src []byte, bitWidth uint) ([]byte, error) { + if bitWidth > 32 { + return dst, errDecodeInvalidBitWidth("INT32", bitWidth) + } + + buf := make([]byte, 2*bitpack.PaddingInt32) + + for i := 0; i < len(src); { + u, n := binary.Uvarint(src[i:]) + if n == 0 { + return dst, fmt.Errorf("decoding run-length block header: %w", io.ErrUnexpectedEOF) + } + if n < 0 { + return dst, fmt.Errorf("overflow after decoding %d/%d bytes of run-length block header", -n+i, len(src)) + } + i += n + + count, bitpacked := uint(u>>1), (u&1) != 0 + if count > maxSupportedValueCount { + return dst, fmt.Errorf("decoded run-length block cannot have more than %d values", maxSupportedValueCount) + } + if bitpacked { + offset := len(dst) + length := int(count * bitWidth) + dst = resize(dst, offset+4*8*int(count)) + + // The bitpack.UnpackInt32 function requires the input to be padded + // or the function panics. If there is enough room in the input + // buffer we can use it, otherwise we have to copy it to a larger + // location (which should rarely happen). + in := src[i : i+length] + if (cap(in) - len(in)) >= bitpack.PaddingInt32 { + in = in[:cap(in)] + } else { + buf = resize(buf, len(in)+bitpack.PaddingInt32) + copy(buf, in) + in = buf + } + + out := unsafecast.Slice[int32](dst[offset:]) + bitpack.UnpackInt32(out, in, bitWidth) + i += length + } else { + j := i + bitpack.ByteCount(bitWidth) + + if j > len(src) { + return dst, fmt.Errorf("decoding run-length block of %d values: %w", count, io.ErrUnexpectedEOF) + } + + bits := [4]byte{} + copy(bits[:], src[i:j]) + + //swap the bytes in the "bits" array to take care of big endian arch + if cpu.IsBigEndian { + for m, n := 0, 3; m < n; m, n = m+1, n-1 { + bits[m], bits[n] = bits[n], bits[m] + } + } + dst = appendRepeat(dst, bits[:], count) + i = j + } + } + + return dst, nil +} + +func errEncodeInvalidBitWidth(typ string, bitWidth uint) error { + return errInvalidBitWidth("encode", typ, bitWidth) +} + +func errDecodeInvalidBitWidth(typ string, bitWidth uint) error { + return errInvalidBitWidth("decode", typ, bitWidth) +} + +func errInvalidBitWidth(op, typ string, bitWidth uint) error { + return fmt.Errorf("cannot %s %s with invalid bit-width=%d", op, typ, bitWidth) +} + +func appendRepeat(dst, pattern []byte, count uint) []byte { + offset := len(dst) + length := int(count) * len(pattern) + dst = resize(dst, offset+length) + i := offset + copy(dst[offset:], pattern) + for i < len(dst) { + i += copy(dst[i:], dst[offset:i]) + } + return dst +} + +func appendUvarint(dst []byte, u uint64) []byte { + var b [binary.MaxVarintLen64]byte + var n = binary.PutUvarint(b[:], u) + return append(dst, b[:n]...) +} + +func appendRunLengthBits(dst []byte, count int, value byte) []byte { + return appendRunLengthBytes(dst, count, value) +} + +func appendBitPackedBits(dst []byte, words []byte) []byte { + n := len(dst) + dst = resize(dst, n+binary.MaxVarintLen64+len(words)) + n += binary.PutUvarint(dst[n:], uint64(len(words)<<1)|1) + n += copy(dst[n:], words) + return dst[:n] +} + +func appendRunLengthBytes(dst []byte, count int, value byte) []byte { + n := len(dst) + dst = resize(dst, n+binary.MaxVarintLen64+1) + n += binary.PutUvarint(dst[n:], uint64(count)<<1) + dst[n] = value + return dst[:n+1] +} + +func appendBitPackedBytes(dst []byte, words []uint64, bitWidth uint) []byte { + n := len(dst) + dst = resize(dst, n+binary.MaxVarintLen64+(len(words)*int(bitWidth))+8) + n += binary.PutUvarint(dst[n:], uint64(len(words)<<1)|1) + n += encodeBytesBitpack(dst[n:], words, bitWidth) + return dst[:n] +} + +func appendRunLengthInt32(dst []byte, count int, value int32, bitWidth uint) []byte { + n := len(dst) + dst = resize(dst, n+binary.MaxVarintLen64+4) + n += binary.PutUvarint(dst[n:], uint64(count)<<1) + binary.LittleEndian.PutUint32(dst[n:], uint32(value)) + return dst[:n+bitpack.ByteCount(bitWidth)] +} + +func appendBitPackedInt32(dst []byte, words [][8]int32, bitWidth uint) []byte { + n := len(dst) + dst = resize(dst, n+binary.MaxVarintLen64+(len(words)*int(bitWidth))+32) + n += binary.PutUvarint(dst[n:], uint64(len(words))<<1|1) + n += encodeInt32Bitpack(dst[n:], words, bitWidth) + return dst[:n] +} + +func broadcast8x1(v uint64) uint64 { + return (v & 0xFF) * 0x0101010101010101 +} + +func broadcast8x4(v int32) [8]int32 { + return [8]int32{v, v, v, v, v, v, v, v} +} + +func isZero(data []byte) bool { + return bytealg.Count(data, 0x00) == len(data) +} + +func isOnes(data []byte) bool { + return bytealg.Count(data, 0xFF) == len(data) +} + +func resize(buf []byte, size int) []byte { + if cap(buf) < size { + return grow(buf, size) + } + return buf[:size] +} + +func grow(buf []byte, size int) []byte { + newCap := max(2*cap(buf), size) + newBuf := make([]byte, size, newCap) + copy(newBuf, buf) + return newBuf +} + +func encodeInt32BitpackDefault(dst []byte, src [][8]int32, bitWidth uint) int { + bits := unsafecast.Slice[int32](src) + bitpack.PackInt32(dst, bits, bitWidth) + return bitpack.ByteCount(uint(len(src)*8) * bitWidth) +} + +func encodeBytesBitpackDefault(dst []byte, src []uint64, bitWidth uint) int { + bitMask := uint64(1<> 8) & bitMask) << (1 * bitWidth)) | + (((word >> 16) & bitMask) << (2 * bitWidth)) | + (((word >> 24) & bitMask) << (3 * bitWidth)) | + (((word >> 32) & bitMask) << (4 * bitWidth)) | + (((word >> 40) & bitMask) << (5 * bitWidth)) | + (((word >> 48) & bitMask) << (6 * bitWidth)) | + (((word >> 56) & bitMask) << (7 * bitWidth)) + binary.LittleEndian.PutUint64(dst[n:], word) + n += int(bitWidth) + } + + return n +} + +func decodeBytesBitpackDefault(dst, src []byte, count, bitWidth uint) { + dst = dst[:0] + + bitMask := uint64(1< 0; count -= 8 { + j := i + byteCount + + bits := [8]byte{} + copy(bits[:], src[i:j]) + word := binary.LittleEndian.Uint64(bits[:]) + + dst = append(dst, + byte((word>>(0*bitWidth))&bitMask), + byte((word>>(1*bitWidth))&bitMask), + byte((word>>(2*bitWidth))&bitMask), + byte((word>>(3*bitWidth))&bitMask), + byte((word>>(4*bitWidth))&bitMask), + byte((word>>(5*bitWidth))&bitMask), + byte((word>>(6*bitWidth))&bitMask), + byte((word>>(7*bitWidth))&bitMask), + ) + + i = j + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.go b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.go new file mode 100644 index 0000000000..960a2b3e16 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.go @@ -0,0 +1,60 @@ +//go:build !purego + +package rle + +import ( + "golang.org/x/sys/cpu" +) + +var ( + encodeInt32IndexEqual8Contiguous func(words [][8]int32) int + encodeInt32Bitpack func(dst []byte, src [][8]int32, bitWidth uint) int + encodeBytesBitpack func(dst []byte, src []uint64, bitWidth uint) int + decodeBytesBitpack func(dst, src []byte, count, bitWidth uint) +) + +func init() { + switch { + case cpu.X86.HasAVX2: + encodeInt32IndexEqual8Contiguous = encodeInt32IndexEqual8ContiguousAVX2 + encodeInt32Bitpack = encodeInt32BitpackAVX2 + default: + encodeInt32IndexEqual8Contiguous = encodeInt32IndexEqual8ContiguousSSE + encodeInt32Bitpack = encodeInt32BitpackDefault + } + + switch { + case cpu.X86.HasBMI2: + encodeBytesBitpack = encodeBytesBitpackBMI2 + decodeBytesBitpack = decodeBytesBitpackBMI2 + default: + encodeBytesBitpack = encodeBytesBitpackDefault + decodeBytesBitpack = decodeBytesBitpackDefault + } +} + +//go:noescape +func encodeBytesBitpackBMI2(dst []byte, src []uint64, bitWidth uint) int + +//go:noescape +func encodeInt32IndexEqual8ContiguousAVX2(words [][8]int32) int + +//go:noescape +func encodeInt32IndexEqual8ContiguousSSE(words [][8]int32) int + +//go:noescape +func encodeInt32Bitpack1to16bitsAVX2(dst []byte, src [][8]int32, bitWidth uint) int + +func encodeInt32BitpackAVX2(dst []byte, src [][8]int32, bitWidth uint) int { + switch { + case bitWidth == 0: + return 0 + case bitWidth <= 16: + return encodeInt32Bitpack1to16bitsAVX2(dst, src, bitWidth) + default: + return encodeInt32BitpackDefault(dst, src, bitWidth) + } +} + +//go:noescape +func decodeBytesBitpackBMI2(dst, src []byte, count, bitWidth uint) diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.s b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.s new file mode 100644 index 0000000000..a03a559d6b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_amd64.s @@ -0,0 +1,174 @@ +//go:build !purego + +#include "textflag.h" + +GLOBL bitMasks<>(SB), RODATA|NOPTR, $64 +DATA bitMasks<>+0(SB)/8, $0b0000000100000001000000010000000100000001000000010000000100000001 +DATA bitMasks<>+8(SB)/8, $0b0000001100000011000000110000001100000011000000110000001100000011 +DATA bitMasks<>+16(SB)/8, $0b0000011100000111000001110000011100000111000001110000011100000111 +DATA bitMasks<>+24(SB)/8, $0b0000111100001111000011110000111100001111000011110000111100001111 +DATA bitMasks<>+32(SB)/8, $0b0001111100011111000111110001111100011111000111110001111100011111 +DATA bitMasks<>+40(SB)/8, $0b0011111100111111001111110011111100111111001111110011111100111111 +DATA bitMasks<>+48(SB)/8, $0b0111111101111111011111110111111101111111011111110111111101111111 +DATA bitMasks<>+56(SB)/8, $0b1111111111111111111111111111111111111111111111111111111111111111 + +// func decodeBytesBitpackBMI2(dst, src []byte, count, bitWidth uint) +TEXT ยทdecodeBytesBitpackBMI2(SB), NOSPLIT, $0-64 + MOVQ dst_base+0(FP), AX + MOVQ src_base+24(FP), BX + MOVQ count+48(FP), CX + MOVQ bitWidth+56(FP), DX + LEAQ bitMasks<>(SB), DI + MOVQ -8(DI)(DX*8), DI + XORQ SI, SI + SHRQ $3, CX + JMP test +loop: + MOVQ (BX), R8 + PDEPQ DI, R8, R8 + MOVQ R8, (AX)(SI*8) + ADDQ DX, BX + INCQ SI +test: + CMPQ SI, CX + JNE loop + RET + +// func encodeBytesBitpackBMI2(dst []byte, src []uint64, bitWidth uint) int +TEXT ยทencodeBytesBitpackBMI2(SB), NOSPLIT, $0-64 + MOVQ dst_base+0(FP), AX + MOVQ src_base+24(FP), BX + MOVQ src_len+32(FP), CX + MOVQ bitWidth+48(FP), DX + LEAQ bitMasks<>(SB), DI + MOVQ -8(DI)(DX*8), DI + XORQ SI, SI + JMP test +loop: + MOVQ (BX)(SI*8), R8 + PEXTQ DI, R8, R8 + MOVQ R8, (AX) + ADDQ DX, AX + INCQ SI +test: + CMPQ SI, CX + JNE loop +done: + SUBQ dst+0(FP), AX + MOVQ AX, ret+56(FP) + RET + +// func encodeInt32IndexEqual8ContiguousAVX2(words [][8]int32) int +TEXT ยทencodeInt32IndexEqual8ContiguousAVX2(SB), NOSPLIT, $0-32 + MOVQ words_base+0(FP), AX + MOVQ words_len+8(FP), BX + XORQ SI, SI + SHLQ $5, BX + JMP test +loop: + VMOVDQU (AX)(SI*1), Y0 + VPSHUFD $0, Y0, Y1 + VPCMPEQD Y1, Y0, Y0 + VMOVMSKPS Y0, CX + CMPL CX, $0xFF + JE done + ADDQ $32, SI +test: + CMPQ SI, BX + JNE loop +done: + VZEROUPPER + SHRQ $5, SI + MOVQ SI, ret+24(FP) + RET + +// func encodeInt32IndexEqual8ContiguousSSE(words [][8]int32) int +TEXT ยทencodeInt32IndexEqual8ContiguousSSE(SB), NOSPLIT, $0-32 + MOVQ words_base+0(FP), AX + MOVQ words_len+8(FP), BX + XORQ SI, SI + SHLQ $5, BX + JMP test +loop: + MOVOU (AX)(SI*1), X0 + MOVOU 16(AX)(SI*1), X1 + PSHUFD $0, X0, X2 + PCMPEQL X2, X0 + PCMPEQL X2, X1 + MOVMSKPS X0, CX + MOVMSKPS X1, DX + ANDL DX, CX + CMPL CX, $0xF + JE done + ADDQ $32, SI +test: + CMPQ SI, BX + JNE loop +done: + SHRQ $5, SI + MOVQ SI, ret+24(FP) + RET + +// func encodeInt32Bitpack1to16bitsAVX2(dst []byte, src [][8]int32, bitWidth uint) int +TEXT ยทencodeInt32Bitpack1to16bitsAVX2(SB), NOSPLIT, $0-64 + MOVQ dst_base+0(FP), AX + MOVQ src_base+24(FP), BX + MOVQ src_len+32(FP), CX + MOVQ bitWidth+48(FP), DX + + MOVQ DX, X0 + VPBROADCASTQ X0, Y6 // [1*bitWidth...] + VPSLLQ $1, Y6, Y7 // [2*bitWidth...] + VPADDQ Y6, Y7, Y8 // [3*bitWidth...] + VPSLLQ $2, Y6, Y9 // [4*bitWidth...] + + MOVQ $64, DI + MOVQ DI, X1 + VPBROADCASTQ X1, Y10 + VPSUBQ Y6, Y10, Y11 // [64-1*bitWidth...] + VPSUBQ Y9, Y10, Y12 // [64-4*bitWidth...] + VPCMPEQQ Y4, Y4, Y4 + VPSRLVQ Y11, Y4, Y4 + + VPXOR Y5, Y5, Y5 + XORQ SI, SI + SHLQ $5, CX + JMP test +loop: + VMOVDQU (BX)(SI*1), Y0 + VPSHUFD $0b01010101, Y0, Y1 + VPSHUFD $0b10101010, Y0, Y2 + VPSHUFD $0b11111111, Y0, Y3 + + VPAND Y4, Y0, Y0 + VPAND Y4, Y1, Y1 + VPAND Y4, Y2, Y2 + VPAND Y4, Y3, Y3 + + VPSLLVQ Y6, Y1, Y1 + VPSLLVQ Y7, Y2, Y2 + VPSLLVQ Y8, Y3, Y3 + + VPOR Y1, Y0, Y0 + VPOR Y3, Y2, Y2 + VPOR Y2, Y0, Y0 + + VPERMQ $0b00001010, Y0, Y1 + + VPSLLVQ X9, X1, X2 + VPSRLQ X12, X1, X3 + VBLENDPD $0b10, X3, X2, X1 + VBLENDPD $0b10, X5, X0, X0 + VPOR X1, X0, X0 + + VMOVDQU X0, (AX) + + ADDQ DX, AX + ADDQ $32, SI +test: + CMPQ SI, CX + JNE loop + VZEROUPPER + SUBQ dst+0(FP), AX + MOVQ AX, ret+56(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_purego.go b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_purego.go new file mode 100644 index 0000000000..8f3462d14c --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/rle/rle_purego.go @@ -0,0 +1,22 @@ +//go:build purego || !amd64 + +package rle + +func encodeBytesBitpack(dst []byte, src []uint64, bitWidth uint) int { + return encodeBytesBitpackDefault(dst, src, bitWidth) +} + +func encodeInt32IndexEqual8Contiguous(words [][8]int32) (n int) { + for n < len(words) && words[n] != broadcast8x4(words[n][0]) { + n++ + } + return n +} + +func encodeInt32Bitpack(dst []byte, src [][8]int32, bitWidth uint) int { + return encodeInt32BitpackDefault(dst, src, bitWidth) +} + +func decodeBytesBitpack(dst, src []byte, count, bitWidth uint) { + decodeBytesBitpackDefault(dst, src, count, bitWidth) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/LICENSE b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/LICENSE new file mode 100644 index 0000000000..1fbffdf72a --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Segment.io, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/binary.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/binary.go new file mode 100644 index 0000000000..73f15b03af --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/binary.go @@ -0,0 +1,369 @@ +package thrift + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// BinaryProtocol is a Protocol implementation for the binary thrift protocol. +// +// https://github.com/apache/thrift/blob/master/doc/specs/thrift-binary-protocol.md +type BinaryProtocol struct { + NonStrict bool +} + +func (p *BinaryProtocol) NewReader(r io.Reader) Reader { + return &binaryReader{p: p, r: r} +} + +func (p *BinaryProtocol) NewWriter(w io.Writer) Writer { + return &binaryWriter{p: p, w: w} +} + +func (p *BinaryProtocol) Features() Features { + return 0 +} + +type binaryReader struct { + p *BinaryProtocol + r io.Reader + b [8]byte +} + +func (r *binaryReader) Protocol() Protocol { + return r.p +} + +func (r *binaryReader) Reader() io.Reader { + return r.r +} + +func (r *binaryReader) ReadBool() (bool, error) { + v, err := r.ReadByte() + return v != 0, err +} + +func (r *binaryReader) ReadInt8() (int8, error) { + b, err := r.ReadByte() + return int8(b), err +} + +func (r *binaryReader) ReadInt16() (int16, error) { + b, err := r.read(2) + if len(b) < 2 { + return 0, err + } + return int16(binary.BigEndian.Uint16(b)), nil +} + +func (r *binaryReader) ReadInt32() (int32, error) { + b, err := r.read(4) + if len(b) < 4 { + return 0, err + } + return int32(binary.BigEndian.Uint32(b)), nil +} + +func (r *binaryReader) ReadInt64() (int64, error) { + b, err := r.read(8) + if len(b) < 8 { + return 0, err + } + return int64(binary.BigEndian.Uint64(b)), nil +} + +func (r *binaryReader) ReadFloat64() (float64, error) { + b, err := r.read(8) + if len(b) < 8 { + return 0, err + } + return math.Float64frombits(binary.BigEndian.Uint64(b)), nil +} + +func (r *binaryReader) ReadBytes() ([]byte, error) { + n, err := r.ReadLength() + if err != nil { + return nil, err + } + b := make([]byte, n) + _, err = io.ReadFull(r.r, b) + return b, err +} + +func (r *binaryReader) ReadString() (string, error) { + b, err := r.ReadBytes() + return unsafecast.String(b), err +} + +func (r *binaryReader) ReadLength() (int, error) { + b, err := r.read(4) + if len(b) < 4 { + return 0, err + } + n := binary.BigEndian.Uint32(b) + if n > math.MaxInt32 { + return 0, fmt.Errorf("length out of range: %d", n) + } + return int(n), nil +} + +func (r *binaryReader) ReadMessage() (Message, error) { + m := Message{} + + b, err := r.read(4) + if len(b) < 4 { + return m, err + } + + if (b[0] >> 7) == 0 { // non-strict + n := int(binary.BigEndian.Uint32(b)) + s := make([]byte, n) + _, err := io.ReadFull(r.r, s) + if err != nil { + return m, dontExpectEOF(err) + } + m.Name = unsafecast.String(s) + + t, err := r.ReadInt8() + if err != nil { + return m, dontExpectEOF(err) + } + + m.Type = MessageType(t & 0x7) + } else { + m.Type = MessageType(b[3] & 0x7) + + if m.Name, err = r.ReadString(); err != nil { + return m, dontExpectEOF(err) + } + } + + m.SeqID, err = r.ReadInt32() + return m, err +} + +func (r *binaryReader) ReadField() (Field, error) { + t, err := r.ReadInt8() + if err != nil { + return Field{}, err + } + i, err := r.ReadInt16() + if err != nil { + return Field{}, err + } + return Field{ID: i, Type: Type(t)}, nil +} + +func (r *binaryReader) ReadList() (List, error) { + t, err := r.ReadInt8() + if err != nil { + return List{}, err + } + n, err := r.ReadInt32() + if err != nil { + return List{}, dontExpectEOF(err) + } + return List{Size: n, Type: Type(t)}, nil +} + +func (r *binaryReader) ReadSet() (Set, error) { + l, err := r.ReadList() + return Set(l), err +} + +func (r *binaryReader) ReadMap() (Map, error) { + k, err := r.ReadByte() + if err != nil { + return Map{}, err + } + v, err := r.ReadByte() + if err != nil { + return Map{}, dontExpectEOF(err) + } + n, err := r.ReadInt32() + if err != nil { + return Map{}, dontExpectEOF(err) + } + return Map{Size: n, Key: Type(k), Value: Type(v)}, nil +} + +func (r *binaryReader) ReadByte() (byte, error) { + switch x := r.r.(type) { + case *bytes.Buffer: + return x.ReadByte() + case *bytes.Reader: + return x.ReadByte() + case *bufio.Reader: + return x.ReadByte() + case io.ByteReader: + return x.ReadByte() + default: + b, err := r.read(1) + if err != nil { + return 0, err + } + return b[0], nil + } +} + +func (r *binaryReader) read(n int) ([]byte, error) { + _, err := io.ReadFull(r.r, r.b[:n]) + return r.b[:n], err +} + +type binaryWriter struct { + p *BinaryProtocol + b [8]byte + w io.Writer +} + +func (w *binaryWriter) Protocol() Protocol { + return w.p +} + +func (w *binaryWriter) Writer() io.Writer { + return w.w +} + +func (w *binaryWriter) WriteBool(v bool) error { + var b byte + if v { + b = 1 + } + return w.writeByte(b) +} + +func (w *binaryWriter) WriteInt8(v int8) error { + return w.writeByte(byte(v)) +} + +func (w *binaryWriter) WriteInt16(v int16) error { + binary.BigEndian.PutUint16(w.b[:2], uint16(v)) + return w.write(w.b[:2]) +} + +func (w *binaryWriter) WriteInt32(v int32) error { + binary.BigEndian.PutUint32(w.b[:4], uint32(v)) + return w.write(w.b[:4]) +} + +func (w *binaryWriter) WriteInt64(v int64) error { + binary.BigEndian.PutUint64(w.b[:8], uint64(v)) + return w.write(w.b[:8]) +} + +func (w *binaryWriter) WriteFloat64(v float64) error { + binary.BigEndian.PutUint64(w.b[:8], math.Float64bits(v)) + return w.write(w.b[:8]) +} + +func (w *binaryWriter) WriteBytes(v []byte) error { + if err := w.WriteLength(len(v)); err != nil { + return err + } + return w.write(v) +} + +func (w *binaryWriter) WriteString(v string) error { + if err := w.WriteLength(len(v)); err != nil { + return err + } + return w.writeString(v) +} + +func (w *binaryWriter) WriteLength(n int) error { + if n < 0 { + return fmt.Errorf("negative length cannot be encoded in thrift: %d", n) + } + if n > math.MaxInt32 { + return fmt.Errorf("length is too large to be encoded in thrift: %d", n) + } + return w.WriteInt32(int32(n)) +} + +func (w *binaryWriter) WriteMessage(m Message) error { + if w.p.NonStrict { + if err := w.WriteString(m.Name); err != nil { + return err + } + if err := w.writeByte(byte(m.Type)); err != nil { + return err + } + } else { + w.b[0] = 1 << 7 + w.b[1] = 0 + w.b[2] = 0 + w.b[3] = byte(m.Type) & 0x7 + binary.BigEndian.PutUint32(w.b[4:], uint32(len(m.Name))) + + if err := w.write(w.b[:8]); err != nil { + return err + } + if err := w.writeString(m.Name); err != nil { + return err + } + } + return w.WriteInt32(m.SeqID) +} + +func (w *binaryWriter) WriteField(f Field) error { + if err := w.writeByte(byte(f.Type)); err != nil { + return err + } + return w.WriteInt16(f.ID) +} + +func (w *binaryWriter) WriteList(l List) error { + if err := w.writeByte(byte(l.Type)); err != nil { + return err + } + return w.WriteInt32(l.Size) +} + +func (w *binaryWriter) WriteSet(s Set) error { + return w.WriteList(List(s)) +} + +func (w *binaryWriter) WriteMap(m Map) error { + if err := w.writeByte(byte(m.Key)); err != nil { + return err + } + if err := w.writeByte(byte(m.Value)); err != nil { + return err + } + return w.WriteInt32(m.Size) +} + +func (w *binaryWriter) write(b []byte) error { + _, err := w.w.Write(b) + return err +} + +func (w *binaryWriter) writeString(s string) error { + _, err := io.WriteString(w.w, s) + return err +} + +func (w *binaryWriter) writeByte(b byte) error { + // The special cases are intended to reduce the runtime overheadof testing + // for the io.ByteWriter interface for common types. Type assertions on a + // concrete type is just a pointer comparison, instead of requiring a + // complex lookup in the type metadata. + switch x := w.w.(type) { + case *bytes.Buffer: + return x.WriteByte(b) + case *bufio.Writer: + return x.WriteByte(b) + case io.ByteWriter: + return x.WriteByte(b) + default: + w.b[0] = b + return w.write(w.b[:1]) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/compact.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/compact.go new file mode 100644 index 0000000000..7bca5771de --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/compact.go @@ -0,0 +1,348 @@ +package thrift + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +// CompactProtocol is a Protocol implementation for the compact thrift protocol. +// +// https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#integer-encoding +type CompactProtocol struct{} + +func (p *CompactProtocol) NewReader(r io.Reader) Reader { + return &compactReader{protocol: p, binary: binaryReader{r: r}} +} + +func (p *CompactProtocol) NewWriter(w io.Writer) Writer { + return &compactWriter{protocol: p, binary: binaryWriter{w: w}} +} + +func (p *CompactProtocol) Features() Features { + return UseDeltaEncoding | CoalesceBoolFields +} + +type compactReader struct { + protocol *CompactProtocol + binary binaryReader +} + +func (r *compactReader) Protocol() Protocol { + return r.protocol +} + +func (r *compactReader) Reader() io.Reader { + return r.binary.Reader() +} + +func (r *compactReader) ReadBool() (bool, error) { + return r.binary.ReadBool() +} + +func (r *compactReader) ReadInt8() (int8, error) { + return r.binary.ReadInt8() +} + +func (r *compactReader) ReadInt16() (int16, error) { + v, err := r.readVarint("int16", math.MinInt16, math.MaxInt16) + return int16(v), err +} + +func (r *compactReader) ReadInt32() (int32, error) { + v, err := r.readVarint("int32", math.MinInt32, math.MaxInt32) + return int32(v), err +} + +func (r *compactReader) ReadInt64() (int64, error) { + return r.readVarint("int64", math.MinInt64, math.MaxInt64) +} + +func (r *compactReader) ReadFloat64() (float64, error) { + return r.binary.ReadFloat64() +} + +func (r *compactReader) ReadBytes() ([]byte, error) { + n, err := r.ReadLength() + if err != nil { + return nil, err + } + b := make([]byte, n) + _, err = io.ReadFull(r.Reader(), b) + return b, err +} + +func (r *compactReader) ReadString() (string, error) { + b, err := r.ReadBytes() + return unsafecast.String(b), err +} + +func (r *compactReader) ReadLength() (int, error) { + n, err := r.readUvarint("length", math.MaxInt32) + return int(n), err +} + +func (r *compactReader) ReadMessage() (Message, error) { + m := Message{} + + b0, err := r.ReadByte() + if err != nil { + return m, err + } + if b0 != 0x82 { + return m, fmt.Errorf("invalid protocol id found when reading thrift message: %#x", b0) + } + + b1, err := r.ReadByte() + if err != nil { + return m, dontExpectEOF(err) + } + + seqID, err := r.readUvarint("seq id", math.MaxInt32) + if err != nil { + return m, dontExpectEOF(err) + } + + m.Type = MessageType(b1) & 0x7 + m.SeqID = int32(seqID) + m.Name, err = r.ReadString() + return m, dontExpectEOF(err) +} + +func (r *compactReader) ReadField() (Field, error) { + f := Field{} + + b, err := r.ReadByte() + if err != nil { + return f, err + } + + if Type(b) == STOP { + return f, nil + } + + if (b >> 4) != 0 { + f = Field{ID: int16(b >> 4), Type: Type(b & 0xF), Delta: true} + } else { + i, err := r.ReadInt16() + if err != nil { + return f, dontExpectEOF(err) + } + f = Field{ID: i, Type: Type(b)} + } + + return f, nil +} + +func (r *compactReader) ReadList() (List, error) { + b, err := r.ReadByte() + if err != nil { + return List{}, err + } + if (b >> 4) != 0xF { + return List{Size: int32(b >> 4), Type: Type(b & 0xF)}, nil + } + n, err := r.readUvarint("list size", math.MaxInt32) + if err != nil { + return List{}, dontExpectEOF(err) + } + return List{Size: int32(n), Type: Type(b & 0xF)}, nil +} + +func (r *compactReader) ReadSet() (Set, error) { + l, err := r.ReadList() + return Set(l), err +} + +func (r *compactReader) ReadMap() (Map, error) { + n, err := r.readUvarint("map size", math.MaxInt32) + if err != nil { + return Map{}, err + } + if n == 0 { // empty map + return Map{}, nil + } + b, err := r.ReadByte() + if err != nil { + return Map{}, dontExpectEOF(err) + } + return Map{Size: int32(n), Key: Type(b >> 4), Value: Type(b & 0xF)}, nil +} + +func (r *compactReader) ReadByte() (byte, error) { + return r.binary.ReadByte() +} + +func (r *compactReader) readUvarint(typ string, max uint64) (uint64, error) { + var br io.ByteReader + + switch x := r.Reader().(type) { + case *bytes.Buffer: + br = x + case *bytes.Reader: + br = x + case *bufio.Reader: + br = x + case io.ByteReader: + br = x + default: + br = &r.binary + } + + u, err := binary.ReadUvarint(br) + if err == nil { + if u > max { + err = fmt.Errorf("%s varint out of range: %d > %d", typ, u, max) + } + } + return u, err +} + +func (r *compactReader) readVarint(typ string, min, max int64) (int64, error) { + var br io.ByteReader + + switch x := r.Reader().(type) { + case *bytes.Buffer: + br = x + case *bytes.Reader: + br = x + case *bufio.Reader: + br = x + case io.ByteReader: + br = x + default: + br = &r.binary + } + + v, err := binary.ReadVarint(br) + if err == nil { + if v < min || v > max { + err = fmt.Errorf("%s varint out of range: %d not in [%d;%d]", typ, v, min, max) + } + } + return v, err +} + +type compactWriter struct { + protocol *CompactProtocol + binary binaryWriter + varint [binary.MaxVarintLen64]byte +} + +func (w *compactWriter) Protocol() Protocol { + return w.protocol +} + +func (w *compactWriter) Writer() io.Writer { + return w.binary.Writer() +} + +func (w *compactWriter) WriteBool(v bool) error { + return w.binary.WriteBool(v) +} + +func (w *compactWriter) WriteInt8(v int8) error { + return w.binary.WriteInt8(v) +} + +func (w *compactWriter) WriteInt16(v int16) error { + return w.writeVarint(int64(v)) +} + +func (w *compactWriter) WriteInt32(v int32) error { + return w.writeVarint(int64(v)) +} + +func (w *compactWriter) WriteInt64(v int64) error { + return w.writeVarint(v) +} + +func (w *compactWriter) WriteFloat64(v float64) error { + return w.binary.WriteFloat64(v) +} + +func (w *compactWriter) WriteBytes(v []byte) error { + if err := w.WriteLength(len(v)); err != nil { + return err + } + return w.binary.write(v) +} + +func (w *compactWriter) WriteString(v string) error { + if err := w.WriteLength(len(v)); err != nil { + return err + } + return w.binary.writeString(v) +} + +func (w *compactWriter) WriteLength(n int) error { + if n < 0 { + return fmt.Errorf("negative length cannot be encoded in thrift: %d", n) + } + if n > math.MaxInt32 { + return fmt.Errorf("length is too large to be encoded in thrift: %d", n) + } + return w.writeUvarint(uint64(n)) +} + +func (w *compactWriter) WriteMessage(m Message) error { + if err := w.binary.writeByte(0x82); err != nil { + return err + } + if err := w.binary.writeByte(byte(m.Type)); err != nil { + return err + } + if err := w.writeUvarint(uint64(m.SeqID)); err != nil { + return err + } + return w.WriteString(m.Name) +} + +func (w *compactWriter) WriteField(f Field) error { + if f.Type == STOP { + return w.binary.writeByte(0) + } + if f.ID <= 15 { + return w.binary.writeByte(byte(f.ID<<4) | byte(f.Type)) + } + if err := w.binary.writeByte(byte(f.Type)); err != nil { + return err + } + return w.WriteInt16(f.ID) +} + +func (w *compactWriter) WriteList(l List) error { + if l.Size <= 14 { + return w.binary.writeByte(byte(l.Size<<4) | byte(l.Type)) + } + if err := w.binary.writeByte(0xF0 | byte(l.Type)); err != nil { + return err + } + return w.writeUvarint(uint64(l.Size)) +} + +func (w *compactWriter) WriteSet(s Set) error { + return w.WriteList(List(s)) +} + +func (w *compactWriter) WriteMap(m Map) error { + if err := w.writeUvarint(uint64(m.Size)); err != nil || m.Size == 0 { + return err + } + return w.binary.writeByte((byte(m.Key) << 4) | byte(m.Value)) +} + +func (w *compactWriter) writeUvarint(v uint64) error { + n := binary.PutUvarint(w.varint[:], v) + return w.binary.write(w.varint[:n]) +} + +func (w *compactWriter) writeVarint(v int64) error { + n := binary.PutVarint(w.varint[:], v) + return w.binary.write(w.varint[:n]) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/debug.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/debug.go new file mode 100644 index 0000000000..25fe7da2f0 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/debug.go @@ -0,0 +1,230 @@ +package thrift + +import ( + "io" + "log" +) + +func NewDebugReader(r Reader, l *log.Logger) Reader { + return &debugReader{ + r: r, + l: l, + } +} + +func NewDebugWriter(w Writer, l *log.Logger) Writer { + return &debugWriter{ + w: w, + l: l, + } +} + +type debugReader struct { + r Reader + l *log.Logger +} + +func (d *debugReader) log(method string, res any, err error) { + if err != nil { + d.l.Printf("(%T).%s() โ†’ ERROR: %v", d.r, method, err) + } else { + d.l.Printf("(%T).%s() โ†’ %#v", d.r, method, res) + } +} + +func (d *debugReader) Protocol() Protocol { + return d.r.Protocol() +} + +func (d *debugReader) Reader() io.Reader { + return d.r.Reader() +} + +func (d *debugReader) ReadBool() (bool, error) { + v, err := d.r.ReadBool() + d.log("ReadBool", v, err) + return v, err +} + +func (d *debugReader) ReadInt8() (int8, error) { + v, err := d.r.ReadInt8() + d.log("ReadInt8", v, err) + return v, err +} + +func (d *debugReader) ReadInt16() (int16, error) { + v, err := d.r.ReadInt16() + d.log("ReadInt16", v, err) + return v, err +} + +func (d *debugReader) ReadInt32() (int32, error) { + v, err := d.r.ReadInt32() + d.log("ReadInt32", v, err) + return v, err +} + +func (d *debugReader) ReadInt64() (int64, error) { + v, err := d.r.ReadInt64() + d.log("ReadInt64", v, err) + return v, err +} + +func (d *debugReader) ReadFloat64() (float64, error) { + v, err := d.r.ReadFloat64() + d.log("ReadFloat64", v, err) + return v, err +} + +func (d *debugReader) ReadBytes() ([]byte, error) { + v, err := d.r.ReadBytes() + d.log("ReadBytes", v, err) + return v, err +} + +func (d *debugReader) ReadString() (string, error) { + v, err := d.r.ReadString() + d.log("ReadString", v, err) + return v, err +} + +func (d *debugReader) ReadLength() (int, error) { + v, err := d.r.ReadLength() + d.log("ReadLength", v, err) + return v, err +} + +func (d *debugReader) ReadMessage() (Message, error) { + v, err := d.r.ReadMessage() + d.log("ReadMessage", v, err) + return v, err +} + +func (d *debugReader) ReadField() (Field, error) { + v, err := d.r.ReadField() + d.log("ReadField", v, err) + return v, err +} + +func (d *debugReader) ReadList() (List, error) { + v, err := d.r.ReadList() + d.log("ReadList", v, err) + return v, err +} + +func (d *debugReader) ReadSet() (Set, error) { + v, err := d.r.ReadSet() + d.log("ReadSet", v, err) + return v, err +} + +func (d *debugReader) ReadMap() (Map, error) { + v, err := d.r.ReadMap() + d.log("ReadMap", v, err) + return v, err +} + +type debugWriter struct { + w Writer + l *log.Logger +} + +func (d *debugWriter) log(method string, arg any, err error) { + if err != nil { + d.l.Printf("(%T).%s(%#v) โ†’ ERROR: %v", d.w, method, arg, err) + } else { + d.l.Printf("(%T).%s(%#v)", d.w, method, arg) + } +} + +func (d *debugWriter) Protocol() Protocol { + return d.w.Protocol() +} + +func (d *debugWriter) Writer() io.Writer { + return d.w.Writer() +} + +func (d *debugWriter) WriteBool(v bool) error { + err := d.w.WriteBool(v) + d.log("WriteBool", v, err) + return err +} + +func (d *debugWriter) WriteInt8(v int8) error { + err := d.w.WriteInt8(v) + d.log("WriteInt8", v, err) + return err +} + +func (d *debugWriter) WriteInt16(v int16) error { + err := d.w.WriteInt16(v) + d.log("WriteInt16", v, err) + return err +} + +func (d *debugWriter) WriteInt32(v int32) error { + err := d.w.WriteInt32(v) + d.log("WriteInt32", v, err) + return err +} + +func (d *debugWriter) WriteInt64(v int64) error { + err := d.w.WriteInt64(v) + d.log("WriteInt64", v, err) + return err +} + +func (d *debugWriter) WriteFloat64(v float64) error { + err := d.w.WriteFloat64(v) + d.log("WriteFloat64", v, err) + return err +} + +func (d *debugWriter) WriteBytes(v []byte) error { + err := d.w.WriteBytes(v) + d.log("WriteBytes", v, err) + return err +} + +func (d *debugWriter) WriteString(v string) error { + err := d.w.WriteString(v) + d.log("WriteString", v, err) + return err +} + +func (d *debugWriter) WriteLength(n int) error { + err := d.w.WriteLength(n) + d.log("WriteLength", n, err) + return err +} + +func (d *debugWriter) WriteMessage(m Message) error { + err := d.w.WriteMessage(m) + d.log("WriteMessage", m, err) + return err +} + +func (d *debugWriter) WriteField(f Field) error { + err := d.w.WriteField(f) + d.log("WriteField", f, err) + return err +} + +func (d *debugWriter) WriteList(l List) error { + err := d.w.WriteList(l) + d.log("WriteList", l, err) + return err +} + +func (d *debugWriter) WriteSet(s Set) error { + err := d.w.WriteSet(s) + d.log("WriteSet", s, err) + return err +} + +func (d *debugWriter) WriteMap(m Map) error { + err := d.w.WriteMap(m) + d.log("WriteMap", m, err) + return err +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/decode.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/decode.go new file mode 100644 index 0000000000..187323c601 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/decode.go @@ -0,0 +1,689 @@ +package thrift + +import ( + "bufio" + "bytes" + "fmt" + "io" + "reflect" + "sync/atomic" +) + +// Unmarshal deserializes the thrift data from b to v using to the protocol p. +// +// The function errors if the data in b does not match the type of v. +// +// The function panics if v cannot be converted to a thrift representation. +// +// As an optimization, the value passed in v may be reused across multiple calls +// to Unmarshal, allowing the function to reuse objects referenced by pointer +// fields of struct values. When reusing objects, the application is responsible +// for resetting the state of v before calling Unmarshal again. +func Unmarshal(p Protocol, b []byte, v any) error { + br := bytes.NewReader(b) + pr := p.NewReader(br) + + if err := NewDecoder(pr).Decode(v); err != nil { + return err + } + + if n := br.Len(); n != 0 { + return fmt.Errorf("unexpected trailing bytes at the end of thrift input: %d", n) + } + + return nil +} + +type Decoder struct { + r Reader + f flags +} + +func NewDecoder(r Reader) *Decoder { + return &Decoder{r: r, f: decoderFlags(r)} +} + +func (d *Decoder) Decode(v any) error { + t := reflect.TypeOf(v) + p := reflect.ValueOf(v) + + if t.Kind() != reflect.Ptr { + panic("thrift.(*Decoder).Decode: expected pointer type but got " + t.String()) + } + + t = t.Elem() + p = p.Elem() + + cache, _ := decoderCache.Load().(map[typeID]decodeFunc) + decode, _ := cache[makeTypeID(t)] + + if decode == nil { + decode = decodeFuncOf(t, make(decodeFuncCache)) + + newCache := make(map[typeID]decodeFunc, len(cache)+1) + newCache[makeTypeID(t)] = decode + for k, v := range cache { + newCache[k] = v + } + + decoderCache.Store(newCache) + } + + return decode(d.r, p, d.f) +} + +func (d *Decoder) Reset(r Reader) { + d.r = r + d.f = d.f.without(protocolFlags).with(decoderFlags(r)) +} + +func (d *Decoder) SetStrict(enabled bool) { + if enabled { + d.f = d.f.with(strict) + } else { + d.f = d.f.without(strict) + } +} + +func decoderFlags(r Reader) flags { + return flags(r.Protocol().Features() << featuresBitOffset) +} + +var decoderCache atomic.Value // map[typeID]decodeFunc + +type decodeFunc func(Reader, reflect.Value, flags) error + +type decodeFuncCache map[reflect.Type]decodeFunc + +func decodeFuncOf(t reflect.Type, seen decodeFuncCache) decodeFunc { + f := seen[t] + if f != nil { + return f + } + switch t.Kind() { + case reflect.Bool: + f = decodeBool + case reflect.Int8: + f = decodeInt8 + case reflect.Int16: + f = decodeInt16 + case reflect.Int32: + f = decodeInt32 + case reflect.Int64, reflect.Int: + f = decodeInt64 + case reflect.Float32, reflect.Float64: + f = decodeFloat64 + case reflect.String: + f = decodeString + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { // []byte + f = decodeBytes + } else { + f = decodeFuncSliceOf(t, seen) + } + case reflect.Map: + f = decodeFuncMapOf(t, seen) + case reflect.Struct: + f = decodeFuncStructOf(t, seen) + case reflect.Ptr: + f = decodeFuncPtrOf(t, seen) + default: + panic("type cannot be decoded in thrift: " + t.String()) + } + seen[t] = f + return f +} + +func decodeBool(r Reader, v reflect.Value, _ flags) error { + b, err := r.ReadBool() + if err != nil { + return err + } + v.SetBool(b) + return nil +} + +func decodeInt8(r Reader, v reflect.Value, _ flags) error { + i, err := r.ReadInt8() + if err != nil { + return err + } + v.SetInt(int64(i)) + return nil +} + +func decodeInt16(r Reader, v reflect.Value, _ flags) error { + i, err := r.ReadInt16() + if err != nil { + return err + } + v.SetInt(int64(i)) + return nil +} + +func decodeInt32(r Reader, v reflect.Value, _ flags) error { + i, err := r.ReadInt32() + if err != nil { + return err + } + v.SetInt(int64(i)) + return nil +} + +func decodeInt64(r Reader, v reflect.Value, _ flags) error { + i, err := r.ReadInt64() + if err != nil { + return err + } + v.SetInt(int64(i)) + return nil +} + +func decodeFloat64(r Reader, v reflect.Value, _ flags) error { + f, err := r.ReadFloat64() + if err != nil { + return err + } + v.SetFloat(f) + return nil +} + +func decodeString(r Reader, v reflect.Value, _ flags) error { + s, err := r.ReadString() + if err != nil { + return err + } + v.SetString(s) + return nil +} + +func decodeBytes(r Reader, v reflect.Value, _ flags) error { + b, err := r.ReadBytes() + if err != nil { + return err + } + v.SetBytes(b) + return nil +} + +func decodeFuncSliceOf(t reflect.Type, seen decodeFuncCache) decodeFunc { + elem := t.Elem() + typ := TypeOf(elem) + dec := decodeFuncOf(elem, seen) + + return func(r Reader, v reflect.Value, flags flags) error { + l, err := r.ReadList() + if err != nil { + return err + } + + // Sometimes the list type is set to TRUE when the list contains only + // TRUE values. Thrift does not seem to optimize the encoding by + // omitting the boolean values that are known to all be TRUE, we still + // need to decode them. + switch l.Type { + case TRUE: + l.Type = BOOL + } + + // TODO: implement type conversions? + if typ != l.Type { + if flags.have(strict) { + return &TypeMismatch{item: "list item", Expect: typ, Found: l.Type} + } + return nil + } + + v.Set(reflect.MakeSlice(t, int(l.Size), int(l.Size))) + flags = flags.only(decodeFlags) + + for i := range int(l.Size) { + if err := dec(r, v.Index(i), flags); err != nil { + return with(dontExpectEOF(err), &decodeErrorList{cause: l, index: i}) + } + } + + return nil + } +} + +func decodeFuncMapOf(t reflect.Type, seen decodeFuncCache) decodeFunc { + key, elem := t.Key(), t.Elem() + if elem.Size() == 0 { // map[?]struct{} + return decodeFuncMapAsSetOf(t, seen) + } + + mapType := reflect.MapOf(key, elem) + keyZero := reflect.Zero(key) + elemZero := reflect.Zero(elem) + keyType := TypeOf(key) + elemType := TypeOf(elem) + decodeKey := decodeFuncOf(key, seen) + decodeElem := decodeFuncOf(elem, seen) + + return func(r Reader, v reflect.Value, flags flags) error { + m, err := r.ReadMap() + if err != nil { + return err + } + + v.Set(reflect.MakeMapWithSize(mapType, int(m.Size))) + + if m.Size == 0 { // empty map + return nil + } + + // TODO: implement type conversions? + if keyType != m.Key { + if flags.have(strict) { + return &TypeMismatch{item: "map key", Expect: keyType, Found: m.Key} + } + return nil + } + + if elemType != m.Value { + if flags.have(strict) { + return &TypeMismatch{item: "map value", Expect: elemType, Found: m.Value} + } + return nil + } + + tmpKey := reflect.New(key).Elem() + tmpElem := reflect.New(elem).Elem() + flags = flags.only(decodeFlags) + + for i := range int(m.Size) { + if err := decodeKey(r, tmpKey, flags); err != nil { + return with(dontExpectEOF(err), &decodeErrorMap{cause: m, index: i}) + } + if err := decodeElem(r, tmpElem, flags); err != nil { + return with(dontExpectEOF(err), &decodeErrorMap{cause: m, index: i}) + } + v.SetMapIndex(tmpKey, tmpElem) + tmpKey.Set(keyZero) + tmpElem.Set(elemZero) + } + + return nil + } +} + +func decodeFuncMapAsSetOf(t reflect.Type, seen decodeFuncCache) decodeFunc { + key, elem := t.Key(), t.Elem() + keyZero := reflect.Zero(key) + elemZero := reflect.Zero(elem) + typ := TypeOf(key) + dec := decodeFuncOf(key, seen) + + return func(r Reader, v reflect.Value, flags flags) error { + s, err := r.ReadSet() + if err != nil { + return err + } + + // See decodeFuncSliceOf for details about why this type conversion + // needs to be done. + switch s.Type { + case TRUE: + s.Type = BOOL + } + + v.Set(reflect.MakeMapWithSize(t, int(s.Size))) + + if s.Size == 0 { + return nil + } + + // TODO: implement type conversions? + if typ != s.Type { + if flags.have(strict) { + return &TypeMismatch{item: "list item", Expect: typ, Found: s.Type} + } + return nil + } + + tmp := reflect.New(key).Elem() + flags = flags.only(decodeFlags) + + for i := range int(s.Size) { + if err := dec(r, tmp, flags); err != nil { + return with(dontExpectEOF(err), &decodeErrorSet{cause: s, index: i}) + } + v.SetMapIndex(tmp, elemZero) + tmp.Set(keyZero) + } + + return nil + } +} + +type structDecoder struct { + fields []structDecoderField + union []int + minID int16 + zero reflect.Value + required []uint64 +} + +func (dec *structDecoder) decode(r Reader, v reflect.Value, flags flags) error { + flags = flags.only(decodeFlags) + coalesceBoolFields := flags.have(coalesceBoolFields) + + lastField := reflect.Value{} + union := len(dec.union) > 0 + seen := make([]uint64, 1) + if len(dec.required) > len(seen) { + seen = make([]uint64, len(dec.required)) + } + + err := readStruct(r, func(r Reader, f Field) error { + i := int(f.ID) - int(dec.minID) + if i < 0 || i >= len(dec.fields) || dec.fields[i].decode == nil { + return skipField(r, f) + } + field := &dec.fields[i] + seen[i/64] |= 1 << (i % 64) + + // TODO: implement type conversions? + if f.Type != field.typ && !(f.Type == TRUE && field.typ == BOOL) { + if flags.have(strict) { + return &TypeMismatch{item: "field value", Expect: field.typ, Found: f.Type} + } + return nil + } + + x := v + for _, i := range field.index { + if x.Kind() == reflect.Ptr { + x = x.Elem() + } + if x = x.Field(i); x.Kind() == reflect.Ptr { + if x.IsNil() { + x.Set(reflect.New(x.Type().Elem())) + } + } + } + + if union { + v.Set(dec.zero) + } + + lastField = x + + if coalesceBoolFields && (f.Type == TRUE || f.Type == FALSE) { + for x.Kind() == reflect.Ptr { + if x.IsNil() { + x.Set(reflect.New(x.Type().Elem())) + } + x = x.Elem() + } + x.SetBool(f.Type == TRUE) + return nil + } + + return field.decode(r, x, flags.with(field.flags)) + }) + if err != nil { + return err + } + + for i, required := range dec.required { + if mask := required & seen[i]; mask != required { + i *= 64 + for (mask & 1) != 0 { + mask >>= 1 + i++ + } + field := &dec.fields[i] + return &MissingField{Field: Field{ID: field.id, Type: field.typ}} + } + } + + if union && lastField.IsValid() { + v.FieldByIndex(dec.union).Set(lastField.Addr()) + } + + return nil +} + +type structDecoderField struct { + index []int + id int16 + flags flags + typ Type + decode decodeFunc +} + +func decodeFuncStructOf(t reflect.Type, seen decodeFuncCache) decodeFunc { + dec := &structDecoder{ + zero: reflect.Zero(t), + } + decode := dec.decode + seen[t] = decode + + fields := make([]structDecoderField, 0, t.NumField()) + forEachStructField(t, nil, func(f structField) { + if f.flags.have(union) { + dec.union = f.index + } else { + fields = append(fields, structDecoderField{ + index: f.index, + id: f.id, + flags: f.flags, + typ: TypeOf(f.typ), + decode: decodeFuncStructFieldOf(f, seen), + }) + } + }) + + minID := int16(0) + maxID := int16(0) + + for _, f := range fields { + if f.id < minID || minID == 0 { + minID = f.id + } + if f.id > maxID { + maxID = f.id + } + } + + dec.fields = make([]structDecoderField, (maxID-minID)+1) + dec.minID = minID + dec.required = make([]uint64, len(fields)/64+1) + + for _, f := range fields { + i := f.id - minID + p := dec.fields[i] + if p.decode != nil { + panic(fmt.Errorf("thrift struct field id %d is present multiple times in %s with types %s and %s", f.id, t, p.typ, f.typ)) + } + dec.fields[i] = f + if f.flags.have(required) { + dec.required[i/64] |= 1 << (i % 64) + } + } + + return decode +} + +func decodeFuncStructFieldOf(f structField, seen decodeFuncCache) decodeFunc { + if f.flags.have(enum) { + switch f.typ.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return decodeInt32 + } + } + return decodeFuncOf(f.typ, seen) +} + +func decodeFuncPtrOf(t reflect.Type, seen decodeFuncCache) decodeFunc { + elem := t.Elem() + decode := decodeFuncOf(t.Elem(), seen) + return func(r Reader, v reflect.Value, f flags) error { + if v.IsNil() { + v.Set(reflect.New(elem)) + } + return decode(r, v.Elem(), f) + } +} + +func readBinary(r Reader, f func(io.Reader) error) error { + n, err := r.ReadLength() + if err != nil { + return err + } + return dontExpectEOF(f(io.LimitReader(r.Reader(), int64(n)))) +} + +func readList(r Reader, f func(Reader, Type) error) error { + l, err := r.ReadList() + if err != nil { + return err + } + + for i := range int(l.Size) { + if err := f(r, l.Type); err != nil { + return with(dontExpectEOF(err), &decodeErrorList{cause: l, index: i}) + } + } + + return nil +} + +func readSet(r Reader, f func(Reader, Type) error) error { + s, err := r.ReadSet() + if err != nil { + return err + } + + for i := range int(s.Size) { + if err := f(r, s.Type); err != nil { + return with(dontExpectEOF(err), &decodeErrorSet{cause: s, index: i}) + } + } + + return nil +} + +func readMap(r Reader, f func(Reader, Type, Type) error) error { + m, err := r.ReadMap() + if err != nil { + return err + } + + for i := range int(m.Size) { + if err := f(r, m.Key, m.Value); err != nil { + return with(dontExpectEOF(err), &decodeErrorMap{cause: m, index: i}) + } + } + + return nil +} + +func readStruct(r Reader, f func(Reader, Field) error) error { + lastFieldID := int16(0) + numFields := 0 + + for { + x, err := r.ReadField() + if err != nil { + if numFields > 0 { + err = dontExpectEOF(err) + } + return err + } + + if x.Type == STOP { + return nil + } + + if x.Delta { + x.ID += lastFieldID + x.Delta = false + } + + if err := f(r, x); err != nil { + return with(dontExpectEOF(err), &decodeErrorField{cause: x}) + } + + lastFieldID = x.ID + numFields++ + } +} + +func skip(r Reader, t Type) error { + var err error + switch t { + case TRUE, FALSE: + _, err = r.ReadBool() + case I8: + _, err = r.ReadInt8() + case I16: + _, err = r.ReadInt16() + case I32: + _, err = r.ReadInt32() + case I64: + _, err = r.ReadInt64() + case DOUBLE: + _, err = r.ReadFloat64() + case BINARY: + err = skipBinary(r) + case LIST: + err = skipList(r) + case SET: + err = skipSet(r) + case MAP: + err = skipMap(r) + case STRUCT: + err = skipStruct(r) + default: + return fmt.Errorf("skipping unsupported thrift type %d", t) + } + return err +} + +func skipBinary(r Reader) error { + n, err := r.ReadLength() + if err != nil { + return err + } + if n == 0 { + return nil + } + switch x := r.Reader().(type) { + case *bufio.Reader: + _, err = x.Discard(int(n)) + default: + _, err = io.CopyN(io.Discard, x, int64(n)) + } + return dontExpectEOF(err) +} + +func skipList(r Reader) error { + return readList(r, skip) +} + +func skipSet(r Reader) error { + return readSet(r, skip) +} + +func skipMap(r Reader) error { + return readMap(r, func(r Reader, k, v Type) error { + if err := skip(r, k); err != nil { + return dontExpectEOF(err) + } + if err := skip(r, v); err != nil { + return dontExpectEOF(err) + } + return nil + }) +} + +func skipStruct(r Reader) error { + return readStruct(r, skipField) +} + +func skipField(r Reader, f Field) error { + return skip(r, f.Type) +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/encode.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/encode.go new file mode 100644 index 0000000000..c591a1f0cc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/encode.go @@ -0,0 +1,400 @@ +package thrift + +import ( + "bytes" + "cmp" + "fmt" + "math" + "reflect" + "slices" + "sync/atomic" +) + +// Marshal serializes v into a thrift representation according to the the +// protocol p. +// +// The function panics if v cannot be converted to a thrift representation. +func Marshal(p Protocol, v any) ([]byte, error) { + buf := new(bytes.Buffer) + enc := NewEncoder(p.NewWriter(buf)) + err := enc.Encode(v) + return buf.Bytes(), err +} + +type Encoder struct { + w Writer + f flags +} + +func NewEncoder(w Writer) *Encoder { + return &Encoder{w: w, f: encoderFlags(w)} +} + +func (e *Encoder) Encode(v any) error { + t := reflect.TypeOf(v) + cache, _ := encoderCache.Load().(map[typeID]encodeFunc) + encode, _ := cache[makeTypeID(t)] + + if encode == nil { + encode = encodeFuncOf(t, make(encodeFuncCache)) + + newCache := make(map[typeID]encodeFunc, len(cache)+1) + newCache[makeTypeID(t)] = encode + for k, v := range cache { + newCache[k] = v + } + + encoderCache.Store(newCache) + } + + return encode(e.w, reflect.ValueOf(v), e.f) +} + +func (e *Encoder) Reset(w Writer) { + e.w = w + e.f = e.f.without(protocolFlags).with(encoderFlags(w)) +} + +func encoderFlags(w Writer) flags { + return flags(w.Protocol().Features() << featuresBitOffset) +} + +var encoderCache atomic.Value // map[typeID]encodeFunc + +type encodeFunc func(Writer, reflect.Value, flags) error + +type encodeFuncCache map[reflect.Type]encodeFunc + +func encodeFuncOf(t reflect.Type, seen encodeFuncCache) encodeFunc { + f := seen[t] + if f != nil { + return f + } + switch t.Kind() { + case reflect.Bool: + f = encodeBool + case reflect.Int8: + f = encodeInt8 + case reflect.Int16: + f = encodeInt16 + case reflect.Int32: + f = encodeInt32 + case reflect.Int64, reflect.Int: + f = encodeInt64 + case reflect.Float32, reflect.Float64: + f = encodeFloat64 + case reflect.String: + f = encodeString + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + f = encodeBytes + } else { + f = encodeFuncSliceOf(t, seen) + } + case reflect.Map: + f = encodeFuncMapOf(t, seen) + case reflect.Struct: + f = encodeFuncStructOf(t, seen) + case reflect.Ptr: + f = encodeFuncPtrOf(t, seen) + default: + panic("type cannot be encoded in thrift: " + t.String()) + } + seen[t] = f + return f +} + +func encodeBool(w Writer, v reflect.Value, _ flags) error { + return w.WriteBool(v.Bool()) +} + +func encodeInt8(w Writer, v reflect.Value, _ flags) error { + return w.WriteInt8(int8(v.Int())) +} + +func encodeInt16(w Writer, v reflect.Value, _ flags) error { + return w.WriteInt16(int16(v.Int())) +} + +func encodeInt32(w Writer, v reflect.Value, _ flags) error { + return w.WriteInt32(int32(v.Int())) +} + +func encodeInt64(w Writer, v reflect.Value, _ flags) error { + return w.WriteInt64(v.Int()) +} + +func encodeFloat64(w Writer, v reflect.Value, _ flags) error { + return w.WriteFloat64(v.Float()) +} + +func encodeString(w Writer, v reflect.Value, _ flags) error { + return w.WriteString(v.String()) +} + +func encodeBytes(w Writer, v reflect.Value, _ flags) error { + return w.WriteBytes(v.Bytes()) +} + +func encodeFuncSliceOf(t reflect.Type, seen encodeFuncCache) encodeFunc { + elem := t.Elem() + typ := TypeOf(elem) + enc := encodeFuncOf(elem, seen) + + return func(w Writer, v reflect.Value, flags flags) error { + n := v.Len() + if n > math.MaxInt32 { + return fmt.Errorf("slice length is too large to be represented in thrift: %d > max(int32)", n) + } + + err := w.WriteList(List{ + Size: int32(n), + Type: typ, + }) + if err != nil { + return err + } + + for i := range n { + if err := enc(w, v.Index(i), flags); err != nil { + return err + } + } + + return nil + } +} + +func encodeFuncMapOf(t reflect.Type, seen encodeFuncCache) encodeFunc { + key, elem := t.Key(), t.Elem() + if elem.Size() == 0 { // map[?]struct{} + return encodeFuncMapAsSetOf(t, seen) + } + + keyType := TypeOf(key) + elemType := TypeOf(elem) + encodeKey := encodeFuncOf(key, seen) + encodeElem := encodeFuncOf(elem, seen) + + return func(w Writer, v reflect.Value, flags flags) error { + n := v.Len() + if n > math.MaxInt32 { + return fmt.Errorf("map length is too large to be represented in thrift: %d > max(int32)", n) + } + + err := w.WriteMap(Map{ + Size: int32(n), + Key: keyType, + Value: elemType, + }) + if err != nil { + return err + } + if n == 0 { // empty map + return nil + } + + for i, iter := 0, v.MapRange(); iter.Next(); i++ { + if err := encodeKey(w, iter.Key(), flags); err != nil { + return err + } + if err := encodeElem(w, iter.Value(), flags); err != nil { + return err + } + } + + return nil + } +} + +func encodeFuncMapAsSetOf(t reflect.Type, seen encodeFuncCache) encodeFunc { + key := t.Key() + typ := TypeOf(key) + enc := encodeFuncOf(key, seen) + + return func(w Writer, v reflect.Value, flags flags) error { + n := v.Len() + if n > math.MaxInt32 { + return fmt.Errorf("map length is too large to be represented in thrift: %d > max(int32)", n) + } + + err := w.WriteSet(Set{ + Size: int32(n), + Type: typ, + }) + if err != nil { + return err + } + if n == 0 { // empty map + return nil + } + + for i, iter := 0, v.MapRange(); iter.Next(); i++ { + if err := enc(w, iter.Key(), flags); err != nil { + return err + } + } + + return nil + } +} + +type structEncoder struct { + fields []structEncoderField + union bool +} + +func dereference(v reflect.Value) reflect.Value { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return v + } + v = v.Elem() + } + return v +} + +func isTrue(v reflect.Value) bool { + v = dereference(v) + return v.IsValid() && v.Kind() == reflect.Bool && v.Bool() +} + +func (enc *structEncoder) encode(w Writer, v reflect.Value, flags flags) error { + useDeltaEncoding := flags.have(useDeltaEncoding) + coalesceBoolFields := flags.have(coalesceBoolFields) + numFields := int16(0) + lastFieldID := int16(0) + +encodeFields: + for _, f := range enc.fields { + x := v + for _, i := range f.index { + if x.Kind() == reflect.Ptr { + x = x.Elem() + } + if x = x.Field(i); x.Kind() == reflect.Ptr { + if x.IsNil() { + continue encodeFields + } + } + } + + if !f.flags.have(required) && x.IsZero() { + continue encodeFields + } + + field := Field{ + ID: f.id, + Type: f.typ, + } + + if useDeltaEncoding { + if delta := field.ID - lastFieldID; delta <= 15 { + field.ID = delta + field.Delta = true + } + } + + skipValue := coalesceBoolFields && field.Type == BOOL + if skipValue && isTrue(x) == true { + field.Type = TRUE + } + + if err := w.WriteField(field); err != nil { + return err + } + + if !skipValue { + if err := f.encode(w, x, flags); err != nil { + return err + } + } + + numFields++ + lastFieldID = f.id + } + + if err := w.WriteField(Field{Type: STOP}); err != nil { + return err + } + + if numFields > 1 && enc.union { + return fmt.Errorf("thrift union had more than one field with a non-zero value (%d)", numFields) + } + + return nil +} + +func (enc *structEncoder) String() string { + if enc.union { + return "union" + } + return "struct" +} + +type structEncoderField struct { + index []int + id int16 + flags flags + typ Type + encode encodeFunc +} + +func encodeFuncStructOf(t reflect.Type, seen encodeFuncCache) encodeFunc { + enc := &structEncoder{ + fields: make([]structEncoderField, 0, t.NumField()), + } + encode := enc.encode + seen[t] = encode + + forEachStructField(t, nil, func(f structField) { + if f.flags.have(union) { + enc.union = true + } else { + enc.fields = append(enc.fields, structEncoderField{ + index: f.index, + id: f.id, + flags: f.flags, + typ: TypeOf(f.typ), + encode: encodeFuncStructFieldOf(f, seen), + }) + } + }) + + slices.SortStableFunc(enc.fields, func(a, b structEncoderField) int { + return cmp.Compare(a.id, b.id) + }) + + for i := len(enc.fields) - 1; i > 0; i-- { + if enc.fields[i-1].id == enc.fields[i].id { + panic(fmt.Errorf("thrift struct field id %d is present multiple times", enc.fields[i].id)) + } + } + + return encode +} + +func encodeFuncStructFieldOf(f structField, seen encodeFuncCache) encodeFunc { + if f.flags.have(enum) { + switch f.typ.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return encodeInt32 + } + } + return encodeFuncOf(f.typ, seen) +} + +func encodeFuncPtrOf(t reflect.Type, seen encodeFuncCache) encodeFunc { + typ := t.Elem() + enc := encodeFuncOf(typ, seen) + zero := reflect.Zero(typ) + + return func(w Writer, v reflect.Value, f flags) error { + if v.IsNil() { + v = zero + } else { + v = v.Elem() + } + return enc(w, v, f) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/error.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/error.go new file mode 100644 index 0000000000..ceeb1ba09b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/error.go @@ -0,0 +1,111 @@ +package thrift + +import ( + "errors" + "fmt" + "io" + "strings" +) + +type MissingField struct { + Field Field +} + +func (e *MissingField) Error() string { + return fmt.Sprintf("missing required field: %s", e.Field) +} + +type TypeMismatch struct { + Expect Type + Found Type + item string +} + +func (e *TypeMismatch) Error() string { + return fmt.Sprintf("%s type mismatch: expected %s but found %s", e.item, e.Expect, e.Found) +} + +type decodeError struct { + base error + path []error +} + +func (e *decodeError) Error() string { + s := strings.Builder{} + s.Grow(256) + s.WriteString("decoding thrift payload: ") + + if len(e.path) != 0 { + n := len(e.path) - 1 + for i := n; i >= 0; i-- { + if i < n { + s.WriteString(" โ†’ ") + } + s.WriteString(e.path[i].Error()) + } + s.WriteString(": ") + } + + s.WriteString(e.base.Error()) + return s.String() +} + +func (e *decodeError) Unwrap() error { return e.base } + +func with(base, elem error) error { + if errors.Is(base, io.EOF) { + return base + } + e, _ := base.(*decodeError) + if e == nil { + e = &decodeError{base: base} + } + e.path = append(e.path, elem) + return e +} + +type decodeErrorField struct { + cause Field +} + +func (d *decodeErrorField) Error() string { + return d.cause.String() +} + +type decodeErrorList struct { + cause List + index int +} + +func (d *decodeErrorList) Error() string { + return fmt.Sprintf("%d/%d:%s", d.index, d.cause.Size, d.cause) +} + +type decodeErrorSet struct { + cause Set + index int +} + +func (d *decodeErrorSet) Error() string { + return fmt.Sprintf("%d/%d:%s", d.index, d.cause.Size, d.cause) +} + +type decodeErrorMap struct { + cause Map + index int +} + +func (d *decodeErrorMap) Error() string { + return fmt.Sprintf("%d/%d:%s", d.index, d.cause.Size, d.cause) +} + +func dontExpectEOF(err error) error { + switch err { + case nil: + return nil + case io.EOF: + return io.ErrUnexpectedEOF + default: + return err + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/protocol.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/protocol.go new file mode 100644 index 0000000000..7c31338cf6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/protocol.go @@ -0,0 +1,73 @@ +package thrift + +import ( + "io" +) + +// Features is a bitset describing the thrift encoding features supported by +// protocol implementations. +type Features uint + +const ( + // DeltaEncoding is advertised by protocols that allow encoders to apply + // delta encoding on struct fields. + UseDeltaEncoding Features = 1 << iota + + // CoalesceBoolFields is advertised by protocols that allow encoders to + // coalesce boolean values into field types. + CoalesceBoolFields +) + +// The Protocol interface abstracts the creation of low-level thrift readers and +// writers implementing the various protocols that the encoding supports. +// +// Protocol instances must be safe to use concurrently from multiple gourintes. +// However, the readers and writer that they instantiates are intended to be +// used by a single goroutine. +type Protocol interface { + NewReader(r io.Reader) Reader + NewWriter(w io.Writer) Writer + Features() Features +} + +// Reader represents a low-level reader of values encoded according to one of +// the thrift protocols. +type Reader interface { + Protocol() Protocol + Reader() io.Reader + ReadBool() (bool, error) + ReadInt8() (int8, error) + ReadInt16() (int16, error) + ReadInt32() (int32, error) + ReadInt64() (int64, error) + ReadFloat64() (float64, error) + ReadBytes() ([]byte, error) + ReadString() (string, error) + ReadLength() (int, error) + ReadMessage() (Message, error) + ReadField() (Field, error) + ReadList() (List, error) + ReadSet() (Set, error) + ReadMap() (Map, error) +} + +// Writer represents a low-level writer of values encoded according to one of +// the thrift protocols. +type Writer interface { + Protocol() Protocol + Writer() io.Writer + WriteBool(bool) error + WriteInt8(int8) error + WriteInt16(int16) error + WriteInt32(int32) error + WriteInt64(int64) error + WriteFloat64(float64) error + WriteBytes([]byte) error + WriteString(string) error + WriteLength(int) error + WriteMessage(Message) error + WriteField(Field) error + WriteList(List) error + WriteSet(Set) error + WriteMap(Map) error +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/struct.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/struct.go new file mode 100644 index 0000000000..aa556f3f89 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/struct.go @@ -0,0 +1,140 @@ +package thrift + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +type flags int16 + +const ( + enum flags = 1 << 0 + union flags = 1 << 1 + required flags = 1 << 2 + optional flags = 1 << 3 + strict flags = 1 << 4 + + featuresBitOffset = 8 + useDeltaEncoding = flags(UseDeltaEncoding) << featuresBitOffset + coalesceBoolFields = flags(CoalesceBoolFields) << featuresBitOffset + + structFlags flags = enum | union | required | optional + encodeFlags flags = strict | protocolFlags + decodeFlags flags = strict | protocolFlags + protocolFlags flags = useDeltaEncoding | coalesceBoolFields +) + +func (f flags) have(x flags) bool { + return (f & x) == x +} + +func (f flags) only(x flags) flags { + return f & x +} + +func (f flags) with(x flags) flags { + return f | x +} + +func (f flags) without(x flags) flags { + return f & ^x +} + +type structField struct { + typ reflect.Type + index []int + id int16 + flags flags +} + +func forEachStructField(t reflect.Type, index []int, do func(structField)) { + for i, n := 0, t.NumField(); i < n; i++ { + f := t.Field(i) + + if f.PkgPath != "" && !f.Anonymous { // unexported + continue + } + + fieldIndex := append(index, i) + fieldIndex = fieldIndex[:len(fieldIndex):len(fieldIndex)] + + if f.Anonymous { + fieldType := f.Type + + for fieldType.Kind() == reflect.Ptr { + fieldType = fieldType.Elem() + } + + if fieldType.Kind() == reflect.Struct { + forEachStructField(fieldType, fieldIndex, do) + continue + } + } + + tag := f.Tag.Get("thrift") + if tag == "" { + continue + } + tags := strings.Split(tag, ",") + flags := flags(0) + + for _, opt := range tags[1:] { + switch opt { + case "enum": + flags = flags.with(enum) + case "union": + flags = flags.with(union) + case "required": + flags = flags.with(required) + case "optional": + flags = flags.with(optional) + default: + panic(fmt.Errorf("thrift struct field contains an unknown tag option %q in `thrift:\"%s\"`", opt, tag)) + } + } + + if flags.have(optional | required) { + panic(fmt.Errorf("thrift struct field cannot be both optional and required in `thrift:\"%s\"`", tag)) + } + + if flags.have(union) { + if f.Type.Kind() != reflect.Interface { + panic(fmt.Errorf("thrift union tag found on a field which is not an interface type `thrift:\"%s\"`", tag)) + } + + if tags[0] != "" { + panic(fmt.Errorf("invalid thrift field id on union field `thrift:\"%s\"`", tag)) + } + + do(structField{ + typ: f.Type, + index: fieldIndex, + flags: flags, + }) + } else { + if flags.have(enum) { + switch f.Type.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + panic(fmt.Errorf("thrift enum tag found on a field which is not an integer type `thrift:\"%s\"`", tag)) + } + } + + if id, err := strconv.ParseInt(tags[0], 10, 16); err != nil { + panic(fmt.Errorf("invalid thrift field id found in struct tag `thrift:\"%s\"`: %w", tag, err)) + } else if id <= 0 { + panic(fmt.Errorf("invalid thrift field id found in struct tag `thrift:\"%s\"`: %d <= 0", tag, id)) + } else { + do(structField{ + typ: f.Type, + index: fieldIndex, + id: int16(id), + flags: flags, + }) + } + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/thrift.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/thrift.go new file mode 100644 index 0000000000..b3682e4d63 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/thrift.go @@ -0,0 +1,164 @@ +package thrift + +import ( + "fmt" + "reflect" +) + +type Message struct { + Type MessageType + Name string + SeqID int32 +} + +type MessageType int8 + +const ( + Call MessageType = iota + Reply + Exception + Oneway +) + +func (m MessageType) String() string { + switch m { + case Call: + return "Call" + case Reply: + return "Reply" + case Exception: + return "Exception" + case Oneway: + return "Oneway" + default: + return "?" + } +} + +type Field struct { + ID int16 + Type Type + Delta bool // whether the field id is a delta +} + +func (f Field) String() string { + return fmt.Sprintf("%d:FIELD<%s>", f.ID, f.Type) +} + +type Type int8 + +const ( + STOP Type = iota + TRUE + FALSE + I8 + I16 + I32 + I64 + DOUBLE + BINARY + LIST + SET + MAP + STRUCT + BOOL = FALSE +) + +func (t Type) String() string { + switch t { + case STOP: + return "STOP" + case TRUE: + return "TRUE" + case BOOL: + return "BOOL" + case I8: + return "I8" + case I16: + return "I16" + case I32: + return "I32" + case I64: + return "I64" + case DOUBLE: + return "DOUBLE" + case BINARY: + return "BINARY" + case LIST: + return "LIST" + case SET: + return "SET" + case MAP: + return "MAP" + case STRUCT: + return "STRUCT" + default: + return "?" + } +} + +func (t Type) GoString() string { + return "thrift." + t.String() +} + +type List struct { + Size int32 + Type Type +} + +func (l List) String() string { + return fmt.Sprintf("LIST<%s>", l.Type) +} + +type Set List + +func (s Set) String() string { + return fmt.Sprintf("SET<%s>", s.Type) +} + +type Map struct { + Size int32 + Key Type + Value Type +} + +func (m Map) String() string { + return fmt.Sprintf("MAP<%s,%s>", m.Key, m.Value) +} + +func TypeOf(t reflect.Type) Type { + switch t.Kind() { + case reflect.Bool: + return BOOL + case reflect.Int8, reflect.Uint8: + return I8 + case reflect.Int16, reflect.Uint16: + return I16 + case reflect.Int32, reflect.Uint32: + return I32 + case reflect.Int64, reflect.Uint64, reflect.Int, reflect.Uint, reflect.Uintptr: + return I64 + case reflect.Float32, reflect.Float64: + return DOUBLE + case reflect.String: + return BINARY + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { // []byte + return BINARY + } else { + return LIST + } + case reflect.Map: + if t.Elem().Size() == 0 { + return SET + } else { + return MAP + } + case reflect.Struct: + return STRUCT + case reflect.Ptr: + return TypeOf(t.Elem()) + default: + panic("type cannot be represented in thrift: " + t.String()) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/thrift/unsafe.go b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/unsafe.go new file mode 100644 index 0000000000..b27c6489d8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/thrift/unsafe.go @@ -0,0 +1,20 @@ +package thrift + +import ( + "reflect" + "unsafe" +) + +// typeID is used as key in encoder and decoder caches to enable using +// the optimize runtime.mapaccess2_fast64 function instead of the more +// expensive lookup if we were to use reflect.Type as map key. +// +// typeID holds the pointer to the reflect.Type value, which is unique +// in the program. +type typeID struct{ ptr unsafe.Pointer } + +func makeTypeID(t reflect.Type) typeID { + return typeID{ + ptr: (*[2]unsafe.Pointer)(unsafe.Pointer(&t))[1], + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/encoding/values.go b/vendor/github.com/parquet-go/parquet-go/encoding/values.go new file mode 100644 index 0000000000..41ab0a23e3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/encoding/values.go @@ -0,0 +1,276 @@ +package encoding + +import ( + "fmt" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type Kind int32 + +const ( + Undefined Kind = iota + Boolean + Int32 + Int64 + Int96 + Float + Double + ByteArray + FixedLenByteArray +) + +func (kind Kind) String() string { + switch kind { + case Boolean: + return "BOOLEAN" + case Int32: + return "INT32" + case Int64: + return "INT64" + case Int96: + return "INT96" + case Float: + return "FLOAT" + case Double: + return "DOUBLE" + case ByteArray: + return "BYTE_ARRAY" + case FixedLenByteArray: + return "FIXED_LEN_BYTE_ARRAY" + default: + return "UNDEFINED" + } +} + +type Values struct { + kind Kind + size int32 + data []byte + offsets []uint32 +} + +func (v *Values) assertKind(kind Kind) { + if kind != v.kind { + panic(fmt.Sprintf("cannot convert values of type %s to type %s", v.kind, kind)) + } +} + +func (v *Values) assertSize(size int) { + if size != int(v.size) { + panic(fmt.Sprintf("cannot convert values of size %d to size %d", v.size, size)) + } +} + +func (v *Values) Size() int64 { + return int64(len(v.data)) +} + +func (v *Values) Kind() Kind { + return v.kind +} + +func (v *Values) Data() (data []byte, offsets []uint32) { + return v.data, v.offsets +} + +func (v *Values) Boolean() []byte { + v.assertKind(Boolean) + return v.data +} + +func (v *Values) Int32() []int32 { + v.assertKind(Int32) + return unsafecast.Slice[int32](v.data) +} + +func (v *Values) Int64() []int64 { + v.assertKind(Int64) + return unsafecast.Slice[int64](v.data) +} + +func (v *Values) Int96() []deprecated.Int96 { + v.assertKind(Int96) + return unsafecast.Slice[deprecated.Int96](v.data) +} + +func (v *Values) Float() []float32 { + v.assertKind(Float) + return unsafecast.Slice[float32](v.data) +} + +func (v *Values) Double() []float64 { + v.assertKind(Double) + return unsafecast.Slice[float64](v.data) +} + +func (v *Values) ByteArray() (data []byte, offsets []uint32) { + v.assertKind(ByteArray) + return v.data, v.offsets +} + +func (v *Values) FixedLenByteArray() (data []byte, size int) { + v.assertKind(FixedLenByteArray) + return v.data, int(v.size) +} + +func (v *Values) Uint32() []uint32 { + v.assertKind(Int32) + return unsafecast.Slice[uint32](v.data) +} + +func (v *Values) Uint64() []uint64 { + v.assertKind(Int64) + return unsafecast.Slice[uint64](v.data) +} + +func (v *Values) Uint128() [][16]byte { + v.assertKind(FixedLenByteArray) + v.assertSize(16) + return unsafecast.Slice[[16]byte](v.data) +} + +func makeValues[T any](kind Kind, values []T) Values { + return Values{kind: kind, data: unsafecast.Slice[byte](values)} +} + +func BooleanValues(values []byte) Values { + return makeValues(Boolean, values) +} + +func Int32Values(values []int32) Values { + return makeValues(Int32, values) +} + +func Int64Values(values []int64) Values { + return makeValues(Int64, values) +} + +func Int96Values(values []deprecated.Int96) Values { + return makeValues(Int96, values) +} + +func FloatValues(values []float32) Values { + return makeValues(Float, values) +} + +func DoubleValues(values []float64) Values { + return makeValues(Double, values) +} + +func ByteArrayValues(values []byte, offsets []uint32) Values { + return Values{kind: ByteArray, data: values, offsets: offsets} +} + +func FixedLenByteArrayValues(values []byte, size int) Values { + return Values{kind: FixedLenByteArray, size: int32(size), data: values} +} + +func Uint32Values(values []uint32) Values { + return Int32Values(unsafecast.Slice[int32](values)) +} + +func Uint64Values(values []uint64) Values { + return Int64Values(unsafecast.Slice[int64](values)) +} + +func Uint128Values(values [][16]byte) Values { + return FixedLenByteArrayValues(unsafecast.Slice[byte](values), 16) +} + +func Int32ValuesFromBytes(values []byte) Values { + return Values{kind: Int32, data: values} +} + +func Int64ValuesFromBytes(values []byte) Values { + return Values{kind: Int64, data: values} +} + +func Int96ValuesFromBytes(values []byte) Values { + return Values{kind: Int96, data: values} +} + +func FloatValuesFromBytes(values []byte) Values { + return Values{kind: Float, data: values} +} + +func DoubleValuesFromBytes(values []byte) Values { + return Values{kind: Double, data: values} +} + +func EncodeBoolean(dst []byte, src Values, enc Encoding) ([]byte, error) { + return enc.EncodeBoolean(dst, src.Boolean()) +} + +func EncodeInt32(dst []byte, src Values, enc Encoding) ([]byte, error) { + return enc.EncodeInt32(dst, src.Int32()) +} + +func EncodeInt64(dst []byte, src Values, enc Encoding) ([]byte, error) { + return enc.EncodeInt64(dst, src.Int64()) +} + +func EncodeInt96(dst []byte, src Values, enc Encoding) ([]byte, error) { + return enc.EncodeInt96(dst, src.Int96()) +} + +func EncodeFloat(dst []byte, src Values, enc Encoding) ([]byte, error) { + return enc.EncodeFloat(dst, src.Float()) +} + +func EncodeDouble(dst []byte, src Values, enc Encoding) ([]byte, error) { + return enc.EncodeDouble(dst, src.Double()) +} + +func EncodeByteArray(dst []byte, src Values, enc Encoding) ([]byte, error) { + values, offsets := src.ByteArray() + return enc.EncodeByteArray(dst, values, offsets) +} + +func EncodeFixedLenByteArray(dst []byte, src Values, enc Encoding) ([]byte, error) { + data, size := src.FixedLenByteArray() + return enc.EncodeFixedLenByteArray(dst, data, size) +} + +func DecodeBoolean(dst Values, src []byte, enc Encoding) (Values, error) { + values, err := enc.DecodeBoolean(dst.Boolean(), src) + return BooleanValues(values), err +} + +func DecodeInt32(dst Values, src []byte, enc Encoding) (Values, error) { + values, err := enc.DecodeInt32(dst.Int32(), src) + return Int32Values(values), err +} + +func DecodeInt64(dst Values, src []byte, enc Encoding) (Values, error) { + values, err := enc.DecodeInt64(dst.Int64(), src) + return Int64Values(values), err +} + +func DecodeInt96(dst Values, src []byte, enc Encoding) (Values, error) { + values, err := enc.DecodeInt96(dst.Int96(), src) + return Int96Values(values), err +} + +func DecodeFloat(dst Values, src []byte, enc Encoding) (Values, error) { + values, err := enc.DecodeFloat(dst.Float(), src) + return FloatValues(values), err +} + +func DecodeDouble(dst Values, src []byte, enc Encoding) (Values, error) { + values, err := enc.DecodeDouble(dst.Double(), src) + return DoubleValues(values), err +} + +func DecodeByteArray(dst Values, src []byte, enc Encoding) (Values, error) { + values, offsets := dst.ByteArray() + values, offsets, err := enc.DecodeByteArray(values, src, offsets) + return ByteArrayValues(values, offsets), err +} + +func DecodeFixedLenByteArray(dst Values, src []byte, enc Encoding) (Values, error) { + data, size := dst.FixedLenByteArray() + values, err := enc.DecodeFixedLenByteArray(data, src, size) + return FixedLenByteArrayValues(values, size), err +} diff --git a/vendor/github.com/parquet-go/parquet-go/errors.go b/vendor/github.com/parquet-go/parquet-go/errors.go new file mode 100644 index 0000000000..651fe740c8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/errors.go @@ -0,0 +1,87 @@ +package parquet + +import ( + "errors" + "fmt" +) + +var ( + // ErrCorrupted is an error returned by the Err method of ColumnPages + // instances when they encountered a mismatch between the CRC checksum + // recorded in a page header and the one computed while reading the page + // data. + ErrCorrupted = errors.New("corrupted parquet page") + + // ErrMissingRootColumn is an error returned when opening an invalid parquet + // file which does not have a root column. + ErrMissingRootColumn = errors.New("parquet file is missing a root column") + + // ErrRowGroupSchemaMissing is an error returned when attempting to write a + // row group but the source has no schema. + ErrRowGroupSchemaMissing = errors.New("cannot write rows to a row group which has no schema") + + // ErrRowGroupSchemaMismatch is an error returned when attempting to write a + // row group but the source and destination schemas differ. + ErrRowGroupSchemaMismatch = errors.New("cannot write row groups with mismatching schemas") + + // ErrRowGroupSortingColumnsMismatch is an error returned when attempting to + // write a row group but the sorting columns differ in the source and + // destination. + ErrRowGroupSortingColumnsMismatch = errors.New("cannot write row groups with mismatching sorting columns") + + // ErrSeekOutOfRange is an error returned when seeking to a row index which + // is less than the first row of a page. + ErrSeekOutOfRange = errors.New("seek to row index out of page range") + + // ErrUnexpectedDictionaryPage is an error returned when a page reader + // encounters a dictionary page after the first page, or in a column + // which does not use a dictionary encoding. + ErrUnexpectedDictionaryPage = errors.New("unexpected dictionary page") + + // ErrMissingPageHeader is an error returned when a page reader encounters + // a malformed page header which is missing page-type-specific information. + ErrMissingPageHeader = errors.New("missing page header") + + // ErrUnexpectedRepetitionLevels is an error returned when attempting to + // decode repetition levels into a page which is not part of a repeated + // column. + ErrUnexpectedRepetitionLevels = errors.New("unexpected repetition levels") + + // ErrUnexpectedDefinitionLevels is an error returned when attempting to + // decode definition levels into a page which is part of a required column. + ErrUnexpectedDefinitionLevels = errors.New("unexpected definition levels") + + // ErrTooManyRowGroups is returned when attempting to generate a parquet + // file with more than MaxRowGroups row groups. + ErrTooManyRowGroups = errors.New("the limit of 32767 row groups has been reached") + + // ErrConversion is used to indicate that a conversion betwen two values + // cannot be done because there are no rules to translate between their + // physical types. + ErrInvalidConversion = errors.New("invalid conversion between parquet values") + + // ErrMalformedRepetitionLevel is returned when a page reader encounters + // a repetition level which does not start at the beginning of a row. + ErrMalformedRepetitionLevel = errors.New("parquet-go encountered a malformed data page which does not start at the beginning of a row") +) + +type errno int + +const ( + ok errno = iota + indexOutOfBounds +) + +func (e errno) check() { + switch e { + case ok: + case indexOutOfBounds: + panic("index out of bounds") + default: + panic("BUG: unknown error code") + } +} + +func errRowIndexOutOfBounds(rowIndex, rowCount int64) error { + return fmt.Errorf("row index out of bounds: %d/%d", rowIndex, rowCount) +} diff --git a/vendor/github.com/parquet-go/parquet-go/file.go b/vendor/github.com/parquet-go/parquet-go/file.go new file mode 100644 index 0000000000..62576c45c3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/file.go @@ -0,0 +1,1078 @@ +package parquet + +import ( + "bufio" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + + "github.com/parquet-go/parquet-go/encoding/thrift" + "github.com/parquet-go/parquet-go/format" +) + +const ( + defaultDictBufferSize = 8192 + defaultReadBufferSize = 4096 +) + +// File represents a parquet file. The layout of a Parquet file can be found +// here: https://github.com/apache/parquet-format#file-format +type File struct { + metadata format.FileMetaData + protocol thrift.CompactProtocol + reader io.ReaderAt + size int64 + schema *Schema + root *Column + columnIndexes []format.ColumnIndex + offsetIndexes []format.OffsetIndex + rowGroups []RowGroup + config *FileConfig +} + +type FileView interface { + Metadata() *format.FileMetaData + Schema() *Schema + NumRows() int64 + Lookup(key string) (string, bool) + Size() int64 + Root() *Column + RowGroups() []RowGroup + ColumnIndexes() []format.ColumnIndex + OffsetIndexes() []format.OffsetIndex +} + +// OpenFile opens a parquet file and reads the content between offset 0 and the given +// size in r. +// +// Only the parquet magic bytes and footer are read, column chunks and other +// parts of the file are left untouched; this means that successfully opening +// a file does not validate that the pages have valid checksums. +func OpenFile(r io.ReaderAt, size int64, options ...FileOption) (*File, error) { + c, err := NewFileConfig(options...) + if err != nil { + return nil, err + } + f := &File{reader: r, size: size, config: c} + + if !c.SkipMagicBytes { + var b [4]byte + if _, err := readAt(r, b[:4], 0); err != nil { + return nil, fmt.Errorf("reading magic header of parquet file: %w", err) + } + if string(b[:4]) != "PAR1" { + return nil, fmt.Errorf("invalid magic header of parquet file: %q", b[:4]) + } + } + + if cast, ok := f.reader.(interface{ SetMagicFooterSection(offset, length int64) }); ok { + cast.SetMagicFooterSection(size-8, 8) + } + + optimisticRead := c.OptimisticRead + optimisticFooterSize := min(int64(c.ReadBufferSize), size) + if !optimisticRead || optimisticFooterSize < 8 { + optimisticFooterSize = 8 + } + optimisticFooterData := make([]byte, optimisticFooterSize) + if optimisticRead { + f.reader = &optimisticFileReaderAt{ + reader: f.reader, + offset: size - optimisticFooterSize, + footer: optimisticFooterData, + } + } + + if n, err := readAt(r, optimisticFooterData, size-optimisticFooterSize); n != len(optimisticFooterData) { + return nil, fmt.Errorf("reading magic footer of parquet file: %w (read: %d)", err, n) + } + optimisticFooterSize -= 8 + b := optimisticFooterData[optimisticFooterSize:] + if string(b[4:]) != "PAR1" { + return nil, fmt.Errorf("invalid magic footer of parquet file: %q", b[4:]) + } + + footerSize := int64(binary.LittleEndian.Uint32(b[:4])) + footerData := []byte(nil) + + if footerSize <= optimisticFooterSize { + footerData = optimisticFooterData[optimisticFooterSize-footerSize : optimisticFooterSize] + } else { + footerData = make([]byte, footerSize) + if cast, ok := f.reader.(interface{ SetFooterSection(offset, length int64) }); ok { + cast.SetFooterSection(size-(footerSize+8), footerSize) + } + if _, err := f.readAt(footerData, size-(footerSize+8)); err != nil { + return nil, fmt.Errorf("reading footer of parquet file: %w", err) + } + } + + if err := thrift.Unmarshal(&f.protocol, footerData, &f.metadata); err != nil { + return nil, fmt.Errorf("reading parquet file metadata: %w", err) + } + if len(f.metadata.Schema) == 0 { + return nil, ErrMissingRootColumn + } + + if !c.SkipPageIndex { + if f.columnIndexes, f.offsetIndexes, err = f.ReadPageIndex(); err != nil { + return nil, fmt.Errorf("reading page index of parquet file: %w", err) + } + } + + if f.root, err = openColumns(f, &f.metadata, f.columnIndexes, f.offsetIndexes); err != nil { + return nil, fmt.Errorf("opening columns of parquet file: %w", err) + } + + if c.Schema != nil { + f.schema = c.Schema + } else { + f.schema = NewSchema(f.root.Name(), f.root) + } + columns := makeLeafColumns(f.root) + rowGroups := makeFileRowGroups(f, columns) + f.rowGroups = makeRowGroups(rowGroups) + + if !c.SkipBloomFilters { + section := io.NewSectionReader(r, 0, size) + rbuf, rbufpool := getBufioReader(section, c.ReadBufferSize) + defer putBufioReader(rbuf, rbufpool) + + header := format.BloomFilterHeader{} + compact := thrift.CompactProtocol{} + decoder := thrift.NewDecoder(compact.NewReader(rbuf)) + + for i := range rowGroups { + g := &rowGroups[i] + + for j := range g.columns { + c := g.columns[j].(*FileColumnChunk) + + if offset := c.chunk.MetaData.BloomFilterOffset; offset > 0 { + section.Seek(offset, io.SeekStart) + rbuf.Reset(section) + + header = format.BloomFilterHeader{} + if err := decoder.Decode(&header); err != nil { + return nil, fmt.Errorf("decoding bloom filter header: %w", err) + } + + offset, _ = section.Seek(0, io.SeekCurrent) + offset -= int64(rbuf.Buffered()) + + if cast, ok := r.(interface{ SetBloomFilterSection(offset, length int64) }); ok { + bloomFilterOffset := c.chunk.MetaData.BloomFilterOffset + bloomFilterLength := (offset - bloomFilterOffset) + int64(header.NumBytes) + cast.SetBloomFilterSection(bloomFilterOffset, bloomFilterLength) + } + + c.bloomFilter.Store(newBloomFilter(r, offset, &header)) + } + } + } + } + + sortKeyValueMetadata(f.metadata.KeyValueMetadata) + f.reader = r // restore in case an optimistic reader was used + return f, nil +} + +// ReadPageIndex reads the page index section of the parquet file f. +// +// If the file did not contain a page index, the method returns two empty slices +// and a nil error. +// +// Only leaf columns have indexes, the returned indexes are arranged using the +// following layout: +// +// ------------------ +// | col 0: chunk 0 | +// ------------------ +// | col 1: chunk 0 | +// ------------------ +// | ... | +// ------------------ +// | col 0: chunk 1 | +// ------------------ +// | col 1: chunk 1 | +// ------------------ +// | ... | +// ------------------ +// +// This method is useful in combination with the SkipPageIndex option to delay +// reading the page index section until after the file was opened. Note that in +// this case the page index is not cached within the file, programs are expected +// to make use of independently from the parquet package. +func (f *File) ReadPageIndex() ([]format.ColumnIndex, []format.OffsetIndex, error) { + if len(f.metadata.RowGroups) == 0 { + return nil, nil, nil + } + + columnIndexOffset := f.metadata.RowGroups[0].Columns[0].ColumnIndexOffset + offsetIndexOffset := f.metadata.RowGroups[0].Columns[0].OffsetIndexOffset + columnIndexLength := int64(0) + offsetIndexLength := int64(0) + + forEachColumnChunk := func(do func(int, int, *format.ColumnChunk) error) error { + for i := range f.metadata.RowGroups { + for j := range f.metadata.RowGroups[i].Columns { + c := &f.metadata.RowGroups[i].Columns[j] + if err := do(i, j, c); err != nil { + return err + } + } + } + return nil + } + + forEachColumnChunk(func(_, _ int, c *format.ColumnChunk) error { + columnIndexLength += int64(c.ColumnIndexLength) + offsetIndexLength += int64(c.OffsetIndexLength) + return nil + }) + + if columnIndexLength == 0 && offsetIndexLength == 0 { + return nil, nil, nil + } + + numRowGroups := len(f.metadata.RowGroups) + numColumns := len(f.metadata.RowGroups[0].Columns) + numColumnChunks := numRowGroups * numColumns + + columnIndexes := make([]format.ColumnIndex, numColumnChunks) + offsetIndexes := make([]format.OffsetIndex, numColumnChunks) + indexBuffer := make([]byte, max(int(columnIndexLength), int(offsetIndexLength))) + + if columnIndexOffset > 0 { + columnIndexData := indexBuffer[:columnIndexLength] + + if cast, ok := f.reader.(interface{ SetColumnIndexSection(offset, length int64) }); ok { + cast.SetColumnIndexSection(columnIndexOffset, columnIndexLength) + } + if _, err := f.readAt(columnIndexData, columnIndexOffset); err != nil { + return nil, nil, fmt.Errorf("reading %d bytes column index at offset %d: %w", columnIndexLength, columnIndexOffset, err) + } + + err := forEachColumnChunk(func(i, j int, c *format.ColumnChunk) error { + // Some parquet files are missing the column index on some columns. + // + // An example of this file is testdata/alltypes_tiny_pages_plain.parquet + // which was added in https://github.com/apache/parquet-testing/pull/24. + if c.ColumnIndexOffset > 0 { + offset := c.ColumnIndexOffset - columnIndexOffset + length := int64(c.ColumnIndexLength) + buffer := columnIndexData[offset : offset+length] + if err := thrift.Unmarshal(&f.protocol, buffer, &columnIndexes[(i*numColumns)+j]); err != nil { + return fmt.Errorf("decoding column index: rowGroup=%d columnChunk=%d/%d: %w", i, j, numColumns, err) + } + } + return nil + }) + if err != nil { + return nil, nil, err + } + } + + if offsetIndexOffset > 0 { + offsetIndexData := indexBuffer[:offsetIndexLength] + + if cast, ok := f.reader.(interface{ SetOffsetIndexSection(offset, length int64) }); ok { + cast.SetOffsetIndexSection(offsetIndexOffset, offsetIndexLength) + } + if _, err := f.readAt(offsetIndexData, offsetIndexOffset); err != nil { + return nil, nil, fmt.Errorf("reading %d bytes offset index at offset %d: %w", offsetIndexLength, offsetIndexOffset, err) + } + + err := forEachColumnChunk(func(i, j int, c *format.ColumnChunk) error { + if c.OffsetIndexOffset > 0 { + offset := c.OffsetIndexOffset - offsetIndexOffset + length := int64(c.OffsetIndexLength) + buffer := offsetIndexData[offset : offset+length] + if err := thrift.Unmarshal(&f.protocol, buffer, &offsetIndexes[(i*numColumns)+j]); err != nil { + return fmt.Errorf("decoding column index: rowGroup=%d columnChunk=%d/%d: %w", i, j, numColumns, err) + } + } + return nil + }) + if err != nil { + return nil, nil, err + } + } + + return columnIndexes, offsetIndexes, nil +} + +// NumRows returns the number of rows in the file. +func (f *File) NumRows() int64 { return f.metadata.NumRows } + +// RowGroups returns the list of row groups in the file. +// +// Elements of the returned slice are guaranteed to be of type *FileRowGroup. +func (f *File) RowGroups() []RowGroup { return f.rowGroups } + +// Root returns the root column of f. +func (f *File) Root() *Column { return f.root } + +// Schema returns the schema of f. +func (f *File) Schema() *Schema { return f.schema } + +// Metadata returns the metadata of f. +func (f *File) Metadata() *format.FileMetaData { return &f.metadata } + +// Size returns the size of f (in bytes). +func (f *File) Size() int64 { return f.size } + +// ReadAt reads bytes into b from f at the given offset. +// +// The method satisfies the io.ReaderAt interface. +func (f *File) ReadAt(b []byte, off int64) (int, error) { + if off < 0 || off >= f.size { + return 0, io.EOF + } + + if limit := f.size - off; limit < int64(len(b)) { + n, err := f.readAt(b[:limit], off) + if err == nil { + err = io.EOF + } + return n, err + } + + return f.readAt(b, off) +} + +// ColumnIndexes returns the page index of the parquet file f. +// +// If the file did not contain a column index, the method returns an empty slice. +func (f *File) ColumnIndexes() []format.ColumnIndex { return f.columnIndexes } + +// OffsetIndexes returns the page index of the parquet file f. +// +// If the file did not contain an offset index, the method returns an empty +// slice. +func (f *File) OffsetIndexes() []format.OffsetIndex { return f.offsetIndexes } + +// Lookup returns the value associated with the given key in the file key/value +// metadata. +// +// The ok boolean will be true if the key was found, false otherwise. +func (f *File) Lookup(key string) (value string, ok bool) { + return lookupKeyValueMetadata(f.metadata.KeyValueMetadata, key) +} + +func (f *File) hasIndexes() bool { + return f.columnIndexes != nil && f.offsetIndexes != nil +} + +var _ io.ReaderAt = (*File)(nil) + +func sortKeyValueMetadata(keyValueMetadata []format.KeyValue) { + slices.SortFunc(keyValueMetadata, func(a, b format.KeyValue) int { + if cmp := strings.Compare(a.Key, b.Key); cmp != 0 { + return cmp + } + return strings.Compare(a.Value, b.Value) + }) +} + +func lookupKeyValueMetadata(keyValueMetadata []format.KeyValue, key string) (value string, ok bool) { + i, found := slices.BinarySearchFunc(keyValueMetadata, key, func(kv format.KeyValue, key string) int { + return strings.Compare(kv.Key, key) + }) + if found { + return keyValueMetadata[i].Value, true + } + return "", false +} + +// FileRowGroup is an implementation of the RowGroup interface on parquet files +// returned by OpenFile. +type FileRowGroup struct { + file *File + rowGroup *format.RowGroup + columns []ColumnChunk + sorting []SortingColumn +} + +func (g *FileRowGroup) init(file *File, columns []*Column, rowGroup *format.RowGroup) { + g.file = file + g.rowGroup = rowGroup + g.columns = make([]ColumnChunk, len(rowGroup.Columns)) + g.sorting = make([]SortingColumn, len(rowGroup.SortingColumns)) + fileColumnChunks := make([]FileColumnChunk, len(rowGroup.Columns)) + fileColumnIndexes := make([]FileColumnIndex, len(rowGroup.Columns)) + fileOffsetIndexes := make([]FileOffsetIndex, len(rowGroup.Columns)) + + for i := range g.columns { + fileColumnChunks[i] = FileColumnChunk{ + file: file, + column: columns[i], + rowGroup: rowGroup, + chunk: &rowGroup.Columns[i], + } + + if file.hasIndexes() { + j := (int(rowGroup.Ordinal) * len(columns)) + i + + fileColumnIndexes[i] = FileColumnIndex{index: &file.columnIndexes[j], kind: columns[i].Type().Kind()} + fileOffsetIndexes[i] = FileOffsetIndex{index: &file.offsetIndexes[j]} + + fileColumnChunks[i].columnIndex.Store(&fileColumnIndexes[i]) + fileColumnChunks[i].offsetIndex.Store(&fileOffsetIndexes[i]) + } + + g.columns[i] = &fileColumnChunks[i] + } + + for i := range g.sorting { + g.sorting[i] = &fileSortingColumn{ + column: columns[rowGroup.SortingColumns[i].ColumnIdx], + descending: rowGroup.SortingColumns[i].Descending, + nullsFirst: rowGroup.SortingColumns[i].NullsFirst, + } + } +} + +// File returns the file that this row group belongs to. +func (g *FileRowGroup) File() *File { return g.file } + +// Schema returns the schema of the row group. +func (g *FileRowGroup) Schema() *Schema { return g.file.schema } + +// NumRows returns the number of rows in the row group. +func (g *FileRowGroup) NumRows() int64 { return g.rowGroup.NumRows } + +// ColumnChunks returns the list of column chunks in the row group. +// +// Elements of the returned slice are guaranteed to be of type *FileColumnChunk. +func (g *FileRowGroup) ColumnChunks() []ColumnChunk { return g.columns } + +// SortingColumns returns the list of sorting columns in the row group. +func (g *FileRowGroup) SortingColumns() []SortingColumn { return g.sorting } + +// Rows returns a row reader for the row group. +func (g *FileRowGroup) Rows() Rows { + rowGroup := RowGroup(g) + if g.file.config.ReadMode == ReadModeAsync { + rowGroup = AsyncRowGroup(rowGroup) + } + return NewRowGroupRowReader(rowGroup) +} + +type fileSortingColumn struct { + column *Column + descending bool + nullsFirst bool +} + +func (s *fileSortingColumn) Path() []string { return s.column.Path() } +func (s *fileSortingColumn) Descending() bool { return s.descending } +func (s *fileSortingColumn) NullsFirst() bool { return s.nullsFirst } +func (s *fileSortingColumn) String() string { + b := new(strings.Builder) + if s.nullsFirst { + b.WriteString("nulls_first+") + } + if s.descending { + b.WriteString("descending(") + } else { + b.WriteString("ascending(") + } + b.WriteString(columnPath(s.Path()).String()) + b.WriteString(")") + return b.String() +} + +// FileColumnChunk is an implementation of the ColumnChunk interface on parquet +// files returned by OpenFile. +type FileColumnChunk struct { + file *File + column *Column + rowGroup *format.RowGroup + chunk *format.ColumnChunk + columnIndex atomic.Pointer[FileColumnIndex] + offsetIndex atomic.Pointer[FileOffsetIndex] + bloomFilter atomic.Pointer[FileBloomFilter] +} + +// File returns the file that this column chunk belongs to. +func (c *FileColumnChunk) File() *File { return c.file } + +// Node returns the node that this column chunk belongs to in the parquet schema. +func (c *FileColumnChunk) Node() Node { return c.column } + +// Type returns the type of the column chunk. +func (c *FileColumnChunk) Type() Type { return c.column.Type() } + +// Column returns the column index of this chunk in its parent row group. +func (c *FileColumnChunk) Column() int { return int(c.column.Index()) } + +// Bounds returns the min and max values found in the column chunk. +func (c *FileColumnChunk) Bounds() (min, max Value, ok bool) { + stats := &c.chunk.MetaData.Statistics + columnKind := c.Type().Kind() + hasMinValue := stats.MinValue != nil + hasMaxValue := stats.MaxValue != nil + if hasMinValue { + min = columnKind.Value(stats.MinValue) + } + if hasMaxValue { + max = columnKind.Value(stats.MaxValue) + } + return min, max, hasMinValue && hasMaxValue +} + +// Pages returns a page reader for the column chunk. +func (c *FileColumnChunk) Pages() Pages { + pages := Pages(c.PagesFrom(c.file.reader, c.file.config.ReadBufferSize)) + if c.file.config.ReadMode == ReadModeAsync { + pages = AsyncPages(pages) + } + return pages +} + +// PagesFrom returns a page reader for the column chunk, using the reader passed +// as argument instead of the one that the file was originally opened from. +// +// Note that unlike when calling Pages, the returned reader is not wrapped in an +// AsyncPages reader if the file was opened in async mode. +func (c *FileColumnChunk) PagesFrom(reader io.ReaderAt, bufferSize int) *FilePages { + pages := new(FilePages) + pages.init(c, bufferSize, reader) + return pages +} + +// ColumnIndex returns the column index of the column chunk, or an error if it +// didn't exist or couldn't be read. +func (c *FileColumnChunk) ColumnIndex() (ColumnIndex, error) { + index, err := c.ColumnIndexFrom(c.file.reader) + if err != nil { + return nil, err + } + return index, nil +} + +// ColumnIndexFrom is like ColumnIndex but uses the reader passed as argument to +// read the column index. +func (c *FileColumnChunk) ColumnIndexFrom(reader io.ReaderAt) (*FileColumnIndex, error) { + index, err := c.readColumnIndexFrom(reader) + if err != nil { + return nil, err + } + if index == nil || c.chunk.ColumnIndexOffset == 0 { + return nil, ErrMissingColumnIndex + } + return index, nil +} + +// OffsetIndex returns the offset index of the column chunk, or an error if it +// didn't exist or couldn't be read. +func (c *FileColumnChunk) OffsetIndex() (OffsetIndex, error) { + index, err := c.OffsetIndexFrom(c.file.reader) + if err != nil { + return nil, err + } + return index, nil +} + +// OffsetIndexFrom is like OffsetIndex but uses the reader passed as argument to +// read the offset index. +func (c *FileColumnChunk) OffsetIndexFrom(reader io.ReaderAt) (*FileOffsetIndex, error) { + index, err := c.readOffsetIndex(reader) + if err != nil { + return nil, err + } + if index == nil || c.chunk.OffsetIndexOffset == 0 { + return nil, ErrMissingOffsetIndex + } + return index, nil +} + +// BloomFilter returns the bloom filter of the column chunk, or nil if it didn't +// have one. +func (c *FileColumnChunk) BloomFilter() BloomFilter { + filter, err := c.BloomFilterFrom(c.file.reader) + switch err { + case nil: + return filter + case ErrMissingBloomFilter: + return nil + default: + return &errorBloomFilter{err: err} + } +} + +// BloomFilterFrom is like BloomFilter but uses the reader passed as argument to +// read the bloom filter. +func (c *FileColumnChunk) BloomFilterFrom(reader io.ReaderAt) (*FileBloomFilter, error) { + filter, err := c.readBloomFilter(reader) + if err != nil { + return nil, err + } + if filter == nil || c.chunk.MetaData.BloomFilterOffset == 0 { + return nil, ErrMissingBloomFilter + } + return filter, nil +} + +// NumValues returns the number of values in the column chunk. +func (c *FileColumnChunk) NumValues() int64 { + return c.chunk.MetaData.NumValues +} + +// NullCount returns the number of null values in the column chunk. +// +// This value is extracted from the column chunk statistics, parquet writers are +// not required to populate it. +func (c *FileColumnChunk) NullCount() int64 { + return c.chunk.MetaData.Statistics.NullCount +} + +func (c *FileColumnChunk) readColumnIndex() (*FileColumnIndex, error) { + return c.readColumnIndexFrom(c.file.reader) +} + +func (c *FileColumnChunk) readColumnIndexFrom(reader io.ReaderAt) (*FileColumnIndex, error) { + if index := c.columnIndex.Load(); index != nil { + return index, nil + } + columnChunk := &c.file.metadata.RowGroups[c.rowGroup.Ordinal].Columns[c.Column()] + offset, length := columnChunk.ColumnIndexOffset, columnChunk.ColumnIndexLength + if offset == 0 { + return nil, nil + } + + indexData := make([]byte, int(length)) + var columnIndex format.ColumnIndex + if _, err := readAt(reader, indexData, offset); err != nil { + return nil, fmt.Errorf("read %d bytes column index at offset %d: %w", length, offset, err) + } + if err := thrift.Unmarshal(&c.file.protocol, indexData, &columnIndex); err != nil { + return nil, fmt.Errorf("decode column index: rowGroup=%d columnChunk=%d/%d: %w", c.rowGroup.Ordinal, c.Column(), len(c.rowGroup.Columns), err) + } + index := &FileColumnIndex{index: &columnIndex, kind: c.column.Type().Kind()} + // We do a CAS (and Load on CAS failure) instead of a simple Store for + // the nice property that concurrent calling goroutines will only ever + // observe a single pointer value for the result. + if !c.columnIndex.CompareAndSwap(nil, index) { + // another goroutine populated it since we last read the pointer + return c.columnIndex.Load(), nil + } + return index, nil +} + +func (c *FileColumnChunk) readOffsetIndex(reader io.ReaderAt) (*FileOffsetIndex, error) { + if index := c.offsetIndex.Load(); index != nil { + return index, nil + } + columnChunk := &c.file.metadata.RowGroups[c.rowGroup.Ordinal].Columns[c.Column()] + offset, length := columnChunk.OffsetIndexOffset, columnChunk.OffsetIndexLength + if offset == 0 { + return nil, nil + } + + indexData := make([]byte, int(length)) + var offsetIndex format.OffsetIndex + if _, err := readAt(reader, indexData, offset); err != nil { + return nil, fmt.Errorf("read %d bytes offset index at offset %d: %w", length, offset, err) + } + if err := thrift.Unmarshal(&c.file.protocol, indexData, &offsetIndex); err != nil { + return nil, fmt.Errorf("decode offset index: rowGroup=%d columnChunk=%d/%d: %w", c.rowGroup.Ordinal, c.Column(), len(c.rowGroup.Columns), err) + } + index := &FileOffsetIndex{index: &offsetIndex} + if !c.offsetIndex.CompareAndSwap(nil, index) { + // another goroutine populated it since we last read the pointer + return c.offsetIndex.Load(), nil + } + return index, nil +} + +func (c *FileColumnChunk) readBloomFilter(reader io.ReaderAt) (*FileBloomFilter, error) { + if filter := c.bloomFilter.Load(); filter != nil { + return filter, nil + } + columnChunkMetaData := &c.file.metadata.RowGroups[c.rowGroup.Ordinal].Columns[c.Column()].MetaData + offset := columnChunkMetaData.BloomFilterOffset + length := c.file.size - offset + if offset == 0 { + return nil, nil + } + + section := io.NewSectionReader(reader, offset, length) + rbuf, rbufpool := getBufioReader(section, 1024) + defer putBufioReader(rbuf, rbufpool) + + header := format.BloomFilterHeader{} + compact := thrift.CompactProtocol{} + decoder := thrift.NewDecoder(compact.NewReader(rbuf)) + + if err := decoder.Decode(&header); err != nil { + return nil, fmt.Errorf("decoding bloom filter header: %w", err) + } + + offset, _ = section.Seek(0, io.SeekCurrent) + filter := newBloomFilter(reader, offset, &header) + + if !c.bloomFilter.CompareAndSwap(nil, filter) { + return c.bloomFilter.Load(), nil + } + return filter, nil +} + +type FilePages struct { + chunk *FileColumnChunk + rbuf *bufio.Reader + rbufpool *sync.Pool + section io.SectionReader + + protocol thrift.CompactProtocol + decoder thrift.Decoder + + baseOffset int64 + dataOffset int64 + dictOffset int64 + index int + skip int64 + dictionary Dictionary + + bufferSize int +} + +func (f *FilePages) init(c *FileColumnChunk, bufferSize int, reader io.ReaderAt) { + f.chunk = c + f.baseOffset = c.chunk.MetaData.DataPageOffset + f.dataOffset = f.baseOffset + f.bufferSize = bufferSize + + if c.chunk.MetaData.DictionaryPageOffset != 0 { + f.baseOffset = c.chunk.MetaData.DictionaryPageOffset + f.dictOffset = f.baseOffset + } + + f.section = *io.NewSectionReader(reader, f.baseOffset, c.chunk.MetaData.TotalCompressedSize) + f.rbuf, f.rbufpool = getBufioReader(&f.section, f.bufferSize) + f.decoder.Reset(f.protocol.NewReader(f.rbuf)) +} + +// ReadDictionary returns the dictionary of the column chunk, or nil if the +// column chunk did not have one. +// +// The program is not required to call this method before calling ReadPage, +// the dictionary is read automatically when needed. It is exposed to allow +// programs to access the dictionary without reading the first page. +func (f *FilePages) ReadDictionary() (Dictionary, error) { + if f.dictionary == nil && f.dictOffset > 0 { + if err := f.readDictionary(); err != nil { + return nil, err + } + } + return f.dictionary, nil +} + +func (f *FilePages) SetDictionary(dict Dictionary) { + f.dictionary = dict +} + +// ReadPages reads the next from from f. +func (f *FilePages) ReadPage() (Page, error) { + if f.chunk == nil { + return nil, io.EOF + } + + for { + // Instantiate a new format.PageHeader for each page. + // + // A previous implementation reused page headers to save allocations. + // https://github.com/segmentio/parquet-go/pull/484 + // The optimization turned out to be less effective than expected, + // because all the values referenced by pointers in the page header + // are lost when the header is reset and put back in the pool. + // https://github.com/parquet-go/parquet-go/pull/11 + // + // Even after being reset, reusing page headers still produced instability + // issues. + // https://github.com/parquet-go/parquet-go/issues/70 + header := new(format.PageHeader) + if err := f.decoder.Decode(header); err != nil { + return nil, err + } + data, err := f.readPage(header, f.rbuf) + if err != nil { + return nil, err + } + + var page Page + switch header.Type { + case format.DataPageV2: + page, err = f.readDataPageV2(header, data) + case format.DataPage: + page, err = f.readDataPageV1(header, data) + case format.DictionaryPage: + // Sometimes parquet files do not have the dictionary page offset + // recorded in the column metadata. We account for this by lazily + // reading dictionary pages when we encounter them. + err = f.readDictionaryPage(header, data) + default: + err = fmt.Errorf("cannot read values of type %s from page", header.Type) + } + + data.unref() + + if err != nil { + return nil, fmt.Errorf("decoding page %d of column %q: %w", f.index, f.columnPath(), err) + } + + if page == nil { + continue + } + + f.index++ + if f.skip == 0 { + return page, nil + } + + // TODO: what about pages that don't embed the number of rows? + // (data page v1 with no offset index in the column chunk). + numRows := page.NumRows() + + if numRows <= f.skip { + Release(page) + } else { + tail := page.Slice(f.skip, numRows) + Release(page) + f.skip = 0 + return tail, nil + } + + f.skip -= numRows + } +} + +func (f *FilePages) readDictionary() error { + chunk := io.NewSectionReader(f.section.Outer()) + rbuf, pool := getBufioReader(chunk, f.bufferSize) + defer putBufioReader(rbuf, pool) + + decoder := thrift.NewDecoder(f.protocol.NewReader(rbuf)) + + header := new(format.PageHeader) + + if err := decoder.Decode(header); err != nil { + return err + } + + page := buffers.get(int(header.CompressedPageSize)) + defer page.unref() + + if _, err := io.ReadFull(rbuf, page.data); err != nil { + return err + } + + return f.readDictionaryPage(header, page) +} + +func (f *FilePages) readDictionaryPage(header *format.PageHeader, page *buffer) error { + if header.DictionaryPageHeader == nil { + return ErrMissingPageHeader + } + d, err := f.chunk.column.decodeDictionary(DictionaryPageHeader{header.DictionaryPageHeader}, page, header.UncompressedPageSize) + if err != nil { + return err + } + f.dictionary = d + return nil +} + +func (f *FilePages) readDataPageV1(header *format.PageHeader, page *buffer) (Page, error) { + if header.DataPageHeader == nil { + return nil, ErrMissingPageHeader + } + if isDictionaryFormat(header.DataPageHeader.Encoding) && f.dictionary == nil { + if err := f.readDictionary(); err != nil { + return nil, err + } + } + return f.chunk.column.decodeDataPageV1(DataPageHeaderV1{header.DataPageHeader}, page, f.dictionary, header.UncompressedPageSize) +} + +func (f *FilePages) readDataPageV2(header *format.PageHeader, page *buffer) (Page, error) { + if header.DataPageHeaderV2 == nil { + return nil, ErrMissingPageHeader + } + if isDictionaryFormat(header.DataPageHeaderV2.Encoding) && f.dictionary == nil { + // If the program seeked to a row passed the first page, the dictionary + // page may not have been seen, in which case we have to lazily load it + // from the beginning of column chunk. + if err := f.readDictionary(); err != nil { + return nil, err + } + } + return f.chunk.column.decodeDataPageV2(DataPageHeaderV2{header.DataPageHeaderV2}, page, f.dictionary, header.UncompressedPageSize) +} + +func (f *FilePages) readPage(header *format.PageHeader, reader *bufio.Reader) (*buffer, error) { + page := buffers.get(int(header.CompressedPageSize)) + defer page.unref() + + if _, err := io.ReadFull(reader, page.data); err != nil { + return nil, err + } + + if header.CRC != 0 { + headerChecksum := uint32(header.CRC) + bufferChecksum := crc32.ChecksumIEEE(page.data) + + if headerChecksum != bufferChecksum { + // The parquet specs indicate that corruption errors could be + // handled gracefully by skipping pages, tho this may not always + // be practical. Depending on how the pages are consumed, + // missing rows may cause unpredictable behaviors in algorithms. + // + // For now, we assume these errors to be fatal, but we may + // revisit later and improve error handling to be more resilient + // to data corruption. + return nil, fmt.Errorf("crc32 checksum mismatch in page of column %q: want=0x%08X got=0x%08X: %w", + f.columnPath(), + headerChecksum, + bufferChecksum, + ErrCorrupted, + ) + } + } + + page.ref() + return page, nil +} + +// SeekToRow seeks to the given row index in the column chunk. +func (f *FilePages) SeekToRow(rowIndex int64) (err error) { + if f.chunk == nil { + return io.ErrClosedPipe + } + if index := f.chunk.offsetIndex.Load(); index == nil { + _, err = f.section.Seek(f.dataOffset-f.baseOffset, io.SeekStart) + f.skip = rowIndex + f.index = 0 + if f.dictOffset > 0 { + f.index = 1 + } + } else { + pages := index.index.PageLocations + index := sort.Search(len(pages), func(i int) bool { + return pages[i].FirstRowIndex > rowIndex + }) - 1 + if index < 0 { + return ErrSeekOutOfRange + } + _, err = f.section.Seek(pages[index].Offset-f.baseOffset, io.SeekStart) + f.skip = rowIndex - pages[index].FirstRowIndex + f.index = index + } + f.rbuf.Reset(&f.section) + return err +} + +// Close closes the page reader. +func (f *FilePages) Close() error { + putBufioReader(f.rbuf, f.rbufpool) + f.chunk = nil + f.section = io.SectionReader{} + f.rbuf = nil + f.rbufpool = nil + f.baseOffset = 0 + f.dataOffset = 0 + f.dictOffset = 0 + f.index = 0 + f.skip = 0 + f.dictionary = nil + return nil +} + +func (f *FilePages) columnPath() columnPath { + return columnPath(f.chunk.column.Path()) +} + +type putBufioReaderFunc func() + +var ( + bufioReaderPoolLock sync.Mutex + bufioReaderPool = map[int]*sync.Pool{} +) + +func getBufioReader(r io.Reader, bufferSize int) (*bufio.Reader, *sync.Pool) { + pool := getBufioReaderPool(bufferSize) + rbuf, _ := pool.Get().(*bufio.Reader) + if rbuf == nil { + rbuf = bufio.NewReaderSize(r, bufferSize) + } else { + rbuf.Reset(r) + } + return rbuf, pool +} + +func putBufioReader(rbuf *bufio.Reader, pool *sync.Pool) { + if rbuf != nil && pool != nil { + rbuf.Reset(nil) + pool.Put(rbuf) + } +} + +func getBufioReaderPool(size int) *sync.Pool { + bufioReaderPoolLock.Lock() + defer bufioReaderPoolLock.Unlock() + + if pool := bufioReaderPool[size]; pool != nil { + return pool + } + + pool := &sync.Pool{} + bufioReaderPool[size] = pool + return pool +} + +func (f *File) readAt(p []byte, off int64) (int, error) { + return readAt(f.reader, p, off) +} + +func readAt(r io.ReaderAt, p []byte, off int64) (n int, err error) { + n, err = r.ReadAt(p, off) + if n == len(p) { + err = nil + // p was fully read.There is no further need to check for errors. This + // operation is a success in principle. + return + } + return +} + +type optimisticFileReaderAt struct { + reader io.ReaderAt + offset int64 + footer []byte +} + +func (r *optimisticFileReaderAt) ReadAt(p []byte, off int64) (n int, err error) { + length := r.offset + int64(len(r.footer)) + + if off >= length { + return 0, io.EOF + } + + if off >= r.offset { + n = copy(p, r.footer[off-r.offset:]) + p = p[n:] + off += int64(n) + if len(p) == 0 { + return n, nil + } + } + + rn, err := r.reader.ReadAt(p, off) + return n + rn, err +} diff --git a/vendor/github.com/parquet-go/parquet-go/filter.go b/vendor/github.com/parquet-go/parquet-go/filter.go new file mode 100644 index 0000000000..26e0f3f457 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/filter.go @@ -0,0 +1,82 @@ +package parquet + +// FilterRowReader constructs a RowReader which exposes rows from reader for +// which the predicate has returned true. +func FilterRowReader(reader RowReader, predicate func(Row) bool) RowReader { + f := &filterRowReader{reader: reader, predicate: predicate} + for i := range f.rows { + f.rows[i] = f.values[i : i : i+1] + } + return f +} + +type filterRowReader struct { + reader RowReader + predicate func(Row) bool + rows [defaultRowBufferSize]Row + values [defaultRowBufferSize]Value +} + +func (f *filterRowReader) ReadRows(rows []Row) (n int, err error) { + for n < len(rows) { + r := min(len(rows)-n, len(f.rows)) + + r, err = f.reader.ReadRows(f.rows[:r]) + + for i := range r { + if f.predicate(f.rows[i]) { + rows[n] = append(rows[n][:0], f.rows[i]...) + n++ + } + } + + if err != nil { + break + } + } + return n, err +} + +// FilterRowWriter constructs a RowWriter which writes rows to writer for which +// the predicate has returned true. +func FilterRowWriter(writer RowWriter, predicate func(Row) bool) RowWriter { + return &filterRowWriter{writer: writer, predicate: predicate} +} + +type filterRowWriter struct { + writer RowWriter + predicate func(Row) bool + rows [defaultRowBufferSize]Row +} + +func (f *filterRowWriter) WriteRows(rows []Row) (n int, err error) { + defer func() { + clear := f.rows[:] + for i := range clear { + clearValues(clear[i]) + } + }() + + for n < len(rows) { + i := 0 + j := min(len(rows)-n, len(f.rows)) + + for _, row := range rows[n : n+j] { + if f.predicate(row) { + f.rows[i] = row + i++ + } + } + + if i > 0 { + _, err := f.writer.WriteRows(f.rows[:i]) + if err != nil { + break + } + } + + n += j + } + + return n, err +} diff --git a/vendor/github.com/parquet-go/parquet-go/format/parquet.go b/vendor/github.com/parquet-go/parquet-go/format/parquet.go new file mode 100644 index 0000000000..5366c919cd --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/format/parquet.go @@ -0,0 +1,1230 @@ +package format + +import ( + "fmt" + + "github.com/parquet-go/parquet-go/deprecated" +) + +// Types supported by Parquet. These types are intended to be used in combination +// with the encodings to control the on disk storage format. For example INT16 +// is not included as a type since a good encoding of INT32 would handle this. +type Type int32 + +const ( + Boolean Type = 0 + Int32 Type = 1 + Int64 Type = 2 + Int96 Type = 3 // deprecated, only used by legacy implementations. + Float Type = 4 + Double Type = 5 + ByteArray Type = 6 + FixedLenByteArray Type = 7 +) + +func (t Type) String() string { + switch t { + case Boolean: + return "BOOLEAN" + case Int32: + return "INT32" + case Int64: + return "INT64" + case Int96: + return "INT96" + case Float: + return "FLOAT" + case Double: + return "DOUBLE" + case ByteArray: + return "BYTE_ARRAY" + case FixedLenByteArray: + return "FIXED_LEN_BYTE_ARRAY" + default: + return "Type(?)" + } +} + +// Representation of Schemas. +type FieldRepetitionType int32 + +const ( + // The field is required (can not be null) and each record has exactly 1 value. + Required FieldRepetitionType = 0 + // The field is optional (can be null) and each record has 0 or 1 values. + Optional FieldRepetitionType = 1 + // The field is repeated and can contain 0 or more values. + Repeated FieldRepetitionType = 2 +) + +func (t FieldRepetitionType) String() string { + switch t { + case Required: + return "REQUIRED" + case Optional: + return "OPTIONAL" + case Repeated: + return "REPEATED" + default: + return "FieldRepeationaType(?)" + } +} + +// A structure for capturing metadata for estimating the unencoded, +// uncompressed size of data written. This is useful for readers to estimate +// how much memory is needed to reconstruct data in their memory model and for +// fine grained filter pushdown on nested structures (the histograms contained +// in this structure can help determine the number of nulls at a particular +// nesting level and maximum length of lists). +type SizeStatistics struct { + // The number of physical bytes stored for BYTE_ARRAY data values assuming + // no encoding. This is exclusive of the bytes needed to store the length of + // each byte array. In other words, this field is equivalent to the `(size + // of PLAIN-ENCODING the byte array values) - (4 bytes * number of values + // written)`. To determine unencoded sizes of other types readers can use + // schema information multiplied by the number of non-null and null values. + // The number of null/non-null values can be inferred from the histograms + // below. + // + // For example, if a column chunk is dictionary-encoded with dictionary + // ["a", "bc", "cde"], and a data page contains the indices [0, 0, 1, 2], + // then this value for that data page should be 7 (1 + 1 + 2 + 3). + // + // This field should only be set for types that use BYTE_ARRAY as their + // physical type. + UnencodedByteArrayDataBytes *int64 `thrift:"1,optional"` + + // When present, there is expected to be one element corresponding to each + // repetition (i.e. size=max repetition_level+1) where each element + // represents the number of times the repetition level was observed in the + // data. + // + // This field may be omitted if max_repetition_level is 0 without loss + // of information. + RepetitionLevelHistogram []int64 `thrift:"2,optional"` + + // Same as repetition_level_histogram except for definition levels. + // + // This field may be omitted if max_definition_level is 0 or 1 without + // loss of information. + DefinitionLevelHistogram []int64 `thrift:"3,optional"` +} + +// Bounding box for GEOMETRY or GEOGRAPHY type in the representation of min/max +// value pair of coordinates from each axis. +type BoundingBox struct { + XMin float64 `thrift:"1,required"` + XMax float64 `thrift:"2,required"` + YMin float64 `thrift:"3,required"` + YMax float64 `thrift:"4,required"` + ZMin *float64 `thrift:"5,optional"` + ZMax *float64 `thrift:"6,optional"` + MMin *float64 `thrift:"7,optional"` + MMax *float64 `thrift:"8,optional"` +} + +// Statistics specific to Geometry and Geography logical types +type GeospatialStatistics struct { + // A bounding box of geospatial instances + BBox *BoundingBox `thrift:"1,optional"` + // Geospatial type codes of all instances, or an empty list if not known + GeoSpatialTypes []int32 `thrift:"2,optional"` +} + +// Statistics per row group and per page. +// All fields are optional. +type Statistics struct { + // DEPRECATED: min and max value of the column. Use min_value and max_value. + // + // Values are encoded using PLAIN encoding, except that variable-length byte + // arrays do not include a length prefix. + // + // These fields encode min and max values determined by signed comparison + // only. New files should use the correct order for a column's logical type + // and store the values in the min_value and max_value fields. + // + // To support older readers, these may be set when the column order is + // signed. + Max []byte `thrift:"1"` + Min []byte `thrift:"2"` + // Count of null value in the column. + NullCount int64 `thrift:"3"` + // Count of distinct values occurring. + DistinctCount int64 `thrift:"4"` + // Min and max values for the column, determined by its ColumnOrder. + // + // Values are encoded using PLAIN encoding, except that variable-length byte + // arrays do not include a length prefix. + MaxValue []byte `thrift:"5"` + MinValue []byte `thrift:"6"` +} + +// Empty structs to use as logical type annotations. +type StringType struct{} // allowed for BINARY, must be encoded with UTF-8 +type UUIDType struct{} // allowed for FIXED[16], must encode raw UUID bytes +type MapType struct{} // see LogicalTypes.md +type ListType struct{} // see LogicalTypes.md +type EnumType struct{} // allowed for BINARY, must be encoded with UTF-8 +type DateType struct{} // allowed for INT32 +type Float16Type struct{} // allowed for FIXED[2], must encoded raw FLOAT16 bytes + +func (*StringType) String() string { return "STRING" } +func (*UUIDType) String() string { return "UUID" } +func (*MapType) String() string { return "MAP" } +func (*ListType) String() string { return "LIST" } +func (*EnumType) String() string { return "ENUM" } +func (*DateType) String() string { return "DATE" } +func (*Float16Type) String() string { return "FLOAT16" } + +// Logical type to annotate a column that is always null. +// +// Sometimes when discovering the schema of existing data, values are always +// null and the physical type can't be determined. This annotation signals +// the case where the physical type was guessed from all null values. +type NullType struct{} + +func (*NullType) String() string { return "NULL" } + +// Decimal logical type annotation +// +// To maintain forward-compatibility in v1, implementations using this logical +// type must also set scale and precision on the annotated SchemaElement. +// +// Allowed for physical types: INT32, INT64, FIXED, and BINARY +type DecimalType struct { + Scale int32 `thrift:"1,required"` + Precision int32 `thrift:"2,required"` +} + +func (t *DecimalType) String() string { + // Matching parquet-cli's decimal string format: https://github.com/apache/parquet-java/blob/d057b39d93014fe40f5067ee4a33621e65c91552/parquet-column/src/test/java/org/apache/parquet/parser/TestParquetParser.java#L249-L265 + return fmt.Sprintf("DECIMAL(%d,%d)", t.Precision, t.Scale) +} + +// Time units for logical types. +type MilliSeconds struct{} +type MicroSeconds struct{} +type NanoSeconds struct{} + +func (*MilliSeconds) String() string { return "MILLIS" } +func (*MicroSeconds) String() string { return "MICROS" } +func (*NanoSeconds) String() string { return "NANOS" } + +type TimeUnit struct { // union + Millis *MilliSeconds `thrift:"1"` + Micros *MicroSeconds `thrift:"2"` + Nanos *NanoSeconds `thrift:"3"` +} + +func (u *TimeUnit) String() string { + switch { + case u.Millis != nil: + return u.Millis.String() + case u.Micros != nil: + return u.Micros.String() + case u.Nanos != nil: + return u.Nanos.String() + default: + return "" + } +} + +// Timestamp logical type annotation +// +// Allowed for physical types: INT64 +type TimestampType struct { + IsAdjustedToUTC bool `thrift:"1,required"` + Unit TimeUnit `thrift:"2,required"` +} + +func (t *TimestampType) String() string { + return fmt.Sprintf("TIMESTAMP(isAdjustedToUTC=%t,unit=%s)", t.IsAdjustedToUTC, &t.Unit) +} + +// Time logical type annotation +// +// Allowed for physical types: INT32 (millis), INT64 (micros, nanos) +type TimeType struct { + IsAdjustedToUTC bool `thrift:"1,required"` + Unit TimeUnit `thrift:"2,required"` +} + +func (t *TimeType) String() string { + return fmt.Sprintf("TIME(isAdjustedToUTC=%t,unit=%s)", t.IsAdjustedToUTC, &t.Unit) +} + +// Integer logical type annotation +// +// bitWidth must be 8, 16, 32, or 64. +// +// Allowed for physical types: INT32, INT64 +type IntType struct { + BitWidth int8 `thrift:"1,required"` + IsSigned bool `thrift:"2,required"` +} + +func (t *IntType) String() string { + return fmt.Sprintf("INT(%d,%t)", t.BitWidth, t.IsSigned) +} + +// Embedded JSON logical type annotation +// +// Allowed for physical types: BINARY +type JsonType struct{} + +func (t *JsonType) String() string { return "JSON" } + +// Embedded BSON logical type annotation +// +// Allowed for physical types: BINARY +type BsonType struct{} + +func (t *BsonType) String() string { return "BSON" } + +// Embedded Variant logical type annotation +type VariantType struct{} + +func (*VariantType) String() string { return "VARIANT" } + +// Edge interpolation algorithm for Geography logical type +type EdgeInterpolationAlgorithm int32 + +const ( + Spherical EdgeInterpolationAlgorithm = 0 + Vincenty EdgeInterpolationAlgorithm = 1 + Thomas EdgeInterpolationAlgorithm = 2 + Andoyer EdgeInterpolationAlgorithm = 3 + Karney EdgeInterpolationAlgorithm = 4 +) + +func (e EdgeInterpolationAlgorithm) String() string { + switch e { + case Spherical: + return "SPHERICAL" + case Vincenty: + return "VINCENTY" + case Thomas: + return "THOMAS" + case Andoyer: + return "ANDOYER" + case Karney: + return "KARNEY" + default: + return "EdgeInterpolationAlgorithm(?)" + } +} + +// Embedded Geometry logical type annotation +// +// Geospatial features in the Well-Known Binary (WKB) format and edges interpolation +// is always linear/planar. +// +// A custom CRS can be set by the crs field. If unset, it defaults to "OGC:CRS84", +// which means that the geometries must be stored in longitude, latitude based on +// the WGS84 datum. +// +// Allowed for physical type: BYTE_ARRAY. +// +// See Geospatial.md for details. +type GeometryType struct { + CRS string `thrift:"1,optional"` +} + +func (t *GeometryType) String() string { + crs := t.CRS + if crs == "" { + crs = "OGC:CRS84" + } + return fmt.Sprintf("GEOMETRY(%q)", crs) +} + +// Embedded Geography logical type annotation +// +// Geospatial features in the WKB format with an explicit (non-linear/non-planar) +// edges interpolation algorithm. +// +// A custom geographic CRS can be set by the crs field, where longitudes are +// bound by [-180, 180] and latitudes are bound by [-90, 90]. If unset, the CRS +// defaults to "OGC:CRS84". +// +// An optional algorithm can be set to correctly interpret edges interpolation +// of the geometries. If unset, the algorithm defaults to SPHERICAL. +// +// Allowed for physical type: BYTE_ARRAY. +// +// See Geospatial.md for details. +type GeographyType struct { + CRS string `thrift:"1,optional"` + Algorithm EdgeInterpolationAlgorithm `thrift:"2,optional"` +} + +func (t *GeographyType) String() string { + crs := t.CRS + if crs == "" { + crs = "OGC:CRS84" + } + return fmt.Sprintf("GEOGRAPHY(%q, %s)", crs, t.Algorithm) +} + +// LogicalType annotations to replace ConvertedType. +// +// To maintain compatibility, implementations using LogicalType for a +// SchemaElement must also set the corresponding ConvertedType (if any) +// from the following table. +type LogicalType struct { // union + UTF8 *StringType `thrift:"1"` // use ConvertedType UTF8 + Map *MapType `thrift:"2"` // use ConvertedType Map + List *ListType `thrift:"3"` // use ConvertedType List + Enum *EnumType `thrift:"4"` // use ConvertedType Enum + Decimal *DecimalType `thrift:"5"` // use ConvertedType Decimal + SchemaElement.{Scale, Precision} + Date *DateType `thrift:"6"` // use ConvertedType Date + + // use ConvertedType TimeMicros for Time{IsAdjustedToUTC: *, Unit: Micros} + // use ConvertedType TimeMillis for Time{IsAdjustedToUTC: *, Unit: Millis} + Time *TimeType `thrift:"7"` + + // use ConvertedType TimestampMicros for Timestamp{IsAdjustedToUTC: *, Unit: Micros} + // use ConvertedType TimestampMillis for Timestamp{IsAdjustedToUTC: *, Unit: Millis} + Timestamp *TimestampType `thrift:"8"` + + // 9: reserved for Interval + Integer *IntType `thrift:"10"` // use ConvertedType Int* or Uint* + Unknown *NullType `thrift:"11"` // no compatible ConvertedType + Json *JsonType `thrift:"12"` // use ConvertedType JSON + Bson *BsonType `thrift:"13"` // use ConvertedType BSON + UUID *UUIDType `thrift:"14"` // no compatible ConvertedType + Float16 *Float16Type `thrift:"15"` // no compatible ConvertedType + Variant *VariantType `thrift:"16"` // no compatible ConvertedType + Geometry *GeometryType `thrift:"17"` // no compatible ConvertedType + Geography *GeographyType `thrift:"18"` // no compatible ConvertedType +} + +func (t *LogicalType) String() string { + switch { + case t.UTF8 != nil: + return t.UTF8.String() + case t.Map != nil: + return t.Map.String() + case t.List != nil: + return t.List.String() + case t.Enum != nil: + return t.Enum.String() + case t.Decimal != nil: + return t.Decimal.String() + case t.Date != nil: + return t.Date.String() + case t.Time != nil: + return t.Time.String() + case t.Timestamp != nil: + return t.Timestamp.String() + case t.Integer != nil: + return t.Integer.String() + case t.Unknown != nil: + return t.Unknown.String() + case t.Json != nil: + return t.Json.String() + case t.Bson != nil: + return t.Bson.String() + case t.UUID != nil: + return t.UUID.String() + case t.Float16 != nil: + return t.Float16.String() + case t.Variant != nil: + return t.Variant.String() + case t.Geometry != nil: + return t.Geometry.String() + case t.Geography != nil: + return t.Geography.String() + default: + return "" + } +} + +// Represents a element inside a schema definition. +// +// - if it is a group (inner node) then type is undefined and num_children is +// defined +// +// - if it is a primitive type (leaf) then type is defined and num_children is +// undefined +// +// The nodes are listed in depth first traversal order. +type SchemaElement struct { + // Data type for this field. Not set if the current element is a non-leaf node. + Type *Type `thrift:"1,optional"` + + // If type is FixedLenByteArray, this is the byte length of the values. + // Otherwise, if specified, this is the maximum bit length to store any of the values. + // (e.g. a low cardinality INT col could have this set to 3). Note that this is + // in the schema, and therefore fixed for the entire file. + TypeLength *int32 `thrift:"2,optional"` + + // repetition of the field. The root of the schema does not have a repetition_type. + // All other nodes must have one. + RepetitionType *FieldRepetitionType `thrift:"3,optional"` + + // Name of the field in the schema. + Name string `thrift:"4,required"` + + // Nested fields. Since thrift does not support nested fields, + // the nesting is flattened to a single list by a depth-first traversal. + // The children count is used to construct the nested relationship. + // This field is not set when the element is a primitive type + NumChildren int32 `thrift:"5,optional"` + + // DEPRECATED: When the schema is the result of a conversion from another model. + // Used to record the original type to help with cross conversion. + // + // This is superseded by logicalType. + ConvertedType *deprecated.ConvertedType `thrift:"6,optional"` + + // DEPRECATED: Used when this column contains decimal data. + // See the DECIMAL converted type for more details. + // + // This is superseded by using the DecimalType annotation in logicalType. + Scale *int32 `thrift:"7,optional"` + Precision *int32 `thrift:"8,optional"` + + // When the original schema supports field ids, this will save the + // original field id in the parquet schema. + FieldID int32 `thrift:"9,optional"` + + // The logical type of this SchemaElement + // + // LogicalType replaces ConvertedType, but ConvertedType is still required + // for some logical types to ensure forward-compatibility in format v1. + LogicalType *LogicalType `thrift:"10,optional"` +} + +// Encodings supported by Parquet. Not all encodings are valid for all types. +// These enums are also used to specify the encoding of definition and +// repetition levels. See the accompanying doc for the details of the more +// complicated encodings. +type Encoding int32 + +const ( + // Default encoding. + // Boolean - 1 bit per value. 0 is false; 1 is true. + // Int32 - 4 bytes per value. Stored as little-endian. + // Int64 - 8 bytes per value. Stored as little-endian. + // Float - 4 bytes per value. IEEE. Stored as little-endian. + // Double - 8 bytes per value. IEEE. Stored as little-endian. + // ByteArray - 4 byte length stored as little endian, followed by bytes. + // FixedLenByteArray - Just the bytes. + Plain Encoding = 0 + + // Group VarInt encoding for Int32/Int64. + // This encoding is deprecated. It was never used. + // GroupVarInt Encoding = 1 + + // Deprecated: Dictionary encoding. The values in the dictionary are encoded + // in the plain type. + // In a data page use RLEDictionary instead. + // In a Dictionary page use Plain instead. + PlainDictionary Encoding = 2 + + // Group packed run length encoding. Usable for definition/repetition levels + // encoding and Booleans (on one bit: 0 is false 1 is true.) + RLE Encoding = 3 + + // Bit packed encoding. This can only be used if the data has a known max + // width. Usable for definition/repetition levels encoding. + BitPacked Encoding = 4 + + // Delta encoding for integers. This can be used for int columns and works best + // on sorted data. + DeltaBinaryPacked Encoding = 5 + + // Encoding for byte arrays to separate the length values and the data. + // The lengths are encoded using DeltaBinaryPacked. + DeltaLengthByteArray Encoding = 6 + + // Incremental-encoded byte array. Prefix lengths are encoded using DELTA_BINARY_PACKED. + // Suffixes are stored as delta length byte arrays. + DeltaByteArray Encoding = 7 + + // Dictionary encoding: the ids are encoded using the RLE encoding + RLEDictionary Encoding = 8 + + // Encoding for floating-point data. + // K byte-streams are created where K is the size in bytes of the data type. + // The individual bytes of an FP value are scattered to the corresponding stream and + // the streams are concatenated. + // This itself does not reduce the size of the data but can lead to better compression + // afterwards. + ByteStreamSplit Encoding = 9 +) + +func (e Encoding) String() string { + switch e { + case Plain: + return "PLAIN" + case PlainDictionary: + return "PLAIN_DICTIONARY" + case RLE: + return "RLE" + case BitPacked: + return "BIT_PACKED" + case DeltaBinaryPacked: + return "DELTA_BINARY_PACKED" + case DeltaLengthByteArray: + return "DELTA_LENGTH_BYTE_ARRAY" + case DeltaByteArray: + return "DELTA_BYTE_ARRAY" + case RLEDictionary: + return "RLE_DICTIONARY" + case ByteStreamSplit: + return "BYTE_STREAM_SPLIT" + default: + return "Encoding(?)" + } +} + +// Supported compression algorithms. +// +// Codecs added in format version X.Y can be read by readers based on X.Y and later. +// Codec support may vary between readers based on the format version and +// libraries available at runtime. +// +// See Compression.md for a detailed specification of these algorithms. +type CompressionCodec int32 + +const ( + Uncompressed CompressionCodec = 0 + Snappy CompressionCodec = 1 + Gzip CompressionCodec = 2 + LZO CompressionCodec = 3 + Brotli CompressionCodec = 4 // Added in 2.4 + Lz4 CompressionCodec = 5 // DEPRECATED (Added in 2.4) + Zstd CompressionCodec = 6 // Added in 2.4 + Lz4Raw CompressionCodec = 7 // Added in 2.9 +) + +func (c CompressionCodec) String() string { + switch c { + case Uncompressed: + return "UNCOMPRESSED" + case Snappy: + return "SNAPPY" + case Gzip: + return "GZIP" + case LZO: + return "LZO" + case Brotli: + return "BROTLI" + case Lz4: + return "LZ4" + case Zstd: + return "ZSTD" + case Lz4Raw: + return "LZ4_RAW" + default: + return "CompressionCodec(?)" + } +} + +type PageType int32 + +const ( + DataPage PageType = 0 + IndexPage PageType = 1 + DictionaryPage PageType = 2 + // Version 2 is indicated in the PageHeader and the use of DataPageHeaderV2, + // and allows you to read repetition and definition level data without + // decompressing the Page. + DataPageV2 PageType = 3 +) + +func (p PageType) String() string { + switch p { + case DataPage: + return "DATA_PAGE" + case IndexPage: + return "INDEX_PAGE" + case DictionaryPage: + return "DICTIONARY_PAGE" + case DataPageV2: + return "DATA_PAGE_V2" + default: + return "PageType(?)" + } +} + +// Enum to annotate whether lists of min/max elements inside ColumnIndex +// are ordered and if so, in which direction. +type BoundaryOrder int32 + +const ( + Unordered BoundaryOrder = 0 + Ascending BoundaryOrder = 1 + Descending BoundaryOrder = 2 +) + +func (b BoundaryOrder) String() string { + switch b { + case Unordered: + return "UNORDERED" + case Ascending: + return "ASCENDING" + case Descending: + return "DESCENDING" + default: + return "BoundaryOrder(?)" + } +} + +// Data page header. +type DataPageHeader struct { + // Number of values, including NULLs, in this data page. + NumValues int32 `thrift:"1,required"` + + // Encoding used for this data page. + Encoding Encoding `thrift:"2,required"` + + // Encoding used for definition levels. + DefinitionLevelEncoding Encoding `thrift:"3,required"` + + // Encoding used for repetition levels. + RepetitionLevelEncoding Encoding `thrift:"4,required"` + + // Optional statistics for the data in this page. + Statistics Statistics `thrift:"5,optional"` +} + +type IndexPageHeader struct { + // TODO +} + +// The dictionary page must be placed at the first position of the column chunk +// if it is partly or completely dictionary encoded. At most one dictionary page +// can be placed in a column chunk. +type DictionaryPageHeader struct { + // Number of values in the dictionary. + NumValues int32 `thrift:"1,required"` + + // Encoding using this dictionary page. + Encoding Encoding `thrift:"2,required"` + + // If true, the entries in the dictionary are sorted in ascending order. + IsSorted bool `thrift:"3,optional"` +} + +// New page format allowing reading levels without decompressing the data +// Repetition and definition levels are uncompressed +// The remaining section containing the data is compressed if is_compressed is +// true. +type DataPageHeaderV2 struct { + // Number of values, including NULLs, in this data page. + NumValues int32 `thrift:"1,required"` + // Number of NULL values, in this data page. + // Number of non-null = num_values - num_nulls which is also the number of + // values in the data section. + NumNulls int32 `thrift:"2,required"` + // Number of rows in this data page. which means pages change on record boundaries (r = 0). + NumRows int32 `thrift:"3,required"` + // Encoding used for data in this page. + Encoding Encoding `thrift:"4,required"` + + // Repetition levels and definition levels are always using RLE (without size in it). + + // Length of the definition levels. + DefinitionLevelsByteLength int32 `thrift:"5,required"` + // Length of the repetition levels. + RepetitionLevelsByteLength int32 `thrift:"6,required"` + + // Whether the values are compressed. + // Which means the section of the page between + // definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included) + // is compressed with the compression_codec. + // If missing it is considered compressed. + IsCompressed *bool `thrift:"7,optional"` + + // Optional statistics for the data in this page. + Statistics Statistics `thrift:"8,optional"` +} + +// Block-based algorithm type annotation. +type SplitBlockAlgorithm struct{} + +// The algorithm used in Bloom filter. +type BloomFilterAlgorithm struct { // union + Block *SplitBlockAlgorithm `thrift:"1"` +} + +// Hash strategy type annotation. xxHash is an extremely fast non-cryptographic +// hash algorithm. It uses 64 bits version of xxHash. +type XxHash struct{} + +// The hash function used in Bloom filter. This function takes the hash of a +// column value using plain encoding. +type BloomFilterHash struct { // union + XxHash *XxHash `thrift:"1"` +} + +// The compression used in the Bloom filter. +type BloomFilterUncompressed struct{} +type BloomFilterCompression struct { // union + Uncompressed *BloomFilterUncompressed `thrift:"1"` +} + +// Bloom filter header is stored at beginning of Bloom filter data of each column +// and followed by its bitset. +type BloomFilterHeader struct { + // The size of bitset in bytes. + NumBytes int32 `thrift:"1,required"` + // The algorithm for setting bits. + Algorithm BloomFilterAlgorithm `thrift:"2,required"` + // The hash function used for Bloom filter. + Hash BloomFilterHash `thrift:"3,required"` + // The compression used in the Bloom filter. + Compression BloomFilterCompression `thrift:"4,required"` +} + +type PageHeader struct { + // The type of the page indicates which of the *Header fields below is set. + Type PageType `thrift:"1,required"` + + // Uncompressed page size in bytes (not including this header). + UncompressedPageSize int32 `thrift:"2,required"` + + // Compressed (and potentially encrypted) page size in bytes, not including + // this header. + CompressedPageSize int32 `thrift:"3,required"` + + // The 32bit CRC for the page, to be be calculated as follows: + // - Using the standard CRC32 algorithm + // - On the data only, i.e. this header should not be included. 'Data' + // hereby refers to the concatenation of the repetition levels, the + // definition levels and the column value, in this exact order. + // - On the encoded versions of the repetition levels, definition levels and + // column values. + // - On the compressed versions of the repetition levels, definition levels + // and column values where possible; + // - For v1 data pages, the repetition levels, definition levels and column + // values are always compressed together. If a compression scheme is + // specified, the CRC shall be calculated on the compressed version of + // this concatenation. If no compression scheme is specified, the CRC + // shall be calculated on the uncompressed version of this concatenation. + // - For v2 data pages, the repetition levels and definition levels are + // handled separately from the data and are never compressed (only + // encoded). If a compression scheme is specified, the CRC shall be + // calculated on the concatenation of the uncompressed repetition levels, + // uncompressed definition levels and the compressed column values. + // If no compression scheme is specified, the CRC shall be calculated on + // the uncompressed concatenation. + // - In encrypted columns, CRC is calculated after page encryption; the + // encryption itself is performed after page compression (if compressed) + // If enabled, this allows for disabling checksumming in HDFS if only a few + // pages need to be read. + CRC int32 `thrift:"4,optional"` + + // Headers for page specific data. One only will be set. + DataPageHeader *DataPageHeader `thrift:"5,optional"` + IndexPageHeader *IndexPageHeader `thrift:"6,optional"` + DictionaryPageHeader *DictionaryPageHeader `thrift:"7,optional"` + DataPageHeaderV2 *DataPageHeaderV2 `thrift:"8,optional"` +} + +// Wrapper struct to store key values. +type KeyValue struct { + Key string `thrift:"1,required"` + Value string `thrift:"2,required"` +} + +// Wrapper struct to specify sort order. +type SortingColumn struct { + // The column index (in this row group) + ColumnIdx int32 `thrift:"1,required"` + + // If true, indicates this column is sorted in descending order. + Descending bool `thrift:"2,required"` + + // If true, nulls will come before non-null values, otherwise, + // nulls go at the end. + NullsFirst bool `thrift:"3,required"` +} + +// Statistics of a given page type and encoding. +type PageEncodingStats struct { + // The page type (data/dic/...). + PageType PageType `thrift:"1,required"` + + // Encoding of the page. + Encoding Encoding `thrift:"2,required"` + + // Number of pages of this type with this encoding. + Count int32 `thrift:"3,required"` +} + +// Description for column metadata. +type ColumnMetaData struct { + // Type of this column. + Type Type `thrift:"1,required"` + + // Set of all encodings used for this column. The purpose is to validate + // whether we can decode those pages. + Encoding []Encoding `thrift:"2,required"` + + // Path in schema. + PathInSchema []string `thrift:"3,required"` + + // Compression codec. + Codec CompressionCodec `thrift:"4,required"` + + // Number of values in this column. + NumValues int64 `thrift:"5,required"` + + // Total byte size of all uncompressed pages in this column chunk (including the headers). + TotalUncompressedSize int64 `thrift:"6,required"` + + // Total byte size of all compressed, and potentially encrypted, pages + // in this column chunk (including the headers). + TotalCompressedSize int64 `thrift:"7,required"` + + // Optional key/value metadata. + KeyValueMetadata []KeyValue `thrift:"8,optional"` + + // Byte offset from beginning of file to first data page. + DataPageOffset int64 `thrift:"9,required"` + + // Byte offset from beginning of file to root index page. + IndexPageOffset int64 `thrift:"10,optional"` + + // Byte offset from the beginning of file to first (only) dictionary page. + DictionaryPageOffset int64 `thrift:"11,optional"` + + // optional statistics for this column chunk. + Statistics Statistics `thrift:"12,optional"` + + // Set of all encodings used for pages in this column chunk. + // This information can be used to determine if all data pages are + // dictionary encoded for example. + EncodingStats []PageEncodingStats `thrift:"13,optional"` + + // Byte offset from beginning of file to Bloom filter data. + BloomFilterOffset int64 `thrift:"14,optional"` + + // Size of Bloom filter data including the serialized header, in bytes. + // Added in 2.10 so readers may not read this field from old files and + // it can be obtained after the BloomFilterHeader has been deserialized. + // Writers should write this field so readers can read the bloom filter + // in a single I/O. + BloomFilterLength *int32 `thrift:"15,optional"` + + // Optional statistics to help estimate total memory when converted to in-memory + // representations. The histograms contained in these statistics can + // also be useful in some cases for more fine-grained nullability/list length + // filter pushdown. + // TODO: Uncomment this field when Thrift decoding is fixed. Strangely, when it is + // uncommented, test cases in file_test.go fail with an inexplicable error decoding + // an unrelated field: + // reading parquet file metadata: decoding thrift payload: 4:FIELD โ†’ 0/1:LIST: missing required field: 2:FIELD + // (Seems to be complaining about field TotalBytesSize of RowGroup). This only occurs + // with testdata/dict-page-offset-zero.parquet, in both TestOpenFileWithoutPageIndex + // and TestOpenFile. + //SizeStatistics *SizeStatistics `thrift:"16,optional"` + + // Optional statistics specific for Geometry and Geography logical types + GeospatialStatistics *GeospatialStatistics `thrift:"17,optional"` +} + +type EncryptionWithFooterKey struct{} + +type EncryptionWithColumnKey struct { + // Column path in schema. + PathInSchema []string `thrift:"1,required"` + + // Retrieval metadata of column encryption key. + KeyMetadata []byte `thrift:"2,optional"` +} + +type ColumnCryptoMetaData struct { + EncryptionWithFooterKey *EncryptionWithFooterKey `thrift:"1"` + EncryptionWithColumnKey *EncryptionWithColumnKey `thrift:"2"` +} + +type ColumnChunk struct { + // File where column data is stored. If not set, assumed to be same file as + // metadata. This path is relative to the current file. + FilePath string `thrift:"1,optional"` + + // Byte offset in file_path to the ColumnMetaData. + FileOffset int64 `thrift:"2,required"` + + // Column metadata for this chunk. This is the same content as what is at + // file_path/file_offset. Having it here has it replicated in the file + // metadata. + MetaData ColumnMetaData `thrift:"3,optional"` + + // File offset of ColumnChunk's OffsetIndex. + OffsetIndexOffset int64 `thrift:"4,optional"` + + // Size of ColumnChunk's OffsetIndex, in bytes. + OffsetIndexLength int32 `thrift:"5,optional"` + + // File offset of ColumnChunk's ColumnIndex. + ColumnIndexOffset int64 `thrift:"6,optional"` + + // Size of ColumnChunk's ColumnIndex, in bytes. + ColumnIndexLength int32 `thrift:"7,optional"` + + // Crypto metadata of encrypted columns. + CryptoMetadata ColumnCryptoMetaData `thrift:"8,optional"` + + // Encrypted column metadata for this chunk. + EncryptedColumnMetadata []byte `thrift:"9,optional"` +} + +type RowGroup struct { + // Metadata for each column chunk in this row group. + // This list must have the same order as the SchemaElement list in FileMetaData. + Columns []ColumnChunk `thrift:"1,required"` + + // Total byte size of all the uncompressed column data in this row group. + TotalByteSize int64 `thrift:"2,required"` + + // Number of rows in this row group. + NumRows int64 `thrift:"3,required"` + + // If set, specifies a sort ordering of the rows in this RowGroup. + // The sorting columns can be a subset of all the columns. + SortingColumns []SortingColumn `thrift:"4,optional"` + + // Byte offset from beginning of file to first page (data or dictionary) + // in this row group + FileOffset int64 `thrift:"5,optional"` + + // Total byte size of all compressed (and potentially encrypted) column data + // in this row group. + TotalCompressedSize int64 `thrift:"6,optional"` + + // Row group ordinal in the file. + Ordinal int16 `thrift:"7,optional"` +} + +// Empty struct to signal the order defined by the physical or logical type. +type TypeDefinedOrder struct{} + +// Union to specify the order used for the min_value and max_value fields for a +// column. This union takes the role of an enhanced enum that allows rich +// elements (which will be needed for a collation-based ordering in the future). +// +// Possible values are: +// +// TypeDefinedOrder - the column uses the order defined by its logical or +// physical type (if there is no logical type). +// +// If the reader does not support the value of this union, min and max stats +// for this column should be ignored. +type ColumnOrder struct { // union + // The sort orders for logical types are: + // UTF8 - unsigned byte-wise comparison + // INT8 - signed comparison + // INT16 - signed comparison + // INT32 - signed comparison + // INT64 - signed comparison + // UINT8 - unsigned comparison + // UINT16 - unsigned comparison + // UINT32 - unsigned comparison + // UINT64 - unsigned comparison + // DECIMAL - signed comparison of the represented value + // DATE - signed comparison + // TIME_MILLIS - signed comparison + // TIME_MICROS - signed comparison + // TIMESTAMP_MILLIS - signed comparison + // TIMESTAMP_MICROS - signed comparison + // INTERVAL - unsigned comparison + // JSON - unsigned byte-wise comparison + // BSON - unsigned byte-wise comparison + // ENUM - unsigned byte-wise comparison + // LIST - undefined + // MAP - undefined + // VARIANT - undefined + // GEOMETRY - undefined + // GEOGRAPHY - undefined + // + // In the absence of logical types, the sort order is determined by the physical type: + // BOOLEAN - false, true + // INT32 - signed comparison + // INT64 - signed comparison + // INT96 (only used for legacy timestamps) - undefined + // FLOAT - signed comparison of the represented value (*) + // DOUBLE - signed comparison of the represented value (*) + // BYTE_ARRAY - unsigned byte-wise comparison + // FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison + // + // (*) Because the sorting order is not specified properly for floating + // point values (relations vs. total ordering) the following + // compatibility rules should be applied when reading statistics: + // - If the min is a NaN, it should be ignored. + // - If the max is a NaN, it should be ignored. + // - If the min is +0, the row group may contain -0 values as well. + // - If the max is -0, the row group may contain +0 values as well. + // - When looking for NaN values, min and max should be ignored. + TypeOrder *TypeDefinedOrder `thrift:"1"` +} + +type PageLocation struct { + // Offset of the page in the file. + Offset int64 `thrift:"1,required"` + + // Size of the page, including header. Sum of compressed_page_size and + // header length. + CompressedPageSize int32 `thrift:"2,required"` + + // Index within the RowGroup of the first row of the page; this means + // pages change on record boundaries (r = 0). + FirstRowIndex int64 `thrift:"3,required"` +} + +type OffsetIndex struct { + // PageLocations, ordered by increasing PageLocation.offset. It is required + // that page_locations[i].first_row_index < page_locations[i+1].first_row_index. + PageLocations []PageLocation `thrift:"1,required"` + + // Unencoded/uncompressed size for BYTE_ARRAY types. + // + // See documention for unencoded_byte_array_data_bytes in SizeStatistics for + // more details on this field. + UnencodedByteArrayDataBytes []int64 `thrift:"2,optional"` +} + +// Description for ColumnIndex. +// Each [i] refers to the page at OffsetIndex.PageLocations[i] +type ColumnIndex struct { + // A list of Boolean values to determine the validity of the corresponding + // min and max values. If true, a page contains only null values, and writers + // have to set the corresponding entries in min_values and max_values to + // byte[0], so that all lists have the same length. If false, the + // corresponding entries in min_values and max_values must be valid. + NullPages []bool `thrift:"1,required"` + + // Two lists containing lower and upper bounds for the values of each page + // determined by the ColumnOrder of the column. These may be the actual + // minimum and maximum values found on a page, but can also be (more compact) + // values that do not exist on a page. For example, instead of storing ""Blart + // Versenwald III", a writer may set min_values[i]="B", max_values[i]="C". + // Such more compact values must still be valid values within the column's + // logical type. Readers must make sure that list entries are populated before + // using them by inspecting null_pages. + MinValues [][]byte `thrift:"2,required"` + MaxValues [][]byte `thrift:"3,required"` + + // Stores whether both min_values and max_values are ordered and if so, in + // which direction. This allows readers to perform binary searches in both + // lists. Readers cannot assume that max_values[i] <= min_values[i+1], even + // if the lists are ordered. + BoundaryOrder BoundaryOrder `thrift:"4,required"` + + // A list containing the number of null values for each page. + NullCounts []int64 `thrift:"5,optional"` + + // Contains repetition level histograms for each page + // concatenated together. The repetition_level_histogram field on + // SizeStatistics contains more details. + // + // When present the length should always be (number of pages * + // (max_repetition_level + 1)) elements. + // + // Element 0 is the first element of the histogram for the first page. + // Element (max_repetition_level + 1) is the first element of the histogram + // for the second page. + RepetitionLevelHistogram []int64 `thrift:"6,optional"` + + // Same as repetition_level_histograms except for definitions levels. + DefinitionLevelHistogram []int64 `thrift:"7,optional"` +} + +type AesGcmV1 struct { + // AAD prefix. + AadPrefix []byte `thrift:"1,optional"` + + // Unique file identifier part of AAD suffix. + AadFileUnique []byte `thrift:"2,optional"` + + // In files encrypted with AAD prefix without storing it, + // readers must supply the prefix. + SupplyAadPrefix bool `thrift:"3,optional"` +} + +type AesGcmCtrV1 struct { + // AAD prefix. + AadPrefix []byte `thrift:"1,optional"` + + // Unique file identifier part of AAD suffix. + AadFileUnique []byte `thrift:"2,optional"` + + // In files encrypted with AAD prefix without storing it, + // readers must supply the prefix. + SupplyAadPrefix bool `thrift:"3,optional"` +} + +type EncryptionAlgorithm struct { // union + AesGcmV1 *AesGcmV1 `thrift:"1"` + AesGcmCtrV1 *AesGcmCtrV1 `thrift:"2"` +} + +// Description for file metadata. +type FileMetaData struct { + // Version of this file. + Version int32 `thrift:"1,required"` + + // Parquet schema for this file. This schema contains metadata for all the columns. + // The schema is represented as a tree with a single root. The nodes of the tree + // are flattened to a list by doing a depth-first traversal. + // The column metadata contains the path in the schema for that column which can be + // used to map columns to nodes in the schema. + // The first element is the root. + Schema []SchemaElement `thrift:"2,required"` + + // Number of rows in this file. + NumRows int64 `thrift:"3,required"` + + // Row groups in this file. + RowGroups []RowGroup `thrift:"4,required"` + + // Optional key/value metadata. + KeyValueMetadata []KeyValue `thrift:"5,optional"` + + // String for application that wrote this file. This should be in the format + // version (build ). + // e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55) + CreatedBy string `thrift:"6,optional"` + + // Sort order used for the min_value and max_value fields in the Statistics + // objects and the min_values and max_values fields in the ColumnIndex + // objects of each column in this file. Sort orders are listed in the order + // matching the columns in the schema. The indexes are not necessary the same + // though, because only leaf nodes of the schema are represented in the list + // of sort orders. + // + // Without column_orders, the meaning of the min_value and max_value fields + // in the Statistics object and the ColumnIndex object is undefined. To ensure + // well-defined behavior, if these fields are written to a Parquet file, + // column_orders must be written as well. + // + // The obsolete min and max fields in the Statistics object are always sorted + // by signed comparison regardless of column_orders. + ColumnOrders []ColumnOrder `thrift:"7,optional"` + + // Encryption algorithm. This field is set only in encrypted files + // with plaintext footer. Files with encrypted footer store algorithm id + // in FileCryptoMetaData structure. + EncryptionAlgorithm EncryptionAlgorithm `thrift:"8,optional"` + + // Retrieval metadata of key used for signing the footer. + // Used only in encrypted files with plaintext footer. + FooterSigningKeyMetadata []byte `thrift:"9,optional"` +} + +// Crypto metadata for files with encrypted footer. +type FileCryptoMetaData struct { + // Encryption algorithm. This field is only used for files + // with encrypted footer. Files with plaintext footer store algorithm id + // inside footer (FileMetaData structure). + EncryptionAlgorithm EncryptionAlgorithm `thrift:"1,required"` + + // Retrieval metadata of key used for encryption of footer, + // and (possibly) columns. + KeyMetadata []byte `thrift:"2,optional"` +} diff --git a/vendor/github.com/parquet-go/parquet-go/go.tools.mod b/vendor/github.com/parquet-go/parquet-go/go.tools.mod new file mode 100644 index 0000000000..73b3aa71c5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/go.tools.mod @@ -0,0 +1,26 @@ +module github.com/parquet-go/parquet-go + +go 1.23.4 + +toolchain go1.24.0 + +tool golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize + +require ( + github.com/andybalholm/brotli v1.1.1 + github.com/google/uuid v1.6.0 + github.com/hexops/gotextdiff v1.0.3 + github.com/klauspost/compress v1.18.0 + github.com/olekukonko/tablewriter v0.0.5 + github.com/pierrec/lz4/v4 v4.1.22 + golang.org/x/sys v0.30.0 + google.golang.org/protobuf v1.36.5 +) + +require ( + github.com/mattn/go-runewidth v0.0.9 // indirect + golang.org/x/mod v0.23.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/tools v0.30.1-0.20250221230316-5055f70f240c // indirect + golang.org/x/tools/gopls v0.18.1 // indirect +) diff --git a/vendor/github.com/parquet-go/parquet-go/go.tools.sum b/vendor/github.com/parquet-go/parquet-go/go.tools.sum new file mode 100644 index 0000000000..c3d8f09d04 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/go.tools.sum @@ -0,0 +1,30 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.30.1-0.20250221230316-5055f70f240c h1:Ja/5gV5a9Vvho3p2NC/T2TtxhHjrWS/2DvCKMvA0a+Y= +golang.org/x/tools v0.30.1-0.20250221230316-5055f70f240c/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/tools/gopls v0.18.1 h1:2xJBNzdImS5u/kV/ZzqDLSvlBSeZX+pWY9uKVP7Pask= +golang.org/x/tools/gopls v0.18.1/go.mod h1:UdNu0zeGjkmjL9L20QDszXu9tP2798pUIHC980kOBrI= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash.go new file mode 100644 index 0000000000..4e1018a017 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash.go @@ -0,0 +1,21 @@ +// Package aeshash implements hashing functions derived from the Go runtime's +// internal hashing based on the support of AES encryption in CPU instructions. +// +// On architecture where the CPU does not provide instructions for AES +// encryption, the aeshash.Enabled function always returns false, and attempting +// to call any other function will trigger a panic. +package aeshash + +import "github.com/parquet-go/parquet-go/sparse" + +func MultiHash32(hashes []uintptr, values []uint32, seed uintptr) { + MultiHashUint32Array(hashes, sparse.MakeUint32Array(values), seed) +} + +func MultiHash64(hashes []uintptr, values []uint64, seed uintptr) { + MultiHashUint64Array(hashes, sparse.MakeUint64Array(values), seed) +} + +func MultiHash128(hashes []uintptr, values [][16]byte, seed uintptr) { + MultiHashUint128Array(hashes, sparse.MakeUint128Array(values), seed) +} diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.go new file mode 100644 index 0000000000..8b6ee21290 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.go @@ -0,0 +1,60 @@ +//go:build !purego + +package aeshash + +import ( + "math/rand" + "unsafe" + + "golang.org/x/sys/cpu" + + "github.com/parquet-go/parquet-go/sparse" +) + +// hashRandomBytes is 48 since this is what the assembly code depends on. +const hashRandomBytes = 48 + +var aeskeysched [hashRandomBytes]byte + +func init() { + for _, v := range aeskeysched { + if v != 0 { + // aeskeysched was initialized somewhere else (e.g. tests), so we + // can skip initialization. No synchronization is needed since init + // functions are called sequentially in a single goroutine (see + // https://go.dev/ref/spec#Package_initialization). + return + } + } + + key := (*[hashRandomBytes / 8]uint64)(unsafe.Pointer(&aeskeysched)) + for i := range key { + key[i] = rand.Uint64() + } +} + +// Enabled returns true if AES hash is available on the system. +// +// The function uses the same logic than the Go runtime since we depend on +// the AES hash state being initialized. +// +// See https://go.dev/src/runtime/alg.go +func Enabled() bool { return cpu.X86.HasAES && cpu.X86.HasSSSE3 && cpu.X86.HasSSE41 } + +//go:noescape +func Hash32(value uint32, seed uintptr) uintptr + +//go:noescape +func Hash64(value uint64, seed uintptr) uintptr + +//go:noescape +func Hash128(value [16]byte, seed uintptr) uintptr + +//go:noescape +func MultiHashUint32Array(hashes []uintptr, values sparse.Uint32Array, seed uintptr) + +//go:noescape +func MultiHashUint64Array(hashes []uintptr, values sparse.Uint64Array, seed uintptr) + +//go:noescape +func MultiHashUint128Array(hashes []uintptr, values sparse.Uint128Array, seed uintptr) diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.s b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.s new file mode 100644 index 0000000000..ad6812aeba --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_amd64.s @@ -0,0 +1,155 @@ +//go:build !purego + +#include "textflag.h" + +// func Hash32(value uint32, seed uintptr) uintptr +TEXT ยทHash32(SB), NOSPLIT, $0-24 + MOVL value+0(FP), AX + MOVQ seed+8(FP), BX + + MOVOU ยทaeskeysched+0(SB), X1 + MOVOU ยทaeskeysched+16(SB), X2 + MOVOU ยทaeskeysched+32(SB), X3 + + MOVQ BX, X0 + PINSRD $2, AX, X0 + + AESENC X1, X0 + AESENC X2, X0 + AESENC X3, X0 + + MOVQ X0, ret+16(FP) + RET + +// func Hash64(value uint64, seed uintptr) uintptr +TEXT ยทHash64(SB), NOSPLIT, $0-24 + MOVQ value+0(FP), AX + MOVQ seed+8(FP), BX + + MOVOU ยทaeskeysched+0(SB), X1 + MOVOU ยทaeskeysched+16(SB), X2 + MOVOU ยทaeskeysched+32(SB), X3 + + MOVQ BX, X0 + PINSRQ $1, AX, X0 + + AESENC X1, X0 + AESENC X2, X0 + AESENC X3, X0 + + MOVQ X0, ret+16(FP) + RET + +// func Hash128(value [16]byte, seed uintptr) uintptr +TEXT ยทHash128(SB), NOSPLIT, $0-32 + LEAQ value+0(FP), AX + MOVQ seed+16(FP), BX + MOVQ $16, CX + + MOVQ BX, X0 // 64 bits of per-table hash seed + PINSRW $4, CX, X0 // 16 bits of length + PSHUFHW $0, X0, X0 // repeat length 4 times total + PXOR ยทaeskeysched(SB), X0 // xor in per-process seed + AESENC X0, X0 // scramble seed + + MOVOU (AX), X1 + PXOR X0, X1 + AESENC X1, X1 + AESENC X1, X1 + AESENC X1, X1 + + MOVQ X1, ret+24(FP) + RET + +// func MultiHashUint32Array(hashes []uintptr, values sparse.Uint32Array, seed uintptr) +TEXT ยทMultiHashUint32Array(SB), NOSPLIT, $0-56 + MOVQ hashes_base+0(FP), AX + MOVQ values_array_ptr+24(FP), BX + MOVQ values_array_len+32(FP), CX + MOVQ values_array_off+40(FP), R8 + MOVQ seed+48(FP), DX + + MOVOU ยทaeskeysched+0(SB), X1 + MOVOU ยทaeskeysched+16(SB), X2 + MOVOU ยทaeskeysched+32(SB), X3 + + XORQ SI, SI + JMP test +loop: + MOVQ DX, X0 + PINSRD $2, (BX), X0 + + AESENC X1, X0 + AESENC X2, X0 + AESENC X3, X0 + + MOVQ X0, (AX)(SI*8) + INCQ SI + ADDQ R8, BX +test: + CMPQ SI, CX + JNE loop + RET + +// func MultiHashUint64Array(hashes []uintptr, values sparse.Uint64Array, seed uintptr) +TEXT ยทMultiHashUint64Array(SB), NOSPLIT, $0-56 + MOVQ hashes_base+0(FP), AX + MOVQ values_array_ptr+24(FP), BX + MOVQ values_array_len+32(FP), CX + MOVQ values_array_off+40(FP), R8 + MOVQ seed+48(FP), DX + + MOVOU ยทaeskeysched+0(SB), X1 + MOVOU ยทaeskeysched+16(SB), X2 + MOVOU ยทaeskeysched+32(SB), X3 + + XORQ SI, SI + JMP test +loop: + MOVQ DX, X0 + PINSRQ $1, (BX), X0 + + AESENC X1, X0 + AESENC X2, X0 + AESENC X3, X0 + + MOVQ X0, (AX)(SI*8) + INCQ SI + ADDQ R8, BX +test: + CMPQ SI, CX + JNE loop + RET + +// func MultiHashUint128Array(hashes []uintptr, values sparse.Uint128Array, seed uintptr) +TEXT ยทMultiHashUint128Array(SB), NOSPLIT, $0-56 + MOVQ hashes_base+0(FP), AX + MOVQ values_array_ptr+24(FP), BX + MOVQ values_array_len+32(FP), CX + MOVQ values_array_off+40(FP), R8 + MOVQ seed+48(FP), DX + MOVQ $16, DI + + MOVQ DX, X0 + PINSRW $4, DI, X0 + PSHUFHW $0, X0, X0 + PXOR ยทaeskeysched(SB), X0 + AESENC X0, X0 + + XORQ SI, SI + JMP test +loop: + MOVOU (BX), X1 + + PXOR X0, X1 + AESENC X1, X1 + AESENC X1, X1 + AESENC X1, X1 + + MOVQ X1, (AX)(SI*8) + INCQ SI + ADDQ R8, BX +test: + CMPQ SI, CX + JNE loop + RET diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_purego.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_purego.go new file mode 100644 index 0000000000..42d367a0de --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/aeshash/aeshash_purego.go @@ -0,0 +1,29 @@ +//go:build purego || !amd64 + +package aeshash + +import "github.com/parquet-go/parquet-go/sparse" + +// Enabled always returns false since we assume that AES instructions are not +// available by default. +func Enabled() bool { return false } + +const unsupported = "BUG: AES hash is not available on this platform" + +func Hash32(value uint32, seed uintptr) uintptr { panic(unsupported) } + +func Hash64(value uint64, seed uintptr) uintptr { panic(unsupported) } + +func Hash128(value [16]byte, seed uintptr) uintptr { panic(unsupported) } + +func MultiHashUint32Array(hashes []uintptr, values sparse.Uint32Array, seed uintptr) { + panic(unsupported) +} + +func MultiHashUint64Array(hashes []uintptr, values sparse.Uint64Array, seed uintptr) { + panic(unsupported) +} + +func MultiHashUint128Array(hashes []uintptr, values sparse.Uint128Array, seed uintptr) { + panic(unsupported) +} diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe.go new file mode 100644 index 0000000000..0a1686f17b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe.go @@ -0,0 +1,783 @@ +// Package hashprobe provides implementations of probing tables for various +// data types. +// +// Probing tables are specialized hash tables supporting only a single +// "probing" operation which behave like a "lookup or insert". When a key +// is probed, either its value is retrieved if it already existed in the table, +// or it is inserted and assigned its index in the insert sequence as value. +// +// Values are represented as signed 32 bits integers, which means that probing +// tables defined in this package may contain at most 2^31-1 entries. +// +// Probing tables have a method named Probe with the following signature: +// +// func (t *Int64Table) Probe(keys []int64, values []int32) int { +// ... +// } +// +// The method takes an array of keys to probe as first argument, an array of +// values where the indexes of each key will be written as second argument, and +// returns the number of keys that were inserted during the call. +// +// Applications that need to determine which keys were inserted can capture the +// length of the probing table prior to the call, and scan the list of values +// looking for indexes greater or equal to the length of the table before the +// call. +package hashprobe + +import ( + cryptoRand "crypto/rand" + "encoding/binary" + "math" + "math/bits" + "math/rand" + "sync" + + "github.com/parquet-go/parquet-go/hashprobe/aeshash" + "github.com/parquet-go/parquet-go/hashprobe/wyhash" + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" +) + +const ( + // Number of probes tested per iteration. This parameter balances between + // the amount of memory allocated on the stack to hold the computed hashes + // of the keys being probed, and amortizing the baseline cost of the probing + // algorithm. + // + // The larger the value, the more memory is required, but lower the baseline + // cost will be. + // + // We chose a value that is somewhat large, resulting in reserving 2KiB of + // stack but mostly erasing the baseline cost. + probesPerLoop = 256 +) + +var ( + prngSeed [8]byte + prngMutex sync.Mutex + prngSource rand.Source64 +) + +func init() { + _, err := cryptoRand.Read(prngSeed[:]) + if err != nil { + panic("cannot seed random number generator from system source: " + err.Error()) + } + seed := int64(binary.LittleEndian.Uint64(prngSeed[:])) + prngSource = rand.NewSource(seed).(rand.Source64) +} + +func tableSizeAndMaxLen(groupSize, numValues int, maxLoad float64) (size, maxLen int) { + n := int(math.Ceil((1 / maxLoad) * float64(numValues))) + size = nextPowerOf2((n + (groupSize - 1)) / groupSize) + maxLen = int(math.Ceil(maxLoad * float64(groupSize*size))) + return +} + +func nextPowerOf2(n int) int { + return 1 << (64 - bits.LeadingZeros64(uint64(n-1))) +} + +func randSeed() uintptr { + prngMutex.Lock() + defer prngMutex.Unlock() + return uintptr(prngSource.Uint64()) +} + +type Int32Table struct{ table32 } + +func NewInt32Table(cap int, maxLoad float64) *Int32Table { + return &Int32Table{makeTable32(cap, maxLoad)} +} + +func (t *Int32Table) Reset() { t.reset() } + +func (t *Int32Table) Len() int { return t.len } + +func (t *Int32Table) Cap() int { return t.size() } + +func (t *Int32Table) Probe(keys, values []int32) int { + return t.probe(unsafecast.Slice[uint32](keys), values) +} + +func (t *Int32Table) ProbeArray(keys sparse.Int32Array, values []int32) int { + return t.probeArray(keys.Uint32Array(), values) +} + +type Float32Table struct{ table32 } + +func NewFloat32Table(cap int, maxLoad float64) *Float32Table { + return &Float32Table{makeTable32(cap, maxLoad)} +} + +func (t *Float32Table) Reset() { t.reset() } + +func (t *Float32Table) Len() int { return t.len } + +func (t *Float32Table) Cap() int { return t.size() } + +func (t *Float32Table) Probe(keys []float32, values []int32) int { + return t.probe(unsafecast.Slice[uint32](keys), values) +} + +func (t *Float32Table) ProbeArray(keys sparse.Float32Array, values []int32) int { + return t.probeArray(keys.Uint32Array(), values) +} + +type Uint32Table struct{ table32 } + +func NewUint32Table(cap int, maxLoad float64) *Uint32Table { + return &Uint32Table{makeTable32(cap, maxLoad)} +} + +func (t *Uint32Table) Reset() { t.reset() } + +func (t *Uint32Table) Len() int { return t.len } + +func (t *Uint32Table) Cap() int { return t.size() } + +func (t *Uint32Table) Probe(keys []uint32, values []int32) int { + return t.probe(keys, values) +} + +func (t *Uint32Table) ProbeArray(keys sparse.Uint32Array, values []int32) int { + return t.probeArray(keys, values) +} + +// table32 is the generic implementation of probing tables for 32 bit types. +// +// The table uses the following memory layout: +// +// [group 0][group 1][...][group N] +// +// Each group contains up to 7 key/value pairs, and is exactly 64 bytes in size, +// which allows it to fit within a single cache line, and ensures that probes +// can be performed with a single memory load per key. +// +// Groups fill up by appending new entries to the keys and values arrays. When a +// group is full, the probe checks the next group. +// +// https://en.wikipedia.org/wiki/Linear_probing +type table32 struct { + len int + maxLen int + maxLoad float64 + seed uintptr + table []table32Group +} + +const table32GroupSize = 7 + +type table32Group struct { + keys [table32GroupSize]uint32 + values [table32GroupSize]uint32 + bits uint32 + _ uint32 +} + +func makeTable32(cap int, maxLoad float64) (t table32) { + if maxLoad < 0 || maxLoad > 1 { + panic("max load of probing table must be a value between 0 and 1") + } + if cap < table32GroupSize { + cap = table32GroupSize + } + t.init(cap, maxLoad) + return t +} + +func (t *table32) size() int { + return table32GroupSize * len(t.table) +} + +func (t *table32) init(cap int, maxLoad float64) { + size, maxLen := tableSizeAndMaxLen(table32GroupSize, cap, maxLoad) + *t = table32{ + maxLen: maxLen, + maxLoad: maxLoad, + seed: randSeed(), + table: make([]table32Group, size), + } +} + +func (t *table32) grow(totalValues int) { + tmp := table32{} + tmp.init(totalValues, t.maxLoad) + tmp.len = t.len + + hashes := make([]uintptr, table32GroupSize) + modulo := uintptr(len(tmp.table)) - 1 + + for i := range t.table { + g := &t.table[i] + n := bits.OnesCount32(g.bits) + + if aeshash.Enabled() { + aeshash.MultiHash32(hashes[:n], g.keys[:n], tmp.seed) + } else { + wyhash.MultiHash32(hashes[:n], g.keys[:n], tmp.seed) + } + + for j, hash := range hashes[:n] { + for { + group := &tmp.table[hash&modulo] + + if n := bits.OnesCount32(group.bits); n < table32GroupSize { + group.bits = (group.bits << 1) | 1 + group.keys[n] = g.keys[j] + group.values[n] = g.values[j] + break + } + + hash++ + } + } + } + + *t = tmp +} + +func (t *table32) reset() { + t.len = 0 + + for i := range t.table { + t.table[i] = table32Group{} + } +} + +func (t *table32) probe(keys []uint32, values []int32) int { + return t.probeArray(sparse.MakeUint32Array(keys), values) +} + +func (t *table32) probeArray(keys sparse.Uint32Array, values []int32) int { + numKeys := keys.Len() + + if totalValues := t.len + numKeys; totalValues > t.maxLen { + t.grow(totalValues) + } + + var hashes [probesPerLoop]uintptr + var baseLength = t.len + var useAesHash = aeshash.Enabled() + + _ = values[:numKeys] + + for i := 0; i < numKeys; { + j := len(hashes) + i + n := len(hashes) + + if j > numKeys { + j = numKeys + n = numKeys - i + } + + k := keys.Slice(i, j) + v := values[i:j:j] + h := hashes[:n:n] + + if useAesHash { + aeshash.MultiHashUint32Array(h, k, t.seed) + } else { + wyhash.MultiHashUint32Array(h, k, t.seed) + } + + t.len = multiProbe32(t.table, t.len, h, k, v) + i = j + } + + return t.len - baseLength +} + +func multiProbe32Default(table []table32Group, numKeys int, hashes []uintptr, keys sparse.Uint32Array, values []int32) int { + modulo := uintptr(len(table)) - 1 + + for i, hash := range hashes { + key := keys.Index(i) + for { + group := &table[hash&modulo] + index := table32GroupSize + value := int32(0) + + for j, k := range group.keys { + if k == key { + index = j + break + } + } + + if n := bits.OnesCount32(group.bits); index < n { + value = int32(group.values[index]) + } else { + if n == table32GroupSize { + hash++ + continue + } + + value = int32(numKeys) + group.bits = (group.bits << 1) | 1 + group.keys[n] = key + group.values[n] = uint32(value) + numKeys++ + } + + values[i] = value + break + } + } + + return numKeys +} + +type Int64Table struct{ table64 } + +func NewInt64Table(cap int, maxLoad float64) *Int64Table { + return &Int64Table{makeTable64(cap, maxLoad)} +} + +func (t *Int64Table) Reset() { t.reset() } + +func (t *Int64Table) Len() int { return t.len } + +func (t *Int64Table) Cap() int { return t.size() } + +func (t *Int64Table) Probe(keys []int64, values []int32) int { + return t.probe(unsafecast.Slice[uint64](keys), values) +} + +func (t *Int64Table) ProbeArray(keys sparse.Int64Array, values []int32) int { + return t.probeArray(keys.Uint64Array(), values) +} + +type Float64Table struct{ table64 } + +func NewFloat64Table(cap int, maxLoad float64) *Float64Table { + return &Float64Table{makeTable64(cap, maxLoad)} +} + +func (t *Float64Table) Reset() { t.reset() } + +func (t *Float64Table) Len() int { return t.len } + +func (t *Float64Table) Cap() int { return t.size() } + +func (t *Float64Table) Probe(keys []float64, values []int32) int { + return t.probe(unsafecast.Slice[uint64](keys), values) +} + +func (t *Float64Table) ProbeArray(keys sparse.Float64Array, values []int32) int { + return t.probeArray(keys.Uint64Array(), values) +} + +type Uint64Table struct{ table64 } + +func NewUint64Table(cap int, maxLoad float64) *Uint64Table { + return &Uint64Table{makeTable64(cap, maxLoad)} +} + +func (t *Uint64Table) Reset() { t.reset() } + +func (t *Uint64Table) Len() int { return t.len } + +func (t *Uint64Table) Cap() int { return t.size() } + +func (t *Uint64Table) Probe(keys []uint64, values []int32) int { + return t.probe(keys, values) +} + +func (t *Uint64Table) ProbeArray(keys sparse.Uint64Array, values []int32) int { + return t.probeArray(keys, values) +} + +// table64 is the generic implementation of probing tables for 64 bit types. +// +// The table uses a layout similar to the one documented on the table for 32 bit +// keys (see table32). Each group holds up to 4 key/value pairs (instead of 7 +// like table32) so that each group fits in a single CPU cache line. This table +// version has a bit lower memory density, with ~23% of table memory being used +// for padding. +// +// Technically we could hold up to 5 entries per group and still fit within the +// 64 bytes of a CPU cache line; on x86 platforms, AVX2 registers can only hold +// four 64 bit values, we would need twice as many instructions per probe if the +// groups were holding 5 values. The trade off of memory for compute efficiency +// appeared to be the right choice at the time. +type table64 struct { + len int + maxLen int + maxLoad float64 + seed uintptr + table []table64Group +} + +const table64GroupSize = 4 + +type table64Group struct { + keys [table64GroupSize]uint64 + values [table64GroupSize]uint32 + bits uint32 + _ uint32 + _ uint32 + _ uint32 +} + +func makeTable64(cap int, maxLoad float64) (t table64) { + if maxLoad < 0 || maxLoad > 1 { + panic("max load of probing table must be a value between 0 and 1") + } + if cap < table64GroupSize { + cap = table64GroupSize + } + t.init(cap, maxLoad) + return t +} + +func (t *table64) size() int { + return table64GroupSize * len(t.table) +} + +func (t *table64) init(cap int, maxLoad float64) { + size, maxLen := tableSizeAndMaxLen(table64GroupSize, cap, maxLoad) + *t = table64{ + maxLen: maxLen, + maxLoad: maxLoad, + seed: randSeed(), + table: make([]table64Group, size), + } +} + +func (t *table64) grow(totalValues int) { + tmp := table64{} + tmp.init(totalValues, t.maxLoad) + tmp.len = t.len + + hashes := make([]uintptr, table64GroupSize) + modulo := uintptr(len(tmp.table)) - 1 + + for i := range t.table { + g := &t.table[i] + n := bits.OnesCount32(g.bits) + + if aeshash.Enabled() { + aeshash.MultiHash64(hashes[:n], g.keys[:n], tmp.seed) + } else { + wyhash.MultiHash64(hashes[:n], g.keys[:n], tmp.seed) + } + + for j, hash := range hashes[:n] { + for { + group := &tmp.table[hash&modulo] + + if n := bits.OnesCount32(group.bits); n < table64GroupSize { + group.bits = (group.bits << 1) | 1 + group.keys[n] = g.keys[j] + group.values[n] = g.values[j] + break + } + + hash++ + } + } + } + + *t = tmp +} + +func (t *table64) reset() { + t.len = 0 + + for i := range t.table { + t.table[i] = table64Group{} + } +} + +func (t *table64) probe(keys []uint64, values []int32) int { + return t.probeArray(sparse.MakeUint64Array(keys), values) +} + +func (t *table64) probeArray(keys sparse.Uint64Array, values []int32) int { + numKeys := keys.Len() + + if totalValues := t.len + numKeys; totalValues > t.maxLen { + t.grow(totalValues) + } + + var hashes [probesPerLoop]uintptr + var baseLength = t.len + var useAesHash = aeshash.Enabled() + + _ = values[:numKeys] + + for i := 0; i < numKeys; { + j := len(hashes) + i + n := len(hashes) + + if j > numKeys { + j = numKeys + n = numKeys - i + } + + k := keys.Slice(i, j) + v := values[i:j:j] + h := hashes[:n:n] + + if useAesHash { + aeshash.MultiHashUint64Array(h, k, t.seed) + } else { + wyhash.MultiHashUint64Array(h, k, t.seed) + } + + t.len = multiProbe64(t.table, t.len, h, k, v) + i = j + } + + return t.len - baseLength +} + +func multiProbe64Default(table []table64Group, numKeys int, hashes []uintptr, keys sparse.Uint64Array, values []int32) int { + modulo := uintptr(len(table)) - 1 + + for i, hash := range hashes { + key := keys.Index(i) + for { + group := &table[hash&modulo] + index := table64GroupSize + value := int32(0) + + for i, k := range group.keys { + if k == key { + index = i + break + } + } + + if n := bits.OnesCount32(group.bits); index < n { + value = int32(group.values[index]) + } else { + if n == table64GroupSize { + hash++ + continue + } + + value = int32(numKeys) + group.bits = (group.bits << 1) | 1 + group.keys[n] = key + group.values[n] = uint32(value) + numKeys++ + } + + values[i] = value + break + } + } + + return numKeys +} + +type Uint128Table struct{ table128 } + +func NewUint128Table(cap int, maxLoad float64) *Uint128Table { + return &Uint128Table{makeTable128(cap, maxLoad)} +} + +func (t *Uint128Table) Reset() { t.reset() } + +func (t *Uint128Table) Len() int { return t.len } + +func (t *Uint128Table) Cap() int { return t.cap } + +func (t *Uint128Table) Probe(keys [][16]byte, values []int32) int { + return t.probe(keys, values) +} + +func (t *Uint128Table) ProbeArray(keys sparse.Uint128Array, values []int32) int { + return t.probeArray(keys, values) +} + +// table128 is the generic implementation of probing tables for 128 bit types. +// +// This table uses the following memory layout: +// +// [key A][key B][...][value A][value B][...] +// +// The table stores values as their actual value plus one, and uses zero as a +// sentinel to determine whether a slot is occupied. A linear probing strategy +// is used to resolve conflicts. This approach results in at most two memory +// loads for every four keys being tested, since the location of a key and its +// corresponding value will not be contiguous on the same CPU cache line, but +// a cache line can hold four 16 byte keys. +type table128 struct { + len int + cap int + maxLen int + maxLoad float64 + seed uintptr + table []byte +} + +func makeTable128(cap int, maxLoad float64) (t table128) { + if maxLoad < 0 || maxLoad > 1 { + panic("max load of probing table must be a value between 0 and 1") + } + if cap < 8 { + cap = 8 + } + t.init(cap, maxLoad) + return t +} + +func (t *table128) init(cap int, maxLoad float64) { + size, maxLen := tableSizeAndMaxLen(1, cap, maxLoad) + *t = table128{ + cap: size, + maxLen: maxLen, + maxLoad: maxLoad, + seed: randSeed(), + table: make([]byte, 16*size+4*size), + } +} + +func (t *table128) kv() (keys [][16]byte, values []int32) { + i := t.cap * 16 + return unsafecast.Slice[[16]byte](t.table[:i]), unsafecast.Slice[int32](t.table[i:]) +} + +func (t *table128) grow(totalValues int) { + tmp := table128{} + tmp.init(totalValues, t.maxLoad) + tmp.len = t.len + + keys, values := t.kv() + hashes := make([]uintptr, probesPerLoop) + useAesHash := aeshash.Enabled() + + _ = values[:len(keys)] + + for i := 0; i < len(keys); { + j := len(hashes) + i + n := len(hashes) + + if j > len(keys) { + j = len(keys) + n = len(keys) - i + } + + h := hashes[:n:n] + k := keys[i:j:j] + v := values[i:j:j] + + if useAesHash { + aeshash.MultiHash128(h, k, tmp.seed) + } else { + wyhash.MultiHash128(h, k, tmp.seed) + } + + tmp.insert(h, k, v) + i = j + } + + *t = tmp +} + +func (t *table128) insert(hashes []uintptr, keys [][16]byte, values []int32) { + tableKeys, tableValues := t.kv() + modulo := uintptr(t.cap) - 1 + + for i, hash := range hashes { + for { + j := hash & modulo + v := tableValues[j] + + if v == 0 { + tableKeys[j] = keys[i] + tableValues[j] = values[i] + break + } + + hash++ + } + } +} + +func (t *table128) reset() { + t.len = 0 + + for i := range t.table { + t.table[i] = 0 + } +} + +func (t *table128) probe(keys [][16]byte, values []int32) int { + return t.probeArray(sparse.MakeUint128Array(keys), values) +} + +func (t *table128) probeArray(keys sparse.Uint128Array, values []int32) int { + numKeys := keys.Len() + + if totalValues := t.len + numKeys; totalValues > t.maxLen { + t.grow(totalValues) + } + + var hashes [probesPerLoop]uintptr + var baseLength = t.len + var useAesHash = aeshash.Enabled() + + _ = values[:numKeys] + + for i := 0; i < numKeys; { + j := len(hashes) + i + n := len(hashes) + + if j > numKeys { + j = numKeys + n = numKeys - i + } + + k := keys.Slice(i, j) + v := values[i:j:j] + h := hashes[:n:n] + + if useAesHash { + aeshash.MultiHashUint128Array(h, k, t.seed) + } else { + wyhash.MultiHashUint128Array(h, k, t.seed) + } + + t.len = multiProbe128(t.table, t.cap, t.len, h, k, v) + i = j + } + + return t.len - baseLength +} + +func multiProbe128Default(table []byte, tableCap, tableLen int, hashes []uintptr, keys sparse.Uint128Array, values []int32) int { + modulo := uintptr(tableCap) - 1 + offset := uintptr(tableCap) * 16 + tableKeys := unsafecast.Slice[[16]byte](table[:offset]) + tableValues := unsafecast.Slice[int32](table[offset:]) + + for i, hash := range hashes { + key := keys.Index(i) + for { + j := hash & modulo + v := tableValues[j] + + if v == 0 { + values[i] = int32(tableLen) + tableLen++ + tableKeys[j] = key + tableValues[j] = int32(tableLen) + break + } + + if key == tableKeys[j] { + values[i] = v - 1 + break + } + + hash++ + } + } + + return tableLen +} diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.go new file mode 100644 index 0000000000..8802d151a6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.go @@ -0,0 +1,38 @@ +//go:build !purego + +package hashprobe + +import ( + "github.com/parquet-go/parquet-go/sparse" + "golang.org/x/sys/cpu" +) + +//go:noescape +func multiProbe32AVX2(table []table32Group, numKeys int, hashes []uintptr, keys sparse.Uint32Array, values []int32) int + +//go:noescape +func multiProbe64AVX2(table []table64Group, numKeys int, hashes []uintptr, keys sparse.Uint64Array, values []int32) int + +//go:noescape +func multiProbe128SSE2(table []byte, tableCap, tableLen int, hashes []uintptr, keys sparse.Uint128Array, values []int32) int + +func multiProbe32(table []table32Group, numKeys int, hashes []uintptr, keys sparse.Uint32Array, values []int32) int { + if cpu.X86.HasAVX2 { + return multiProbe32AVX2(table, numKeys, hashes, keys, values) + } + return multiProbe32Default(table, numKeys, hashes, keys, values) +} + +func multiProbe64(table []table64Group, numKeys int, hashes []uintptr, keys sparse.Uint64Array, values []int32) int { + if cpu.X86.HasAVX2 { + return multiProbe64AVX2(table, numKeys, hashes, keys, values) + } + return multiProbe64Default(table, numKeys, hashes, keys, values) +} + +func multiProbe128(table []byte, tableCap, tableLen int, hashes []uintptr, keys sparse.Uint128Array, values []int32) int { + if cpu.X86.HasSSE2 { + return multiProbe128SSE2(table, tableCap, tableLen, hashes, keys, values) + } + return multiProbe128Default(table, tableCap, tableLen, hashes, keys, values) +} diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.s b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.s new file mode 100644 index 0000000000..5c5c0c02f7 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_amd64.s @@ -0,0 +1,197 @@ +//go:build !purego + +#include "textflag.h" + +// This version of the probing algorithm for 32 bit keys takes advantage of +// the memory layout of table groups and SIMD instructions to accelerate the +// probing operations. +// +// The first 32 bytes of a table group contain the bit mask indicating which +// slots are in use, and the array of keys, which fits into a single vector +// register (YMM) and can be loaded and tested with a single instruction. +// +// A first version of the table group used the number of keys held in the +// group instead of a bit mask, which required the probing operation to +// reconstruct the bit mask during the lookup operation in order to identify +// which elements of the VPCMPEQD result should be retained. The extra CPU +// instructions used to reconstruct the bit mask had a measurable overhead. +// By holding the bit mask in the data structure, we can determine the number +// of keys in a group using the POPCNT instruction, and avoid recomputing the +// mask during lookups. +// +// func multiProbe32AVX2(table []table32Group, numKeys int, hashes []uintptr, keys sparse.Uint32Array, values []int32) int +TEXT ยทmultiProbe32AVX2(SB), NOSPLIT, $0-112 + MOVQ table_base+0(FP), AX + MOVQ table_len+8(FP), BX + MOVQ numKeys+24(FP), CX + MOVQ hashes_base+32(FP), DX + MOVQ hashes_len+40(FP), DI + MOVQ keys_array_ptr+56(FP), R8 + MOVQ keys_array_off+72(FP), R15 + MOVQ values_base+80(FP), R9 + DECQ BX // modulo = len(table) - 1 + + XORQ SI, SI + JMP test +loop: + MOVQ (DX)(SI*8), R10 // hash + VPBROADCASTD (R8), Y0 // [key] +probe: + MOVQ R10, R11 + ANDQ BX, R11 // hash & modulo + SHLQ $6, R11 // x 64 (size of table32Group) + LEAQ (AX)(R11*1), R12 + + VMOVDQU (R12), Y1 + VPCMPEQD Y0, Y1, Y2 + VMOVMSKPS Y2, R11 + MOVL 56(R12), R13 + TESTL R11, R13 + JZ insert + + TZCNTL R11, R13 + MOVL 28(R12)(R13*4), R14 +next: + MOVL R14, (R9)(SI*4) + INCQ SI + ADDQ R15, R8 +test: + CMPQ SI, DI + JNE loop + MOVQ CX, ret+104(FP) + VZEROUPPER + RET +insert: + CMPL R13, $0b1111111 + JE probeNextGroup + + MOVL R13, R11 + POPCNTL R13, R13 + MOVQ X0, R14 // key + SHLL $1, R11 + ORL $1, R11 + MOVL R11, 56(R12) // group.len = (group.len << 1) | 1 + MOVL R14, (R12)(R13*4) // group.keys[i] = key + MOVL CX, 28(R12)(R13*4) // group.values[i] = value + MOVL CX, R14 + INCL CX + JMP next +probeNextGroup: + INCQ R10 + JMP probe + +// func multiProbe64AVX2(table []table64Group, numKeys int, hashes []uintptr, keys sparse.Uint64Array, values []int32) int +TEXT ยทmultiProbe64AVX2(SB), NOSPLIT, $0-112 + MOVQ table_base+0(FP), AX + MOVQ table_len+8(FP), BX + MOVQ numKeys+24(FP), CX + MOVQ hashes_base+32(FP), DX + MOVQ hashes_len+40(FP), DI + MOVQ keys_array_ptr+56(FP), R8 + MOVQ keys_array_off+72(FP), R15 + MOVQ values_base+80(FP), R9 + DECQ BX // modulo = len(table) - 1 + + XORQ SI, SI + JMP test +loop: + MOVQ (DX)(SI*8), R10 // hash + VPBROADCASTQ (R8), Y0 // [key] +probe: + MOVQ R10, R11 + ANDQ BX, R11 // hash & modulo + SHLQ $6, R11 // x 64 (size of table64Group) + LEAQ (AX)(R11*1), R12 + + VMOVDQU (R12), Y1 + VPCMPEQQ Y0, Y1, Y2 + VMOVMSKPD Y2, R11 + MOVL 48(R12), R13 + TESTL R11, R13 + JZ insert + + TZCNTL R11, R13 + MOVL 32(R12)(R13*4), R14 +next: + MOVL R14, (R9)(SI*4) + INCQ SI + ADDQ R15, R8 +test: + CMPQ SI, DI + JNE loop + MOVQ CX, ret+104(FP) + VZEROUPPER + RET +insert: + CMPL R13, $0b1111 + JE probeNextGroup + + MOVL R13, R11 + POPCNTL R13, R13 + SHLL $1, R11 + ORL $1, R11 + MOVL R11, 48(R12) // group.len = (group.len << 1) | 1 + MOVQ X0, (R12)(R13*8) // group.keys[i] = key + MOVL CX, 32(R12)(R13*4) // group.values[i] = value + MOVL CX, R14 + INCL CX + JMP next +probeNextGroup: + INCQ R10 + JMP probe + +// func multiProbe128SSE2(table []byte, tableCap, tableLen int, hashes []uintptr, keys sparse.Uint128Array, values []int32) int +TEXT ยทmultiProbe128SSE2(SB), NOSPLIT, $0-120 + MOVQ table_base+0(FP), AX + MOVQ tableCap+24(FP), BX + MOVQ tableLen+32(FP), CX + MOVQ hashes_base+40(FP), DX + MOVQ hashes_len+48(FP), DI + MOVQ keys_array_ptr+64(FP), R8 + MOVQ keys_array_off+80(FP), R15 + MOVQ values_base+88(FP), R9 + + MOVQ BX, R10 + SHLQ $4, R10 + LEAQ (AX)(R10*1), R10 + DECQ BX // modulo = tableCap - 1 + + XORQ SI, SI + JMP test +loop: + MOVQ (DX)(SI*8), R11 // hash + MOVOU (R8), X0 // key +probe: + MOVQ R11, R12 + ANDQ BX, R12 + + MOVL (R10)(R12*4), R14 + CMPL R14, $0 + JE insert + + SHLQ $4, R12 + MOVOU (AX)(R12*1), X1 + PCMPEQL X0, X1 + MOVMSKPS X1, R13 + CMPL R13, $0b1111 + JE next + + INCQ R11 + JMP probe +next: + DECL R14 + MOVL R14, (R9)(SI*4) + INCQ SI + ADDQ R15, R8 +test: + CMPQ SI, DI + JNE loop + MOVQ CX, ret+112(FP) + RET +insert: + INCL CX + MOVL CX, (R10)(R12*4) + MOVL CX, R14 + SHLQ $4, R12 + MOVOU X0, (AX)(R12*1) + JMP next diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_purego.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_purego.go new file mode 100644 index 0000000000..5afb40c74e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/hashprobe_purego.go @@ -0,0 +1,19 @@ +//go:build purego || !amd64 + +package hashprobe + +import ( + "github.com/parquet-go/parquet-go/sparse" +) + +func multiProbe32(table []table32Group, numKeys int, hashes []uintptr, keys sparse.Uint32Array, values []int32) int { + return multiProbe32Default(table, numKeys, hashes, keys, values) +} + +func multiProbe64(table []table64Group, numKeys int, hashes []uintptr, keys sparse.Uint64Array, values []int32) int { + return multiProbe64Default(table, numKeys, hashes, keys, values) +} + +func multiProbe128(table []byte, tableCap, tableLen int, hashes []uintptr, keys sparse.Uint128Array, values []int32) int { + return multiProbe128Default(table, tableCap, tableLen, hashes, keys, values) +} diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash.go new file mode 100644 index 0000000000..457bd8e114 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash.go @@ -0,0 +1,49 @@ +// Package wyhash implements a hashing algorithm derived from the Go runtime's +// internal hashing fallback, which uses a variation of the wyhash algorithm. +package wyhash + +import ( + "encoding/binary" + "math/bits" + + "github.com/parquet-go/parquet-go/sparse" +) + +const ( + m1 = 0xa0761d6478bd642f + m2 = 0xe7037ed1a0b428db + m3 = 0x8ebc6af09c88c6e3 + m4 = 0x589965cc75374cc3 + m5 = 0x1d8e4e27c47d124f +) + +func mix(a, b uint64) uint64 { + hi, lo := bits.Mul64(a, b) + return hi ^ lo +} + +func Hash32(value uint32, seed uintptr) uintptr { + return uintptr(mix(m5^4, mix(uint64(value)^m2, uint64(value)^uint64(seed)^m1))) +} + +func Hash64(value uint64, seed uintptr) uintptr { + return uintptr(mix(m5^8, mix(value^m2, value^uint64(seed)^m1))) +} + +func Hash128(value [16]byte, seed uintptr) uintptr { + a := binary.LittleEndian.Uint64(value[:8]) + b := binary.LittleEndian.Uint64(value[8:]) + return uintptr(mix(m5^16, mix(a^m2, b^uint64(seed)^m1))) +} + +func MultiHash32(hashes []uintptr, values []uint32, seed uintptr) { + MultiHashUint32Array(hashes, sparse.MakeUint32Array(values), seed) +} + +func MultiHash64(hashes []uintptr, values []uint64, seed uintptr) { + MultiHashUint64Array(hashes, sparse.MakeUint64Array(values), seed) +} + +func MultiHash128(hashes []uintptr, values [][16]byte, seed uintptr) { + MultiHashUint128Array(hashes, sparse.MakeUint128Array(values), seed) +} diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.go new file mode 100644 index 0000000000..55f499debc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.go @@ -0,0 +1,14 @@ +//go:build !purego + +package wyhash + +import "github.com/parquet-go/parquet-go/sparse" + +//go:noescape +func MultiHashUint32Array(hashes []uintptr, values sparse.Uint32Array, seed uintptr) + +//go:noescape +func MultiHashUint64Array(hashes []uintptr, values sparse.Uint64Array, seed uintptr) + +//go:noescape +func MultiHashUint128Array(hashes []uintptr, values sparse.Uint128Array, seed uintptr) diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.s b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.s new file mode 100644 index 0000000000..7b99879f02 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_amd64.s @@ -0,0 +1,118 @@ +//go:build !purego + +#include "textflag.h" + +#define m1 0xa0761d6478bd642f +#define m2 0xe7037ed1a0b428db +#define m3 0x8ebc6af09c88c6e3 +#define m4 0x589965cc75374cc3 +#define m5 0x1d8e4e27c47d124f + +// func MultiHashUint32Array(hashes []uintptr, values sparse.Uint32Array, seed uintptr) +TEXT ยทMultiHashUint32Array(SB), NOSPLIT, $0-56 + MOVQ hashes_base+0(FP), R12 + MOVQ values_array_ptr+24(FP), R13 + MOVQ values_array_len+32(FP), R14 + MOVQ values_array_off+40(FP), R15 + MOVQ seed+48(FP), R11 + + MOVQ $m1, R8 + MOVQ $m2, R9 + MOVQ $m5^4, R10 + XORQ R11, R8 + + XORQ SI, SI + JMP test +loop: + MOVL (R13), AX + MOVQ R8, BX + + XORQ AX, BX + XORQ R9, AX + + MULQ BX + XORQ DX, AX + + MULQ R10 + XORQ DX, AX + + MOVQ AX, (R12)(SI*8) + INCQ SI + ADDQ R15, R13 +test: + CMPQ SI, R14 + JNE loop + RET + +// func MultiHashUint64Array(hashes []uintptr, values sparse.Uint64Array, seed uintptr) +TEXT ยทMultiHashUint64Array(SB), NOSPLIT, $0-56 + MOVQ hashes_base+0(FP), R12 + MOVQ values_array_ptr+24(FP), R13 + MOVQ values_array_len+32(FP), R14 + MOVQ values_array_off+40(FP), R15 + MOVQ seed+48(FP), R11 + + MOVQ $m1, R8 + MOVQ $m2, R9 + MOVQ $m5^8, R10 + XORQ R11, R8 + + XORQ SI, SI + JMP test +loop: + MOVQ (R13), AX + MOVQ R8, BX + + XORQ AX, BX + XORQ R9, AX + + MULQ BX + XORQ DX, AX + + MULQ R10 + XORQ DX, AX + + MOVQ AX, (R12)(SI*8) + INCQ SI + ADDQ R15, R13 +test: + CMPQ SI, R14 + JNE loop + RET + +// func MultiHashUint128Array(hashes []uintptr, values sparse.Uint128Array, seed uintptr) +TEXT ยทMultiHashUint128Array(SB), NOSPLIT, $0-56 + MOVQ hashes_base+0(FP), R12 + MOVQ values_array_ptr+24(FP), R13 + MOVQ values_array_len+32(FP), R14 + MOVQ values_array_off+40(FP), R15 + MOVQ seed+48(FP), R11 + + MOVQ $m1, R8 + MOVQ $m2, R9 + MOVQ $m5^16, R10 + XORQ R11, R8 + + XORQ SI, SI + JMP test +loop: + MOVQ 0(R13), AX + MOVQ 8(R13), DX + MOVQ R8, BX + + XORQ DX, BX + XORQ R9, AX + + MULQ BX + XORQ DX, AX + + MULQ R10 + XORQ DX, AX + + MOVQ AX, (R12)(SI*8) + INCQ SI + ADDQ R15, R13 +test: + CMPQ SI, R14 + JNE loop + RET diff --git a/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_purego.go b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_purego.go new file mode 100644 index 0000000000..b5760f7ec5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/hashprobe/wyhash/wyhash_purego.go @@ -0,0 +1,23 @@ +//go:build purego || !amd64 + +package wyhash + +import "github.com/parquet-go/parquet-go/sparse" + +func MultiHashUint32Array(hashes []uintptr, values sparse.Uint32Array, seed uintptr) { + for i := range hashes { + hashes[i] = Hash32(values.Index(i), seed) + } +} + +func MultiHashUint64Array(hashes []uintptr, values sparse.Uint64Array, seed uintptr) { + for i := range hashes { + hashes[i] = Hash64(values.Index(i), seed) + } +} + +func MultiHashUint128Array(hashes []uintptr, values sparse.Uint128Array, seed uintptr) { + for i := range hashes { + hashes[i] = Hash128(values.Index(i), seed) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/bitpack.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/bitpack.go new file mode 100644 index 0000000000..e6a11884d6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/bitpack.go @@ -0,0 +1,8 @@ +// Package bitpack implements efficient bit packing and unpacking routines for +// integers of various bit widths. +package bitpack + +// ByteCount returns the number of bytes needed to hold the given bit count. +func ByteCount(bitCount uint) int { + return int((bitCount + 7) / 8) +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/masks_int32_amd64.s b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/masks_int32_amd64.s new file mode 100644 index 0000000000..6ffe4e2c53 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/masks_int32_amd64.s @@ -0,0 +1,1288 @@ +//go:build !purego + +#include "textflag.h" + +// ----------------------------------------------------------------------------- +// Shuffle masks used to broadcast bytes of bit-packed valued into vector +// registers at positions where they can then be shifted into the right +// locations. +// ----------------------------------------------------------------------------- + +// Shuffle masks for unpacking values from bit widths 1 to 16. +// +// The masks are grouped in 32 bytes chunks containing 2 masks of 16 bytes, with +// the following layout: +// +// - The first mask is used to shuffle values from the 16 bytes of input into +// the lower 16 bytes of output. These values are then shifted RIGHT to be +// aligned on the begining of each 32 bit word. +// +// - The second mask selects values from the 16 bytes of input into the upper +// 16 bytes of output. These values are then shifted RIGHT to be aligned on +// the beginning of each 32 bit word. +// +// The bit width is intended to be used as an index into this array, using this +// formula to convert from the index to a byte offset: +// +// offset = 32 * (bitWidth - 1) +// +GLOBL ยทshuffleInt32x1to16bits(SB), RODATA|NOPTR, $512 + +// 1 bit => 32 bits +// ----------------- +// 0: [a,b,c,d,e,f,g,h] +// ... +DATA ยทshuffleInt32x1to16bits+0+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+0+4(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+0+8(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+0+12(SB)/4, $0x80808000 + +DATA ยทshuffleInt32x1to16bits+0+16(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+0+20(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+0+24(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+0+28(SB)/4, $0x80808000 + +// 2 bits => 32 bits +// ----------------- +// 0: [a,a,b,b,c,c,d,d] +// 1: [e,e,f,f,g,g,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+32+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+32+4(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+32+8(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+32+12(SB)/4, $0x80808000 + +DATA ยทshuffleInt32x1to16bits+32+16(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+32+20(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+32+24(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+32+28(SB)/4, $0x80808001 + +// 3 bits => 32 bits +// ----------------- +// 0: [a,a,a,b,b,b,c,c] +// 1: [c,d,d,d,e,e,e,f] +// 2: [f,f,g,g,g,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+64+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+64+4(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+64+8(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+64+12(SB)/4, $0x80808001 + +DATA ยทshuffleInt32x1to16bits+64+16(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+64+20(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+64+24(SB)/4, $0x80808002 +DATA ยทshuffleInt32x1to16bits+64+28(SB)/4, $0x80808002 + +// 4 bits => 32 bits +// ----------------- +// 0: [a,a,a,a,b,b,b,b] +// 1: [c,c,c,c,d,d,d,d] +// 2: [e,e,e,e,f,f,f,f] +// 3: [g,g,g,g,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+96+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+96+4(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+96+8(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+96+12(SB)/4, $0x80808001 + +DATA ยทshuffleInt32x1to16bits+96+16(SB)/4, $0x80808002 +DATA ยทshuffleInt32x1to16bits+96+20(SB)/4, $0x80808002 +DATA ยทshuffleInt32x1to16bits+96+24(SB)/4, $0x80808003 +DATA ยทshuffleInt32x1to16bits+96+28(SB)/4, $0x80808003 + +// 5 bits => 32 bits +// ----------------- +// 0: [a,a,a,a,a,b,b,b] +// 1: [b,b,c,c,c,c,c,d] +// 2: [d,d,d,d,e,e,e,e] +// 3: [e,f,f,f,f,f,g,g] +// 4: [g,g,g,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+128+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+128+4(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+128+8(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+128+12(SB)/4, $0x80800201 + +DATA ยทshuffleInt32x1to16bits+128+16(SB)/4, $0x80800302 +DATA ยทshuffleInt32x1to16bits+128+20(SB)/4, $0x80808003 +DATA ยทshuffleInt32x1to16bits+128+24(SB)/4, $0x80800403 +DATA ยทshuffleInt32x1to16bits+128+28(SB)/4, $0x80808004 + +// 6 bits => 32 bits +// ----------------- +// 0: [a,a,a,a,a,a,b,b] +// 1: [b,b,b,b,c,c,c,c] +// 2: [c,c,d,d,d,d,d,d] +// 3: [e,e,e,e,e,e,f,f] +// 4: [f,f,f,f,g,g,g,g] +// 5: [g,g,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+160+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+160+4(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+160+8(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+160+12(SB)/4, $0x80808002 + +DATA ยทshuffleInt32x1to16bits+160+16(SB)/4, $0x80808003 +DATA ยทshuffleInt32x1to16bits+160+20(SB)/4, $0x80800403 +DATA ยทshuffleInt32x1to16bits+160+24(SB)/4, $0x80800504 +DATA ยทshuffleInt32x1to16bits+160+28(SB)/4, $0x80808005 + +// 7 bits => 32 bits +// ----------------- +// 0: [a,a,a,a,a,a,a,b] +// 1: [b,b,b,b,b,b,c,c] +// 2: [c,c,c,c,c,d,d,d] +// 3: [d,d,d,d,e,e,e,e] +// 4: [e,e,e,f,f,f,f,f] +// 5: [f,f,g,g,g,g,g,g] +// 6: [g,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+192+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+192+4(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+192+8(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+192+12(SB)/4, $0x80800302 + +DATA ยทshuffleInt32x1to16bits+192+16(SB)/4, $0x80800403 +DATA ยทshuffleInt32x1to16bits+192+20(SB)/4, $0x80800504 +DATA ยทshuffleInt32x1to16bits+192+24(SB)/4, $0x80800605 +DATA ยทshuffleInt32x1to16bits+192+28(SB)/4, $0x80808006 + +// 8 bits => 32 bits +// ----------------- +// 0: [a,a,a,a,a,a,a,a] +// 1: [b,b,b,b,b,b,b,b] +// 2: [c,c,c,c,c,c,c,c] +// 3: [d,d,d,d,d,d,d,d] +// 4: [e,e,e,e,e,e,e,e] +// 5: [f,f,f,f,f,f,f,f] +// 6: [g,g,g,g,g,g,g,g] +// 7: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+224+0(SB)/4, $0x80808000 +DATA ยทshuffleInt32x1to16bits+224+4(SB)/4, $0x80808001 +DATA ยทshuffleInt32x1to16bits+224+8(SB)/4, $0x80808002 +DATA ยทshuffleInt32x1to16bits+224+12(SB)/4, $0x80808003 + +DATA ยทshuffleInt32x1to16bits+224+16(SB)/4, $0x80808004 +DATA ยทshuffleInt32x1to16bits+224+20(SB)/4, $0x80808005 +DATA ยทshuffleInt32x1to16bits+224+24(SB)/4, $0x80808006 +DATA ยทshuffleInt32x1to16bits+224+28(SB)/4, $0x80808007 + +// 9 bits => 32 bits +// ----------------- +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,b,b,b,b,b,b,b] +// 2: [b,b,c,c,c,c,c,c] +// 3: [c,c,c,d,d,d,d,d] +// 4: [d,d,d,d,e,e,e,e] +// 5: [e,e,e,e,e,f,f,f] +// 6: [f,f,f,f,f,f,g,g] +// 7: [g,g,g,g,g,g,g,h] +// 8: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+256+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+256+4(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+256+8(SB)/4, $0x80800302 +DATA ยทshuffleInt32x1to16bits+256+12(SB)/4, $0x80800403 + +DATA ยทshuffleInt32x1to16bits+256+16(SB)/4, $0x80800504 +DATA ยทshuffleInt32x1to16bits+256+20(SB)/4, $0x80800605 +DATA ยทshuffleInt32x1to16bits+256+24(SB)/4, $0x80800706 +DATA ยทshuffleInt32x1to16bits+256+28(SB)/4, $0x80800807 + +// 10 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,b,b,b,b,b,b] +// 2: [b,b,b,b,c,c,c,c] +// 3: [c,c,c,c,c,c,d,d] +// 4: [d,d,d,d,d,d,d,d] +// 5: [e,e,e,e,e,e,e,e] +// 6: [e,e,f,f,f,f,f,f] +// 7: [f,f,f,f,g,g,g,g] +// 8: [g,g,g,g,g,g,h,h] +// 9: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+288+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+288+4(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+288+8(SB)/4, $0x80800302 +DATA ยทshuffleInt32x1to16bits+288+12(SB)/4, $0x80800403 + +DATA ยทshuffleInt32x1to16bits+288+16(SB)/4, $0x80800605 +DATA ยทshuffleInt32x1to16bits+288+20(SB)/4, $0x80800706 +DATA ยทshuffleInt32x1to16bits+288+24(SB)/4, $0x80800807 +DATA ยทshuffleInt32x1to16bits+288+28(SB)/4, $0x80800908 + +// 11 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,b,b,b,b,b] +// 2: [b,b,b,b,b,b,c,c] +// 3: [c,c,c,c,c,c,c,c] +// 4: [c,d,d,d,d,d,d,d] +// 5: [d,d,d,d,e,e,e,e] +// 6: [e,e,e,e,e,e,e,f] +// 7: [f,f,f,f,f,f,f,f] +// 8: [f,f,g,g,g,g,g,g] +// 9: [g,g,g,g,g,h,h,h] +// A: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+320+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+320+4(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+320+8(SB)/4, $0x80040302 +DATA ยทshuffleInt32x1to16bits+320+12(SB)/4, $0x80800504 + +DATA ยทshuffleInt32x1to16bits+320+16(SB)/4, $0x80800605 +DATA ยทshuffleInt32x1to16bits+320+20(SB)/4, $0x80080706 +DATA ยทshuffleInt32x1to16bits+320+24(SB)/4, $0x80800908 +DATA ยทshuffleInt32x1to16bits+320+28(SB)/4, $0x80800A09 + +// 12 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,b,b,b,b] +// 2: [b,b,b,b,b,b,b,b] +// 3: [c,c,c,c,c,c,c,c] +// 4: [c,c,c,c,d,d,d,d] +// 5: [d,d,d,d,d,d,d,d] +// 6: [e,e,e,e,e,e,e,e] +// 7: [e,e,e,e,f,f,f,f] +// 8: [f,f,f,f,f,f,f,f] +// 9: [g,g,g,g,g,g,g,g] +// A: [g,g,g,g,h,h,h,h] +// B: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+352+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+352+4(SB)/4, $0x80800201 +DATA ยทshuffleInt32x1to16bits+352+8(SB)/4, $0x80080403 +DATA ยทshuffleInt32x1to16bits+352+12(SB)/4, $0x80800504 + +DATA ยทshuffleInt32x1to16bits+352+16(SB)/4, $0x80800706 +DATA ยทshuffleInt32x1to16bits+352+20(SB)/4, $0x80800807 +DATA ยทshuffleInt32x1to16bits+352+24(SB)/4, $0x80800A09 +DATA ยทshuffleInt32x1to16bits+352+28(SB)/4, $0x80800B0A + +// 13 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,b,b,b] +// 2: [b,b,b,b,b,b,b,b] +// 3: [b,b,c,c,c,c,c,c] +// 4: [c,c,c,c,c,c,c,d] +// 5: [d,d,d,d,d,d,d,d] +// 6: [d,d,d,d,e,e,e,e] +// 7: [e,e,e,e,e,e,e,e] +// 8: [e,f,f,f,f,f,f,f] +// 9: [f,f,f,f,f,f,g,g] +// A: [g,g,g,g,g,g,g,g] +// B: [g,g,g,h,h,h,h,h] +// C: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+384+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+384+4(SB)/4, $0x80030201 +DATA ยทshuffleInt32x1to16bits+384+8(SB)/4, $0x80800403 +DATA ยทshuffleInt32x1to16bits+384+12(SB)/4, $0x80060504 + +DATA ยทshuffleInt32x1to16bits+384+16(SB)/4, $0x80080706 +DATA ยทshuffleInt32x1to16bits+384+20(SB)/4, $0x80800908 +DATA ยทshuffleInt32x1to16bits+384+24(SB)/4, $0x800B0A09 +DATA ยทshuffleInt32x1to16bits+384+28(SB)/4, $0x80800C0B + +// 14 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,b,b] +// 2: [b,b,b,b,b,b,b,b] +// 3: [b,b,b,b,c,c,c,c] +// 4: [c,c,c,c,c,c,c,c] +// 5: [c,c,d,d,d,d,d,d] +// 6: [d,d,d,d,d,d,d,d] +// 7: [e,e,e,e,e,e,e,e] +// 8: [e,e,e,e,e,e,f,f] +// 9: [f,f,f,f,f,f,f,f] +// A: [f,f,f,f,g,g,g,g] +// B: [g,g,g,g,g,g,g,g] +// C: [g,g,h,h,h,h,h,h] +// D: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+416+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+416+4(SB)/4, $0x80030201 +DATA ยทshuffleInt32x1to16bits+416+8(SB)/4, $0x80050403 +DATA ยทshuffleInt32x1to16bits+416+12(SB)/4, $0x80800605 + +DATA ยทshuffleInt32x1to16bits+416+16(SB)/4, $0x80080807 +DATA ยทshuffleInt32x1to16bits+416+20(SB)/4, $0x800A0908 +DATA ยทshuffleInt32x1to16bits+416+24(SB)/4, $0x800C0B0A +DATA ยทshuffleInt32x1to16bits+416+28(SB)/4, $0x80800D0C + +// 15 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,b] +// 2: [b,b,b,b,b,b,b,b] +// 3: [b,b,b,b,b,b,c,c] +// 4: [c,c,c,c,c,c,c,c] +// 5: [c,c,c,c,c,d,d,d] +// 6: [d,d,d,d,d,d,d,d] +// 7: [d,d,d,d,e,e,e,e] +// 8: [e,e,e,e,e,e,e,e] +// 9: [e,e,e,f,f,f,f,f] +// A: [f,f,f,f,f,f,f,f] +// B: [f,f,g,g,g,g,g,g] +// C: [g,g,g,g,g,g,g,g] +// D: [g,h,h,h,h,h,h,h] +// E: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+448+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+448+4(SB)/4, $0x80030201 +DATA ยทshuffleInt32x1to16bits+448+8(SB)/4, $0x80050403 +DATA ยทshuffleInt32x1to16bits+448+12(SB)/4, $0x80070605 + +DATA ยทshuffleInt32x1to16bits+448+16(SB)/4, $0x80090807 +DATA ยทshuffleInt32x1to16bits+448+20(SB)/4, $0x800B0A09 +DATA ยทshuffleInt32x1to16bits+448+24(SB)/4, $0x800D0C0B +DATA ยทshuffleInt32x1to16bits+448+28(SB)/4, $0x80800E0D + +// 16 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [b,b,b,b,b,b,b,b] +// 3: [b,b,b,b,b,b,c,b] +// 4: [c,c,c,c,c,c,c,c] +// 5: [c,c,c,c,c,c,c,c] +// 6: [d,d,d,d,d,d,d,d] +// 7: [d,d,d,d,d,d,d,d] +// 8: [e,e,e,e,e,e,e,e] +// 9: [e,e,e,e,e,e,e,e] +// A: [f,f,f,f,f,f,f,f] +// B: [f,f,f,f,f,f,f,f] +// C: [g,g,g,g,g,g,g,g] +// D: [g,g,g,g,g,g,g,g] +// E: [h,h,h,h,h,h,h,h] +// F: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x1to16bits+480+0(SB)/4, $0x80800100 +DATA ยทshuffleInt32x1to16bits+480+4(SB)/4, $0x80800302 +DATA ยทshuffleInt32x1to16bits+480+8(SB)/4, $0x80800504 +DATA ยทshuffleInt32x1to16bits+480+12(SB)/4, $0x80800706 + +DATA ยทshuffleInt32x1to16bits+480+16(SB)/4, $0x80800908 +DATA ยทshuffleInt32x1to16bits+480+20(SB)/4, $0x80800B0A +DATA ยทshuffleInt32x1to16bits+480+24(SB)/4, $0x80800D0C +DATA ยทshuffleInt32x1to16bits+480+28(SB)/4, $0x80800F0E + +// Shuffle masks for unpacking values from bit widths 17 to 26. +// +// The masks are grouped in 48 bytes chunks containing 3 masks of 16 bytes, with +// the following layout: +// +// - The first mask is used to shuffle values from the first 16 bytes of input +// into the lower 16 bytes of output. These values are then shifted RIGHT to +// be aligned on the begining of each 32 bit word. +// +// - The second mask selects values from the first 16 bytes of input into the +// upper 16 bytes of output. These values are then shifted RIGHT to be aligned +// on the beginning of each 32 bit word. +// +// - The third mask selects values from the second 16 bytes of input into the +// upper 16 bytes of output. These values are then shifted RIGHT to be aligned +// on the beginning of each 32 bit word. +// +// The bit width is intended to be used as an index into this array, using this +// formula to convert from the index to a byte offset: +// +// offset = 48 * (bitWidth - 17) +// +GLOBL ยทshuffleInt32x17to26bits(SB), RODATA|NOPTR, $480 + +// 17 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,b,b,b,b,b,b,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,c,c,c,c,c,c] +// 5: [c,c,c,c,c,c,c,c] +// 6: [c,c,c,d,d,d,d,d] +// 7: [d,d,d,d,d,d,d,d] +// 8: [d,d,d,d,e,e,e,e] +// 9: [e,e,e,e,e,e,e,e] +// A: [e,e,e,e,e,f,f,f] +// B: [f,f,f,f,f,f,f,f] +// C: [f,f,f,f,f,f,g,g] +// D: [g,g,g,g,g,g,g,g] +// E: [g,g,g,g,g,g,g,h] +// F: [h,h,h,h,h,h,h,h] +// --- +// 0: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+0+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+0+4(SB)/4, $0x80040302 +DATA ยทshuffleInt32x17to26bits+0+8(SB)/4, $0x80060504 +DATA ยทshuffleInt32x17to26bits+0+12(SB)/4, $0x80080706 + +DATA ยทshuffleInt32x17to26bits+0+16(SB)/4, $0x800A0908 +DATA ยทshuffleInt32x17to26bits+0+20(SB)/4, $0x800C0B0A +DATA ยทshuffleInt32x17to26bits+0+24(SB)/4, $0x800E0D0C +DATA ยทshuffleInt32x17to26bits+0+28(SB)/4, $0x80800F0E + +DATA ยทshuffleInt32x17to26bits+0+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+0+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+0+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+0+44(SB)/4, $0x80008080 + +// 18 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,b,b,b,b,b,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,c,c,c,c] +// 5: [c,c,c,c,c,c,c,c] +// 6: [c,c,c,c,c,c,d,d] +// 7: [d,d,d,d,d,d,d,d] +// 8: [d,d,d,d,d,d,d,d] +// 9: [e,e,e,e,e,e,e,e] +// A: [e,e,e,e,e,e,e,e] +// B: [e,e,f,f,f,f,f,f] +// C: [f,f,f,f,f,f,f,f] +// D: [f,f,f,f,g,g,g,g] +// E: [g,g,g,g,g,g,g,g] +// F: [g,g,g,g,g,g,h,h] +// --- +// 0: [h,h,h,h,h,h,h,h] +// 1: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+48+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+48+4(SB)/4, $0x80040302 +DATA ยทshuffleInt32x17to26bits+48+8(SB)/4, $0x80060504 +DATA ยทshuffleInt32x17to26bits+48+12(SB)/4, $0x80080706 + +DATA ยทshuffleInt32x17to26bits+48+16(SB)/4, $0x800B0A09 +DATA ยทshuffleInt32x17to26bits+48+20(SB)/4, $0x800D0C0B +DATA ยทshuffleInt32x17to26bits+48+24(SB)/4, $0x800F0E0D +DATA ยทshuffleInt32x17to26bits+48+28(SB)/4, $0x8080800F + +DATA ยทshuffleInt32x17to26bits+48+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+48+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+48+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+48+44(SB)/4, $0x80010080 + +// 19 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,b,b,b,b,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,c,c] +// 5: [c,c,c,c,c,c,c,c] +// 6: [c,c,c,c,c,c,c,c] +// 7: [c,d,d,d,d,d,d,d] +// 8: [d,d,d,d,d,d,d,d] +// 9: [d,d,d,d,e,e,e,e] +// A: [e,e,e,e,e,e,e,e] +// B: [e,e,e,e,e,e,e,f] +// C: [f,f,f,f,f,f,f,f] +// D: [f,f,f,f,f,f,f,f] +// E: [f,f,g,g,g,g,g,g] +// F: [g,g,g,g,g,g,g,g] +// --- +// 0: [g,g,g,g,g,h,h,h] +// 1: [h,h,h,h,h,h,h,h] +// 2: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+96+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+96+4(SB)/4, $0x80040302 +DATA ยทshuffleInt32x17to26bits+96+8(SB)/4, $0x07060504 +DATA ยทshuffleInt32x17to26bits+96+12(SB)/4, $0x80090807 + +DATA ยทshuffleInt32x17to26bits+96+16(SB)/4, $0x800B0A09 +DATA ยทshuffleInt32x17to26bits+96+20(SB)/4, $0x0E0D0C0B +DATA ยทshuffleInt32x17to26bits+96+24(SB)/4, $0x80800F0E +DATA ยทshuffleInt32x17to26bits+96+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+96+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+96+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+96+40(SB)/4, $0x80008080 +DATA ยทshuffleInt32x17to26bits+96+44(SB)/4, $0x80020100 + +// 20 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,b,b,b,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [c,c,c,c,c,c,c,c] +// 6: [c,c,c,c,c,c,c,c] +// 7: [c,c,c,c,d,d,d,d] +// 8: [d,d,d,d,d,d,d,d] +// 9: [d,d,d,d,d,d,d,d] +// A: [e,e,e,e,e,e,e,e] +// B: [e,e,e,e,e,e,e,e] +// C: [e,e,e,e,f,f,f,f] +// D: [f,f,f,f,f,f,f,f] +// E: [f,f,f,f,f,f,f,f] +// F: [g,g,g,g,g,g,g,g] +// --- +// 0: [g,g,g,g,g,g,g,g] +// 1: [g,g,g,g,h,h,h,h] +// 2: [h,h,h,h,h,h,h,h] +// 3: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+144+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+144+4(SB)/4, $0x80040302 +DATA ยทshuffleInt32x17to26bits+144+8(SB)/4, $0x80070605 +DATA ยทshuffleInt32x17to26bits+144+12(SB)/4, $0x80090807 + +DATA ยทshuffleInt32x17to26bits+144+16(SB)/4, $0x800C0B0A +DATA ยทshuffleInt32x17to26bits+144+20(SB)/4, $0x800E0D0C +DATA ยทshuffleInt32x17to26bits+144+24(SB)/4, $0x8080800F +DATA ยทshuffleInt32x17to26bits+144+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+144+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+144+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+144+40(SB)/4, $0x80010080 +DATA ยทshuffleInt32x17to26bits+144+44(SB)/4, $0x80030201 + +// 21 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,b,b,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,c,c,c,c,c,c] +// 6: [c,c,c,c,c,c,c,c] +// 7: [c,c,c,c,c,c,c,d] +// 8: [d,d,d,d,d,d,d,d] +// 9: [d,d,d,d,d,d,d,d] +// A: [d,d,d,d,e,e,e,e] +// B: [e,e,e,e,e,e,e,e] +// C: [e,e,e,e,e,e,e,e] +// D: [e,f,f,f,f,f,f,f] +// E: [f,f,f,f,f,f,f,f] +// F: [f,f,f,f,f,f,g,g] +// --- +// 0: [g,g,g,g,g,g,g,g] +// 1: [g,g,g,g,g,g,g,g] +// 2: [g,g,g,h,h,h,h,h] +// 3: [h,h,h,h,h,h,h,h] +// 4: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+192+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+192+4(SB)/4, $0x05040302 +DATA ยทshuffleInt32x17to26bits+192+8(SB)/4, $0x80070605 +DATA ยทshuffleInt32x17to26bits+192+12(SB)/4, $0x0A090807 + +DATA ยทshuffleInt32x17to26bits+192+16(SB)/4, $0x0D0C0B0A +DATA ยทshuffleInt32x17to26bits+192+20(SB)/4, $0x800F0E0D +DATA ยทshuffleInt32x17to26bits+192+24(SB)/4, $0x8080800F +DATA ยทshuffleInt32x17to26bits+192+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+192+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+192+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+192+40(SB)/4, $0x02010080 +DATA ยทshuffleInt32x17to26bits+192+44(SB)/4, $0x80040302 + +// 22 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,b,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,c,c,c,c] +// 6: [c,c,c,c,c,c,c,c] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,d,d,d,d,d,d] +// 9: [d,d,d,d,d,d,d,d] +// A: [d,d,d,d,d,d,d,d] +// B: [e,e,e,e,e,e,e,e] +// C: [e,e,e,e,e,e,e,e] +// D: [e,e,e,e,e,e,f,f] +// E: [f,f,f,f,f,f,f,f] +// F: [f,f,f,f,f,f,f,f] +// --- +// 0: [f,f,f,f,g,g,g,g] +// 1: [g,g,g,g,g,g,g,g] +// 2: [g,g,g,g,g,g,g,g] +// 3: [g,g,h,h,h,h,h,h] +// 4: [h,h,h,h,h,h,h,h] +// 5: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+240+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+240+4(SB)/4, $0x05040302 +DATA ยทshuffleInt32x17to26bits+240+8(SB)/4, $0x08070605 +DATA ยทshuffleInt32x17to26bits+240+12(SB)/4, $0x800A0908 + +DATA ยทshuffleInt32x17to26bits+240+16(SB)/4, $0x800D0C0B +DATA ยทshuffleInt32x17to26bits+240+20(SB)/4, $0x800F0E0D +DATA ยทshuffleInt32x17to26bits+240+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+240+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+240+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+240+36(SB)/4, $0x00808080 +DATA ยทshuffleInt32x17to26bits+240+40(SB)/4, $0x03020100 +DATA ยทshuffleInt32x17to26bits+240+44(SB)/4, $0x80050403 + +// 23 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,b] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,c,c] +// 6: [c,c,c,c,c,c,c,c] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,c,c,c,d,d,d] +// 9: [d,d,d,d,d,d,d,d] +// A: [d,d,d,d,d,d,d,d] +// B: [d,d,d,d,e,e,e,e] +// C: [e,e,e,e,e,e,e,e] +// D: [e,e,e,e,e,e,e,e] +// E: [e,e,e,f,f,f,f,f] +// F: [f,f,f,f,f,f,f,f] +// --- +// 0: [f,f,f,f,f,f,f,f] +// 1: [f,f,g,g,g,g,g,g] +// 2: [g,g,g,g,g,g,g,g] +// 3: [g,g,g,g,g,g,g,g] +// 4: [g,h,h,h,h,h,h,h] +// 5: [h,h,h,h,h,h,h,h] +// 6: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+288+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+288+4(SB)/4, $0x05040302 +DATA ยทshuffleInt32x17to26bits+288+8(SB)/4, $0x08070605 +DATA ยทshuffleInt32x17to26bits+288+12(SB)/4, $0x0B0A0908 + +DATA ยทshuffleInt32x17to26bits+288+16(SB)/4, $0x0E0D0C0B +DATA ยทshuffleInt32x17to26bits+288+20(SB)/4, $0x80800F0E +DATA ยทshuffleInt32x17to26bits+288+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+288+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+288+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+288+36(SB)/4, $0x01008080 +DATA ยทshuffleInt32x17to26bits+288+40(SB)/4, $0x04030201 +DATA ยทshuffleInt32x17to26bits+288+44(SB)/4, $0x80060504 + +// 24 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [b,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [c,c,c,c,c,c,c,c] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [d,d,d,d,d,d,d,d] +// A: [d,d,d,d,d,d,d,d] +// B: [d,d,d,d,d,d,d,d] +// C: [e,e,e,e,e,e,e,e] +// D: [e,e,e,e,e,e,e,e] +// E: [e,e,e,e,e,e,e,e] +// F: [f,f,f,f,f,f,f,f] +// --- +// 0: [f,f,f,f,f,f,f,f] +// 1: [f,f,f,f,f,f,f,f] +// 2: [g,g,g,g,g,g,g,g] +// 3: [g,g,g,g,g,g,g,g] +// 4: [g,g,g,g,g,g,g,g] +// 5: [h,h,h,h,h,h,h,h] +// 6: [h,h,h,h,h,h,h,h] +// 7: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+336+0(SB)/4, $0x80020100 +DATA ยทshuffleInt32x17to26bits+336+4(SB)/4, $0x80050403 +DATA ยทshuffleInt32x17to26bits+336+8(SB)/4, $0x80080706 +DATA ยทshuffleInt32x17to26bits+336+12(SB)/4, $0x800B0A09 + +DATA ยทshuffleInt32x17to26bits+336+16(SB)/4, $0x800E0D0C +DATA ยทshuffleInt32x17to26bits+336+20(SB)/4, $0x8080800F +DATA ยทshuffleInt32x17to26bits+336+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+336+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+336+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+336+36(SB)/4, $0x80010080 +DATA ยทshuffleInt32x17to26bits+336+40(SB)/4, $0x80040302 +DATA ยทshuffleInt32x17to26bits+336+44(SB)/4, $0x80070605 + +// 25 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,b,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,c,c,c,c,c,c] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,d,d,d,d,d] +// A: [d,d,d,d,d,d,d,d] +// B: [d,d,d,d,d,d,d,d] +// C: [d,d,d,d,e,e,e,e] +// D: [e,e,e,e,e,e,e,e] +// E: [e,e,e,e,e,e,e,e] +// F: [e,e,e,e,e,f,f,f] +// --- +// 0: [f,f,f,f,f,f,f,f] +// 1: [f,f,f,f,f,f,f,f] +// 2: [f,f,f,f,f,f,g,g] +// 3: [g,g,g,g,g,g,g,g] +// 4: [g,g,g,g,g,g,g,g] +// 5: [g,g,g,g,g,g,g,h] +// 6: [h,h,h,h,h,h,h,h] +// 7: [h,h,h,h,h,h,h,h] +// 8: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+384+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x17to26bits+384+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x17to26bits+384+8(SB)/4, $0x09080706 +DATA ยทshuffleInt32x17to26bits+384+12(SB)/4, $0x0C0B0A09 + +DATA ยทshuffleInt32x17to26bits+384+16(SB)/4, $0x0F0E0D0C +DATA ยทshuffleInt32x17to26bits+384+20(SB)/4, $0x8080800F +DATA ยทshuffleInt32x17to26bits+384+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+384+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+384+32(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+384+36(SB)/4, $0x02010080 +DATA ยทshuffleInt32x17to26bits+384+40(SB)/4, $0x05040302 +DATA ยทshuffleInt32x17to26bits+384+44(SB)/4, $0x08070605 + +// 26 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,a,b,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,b,b,c,c,c,c] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,c,c,c,d,d] +// A: [d,d,d,d,d,d,d,d] +// B: [d,d,d,d,d,d,d,d] +// C: [d,d,d,d,d,d,d,d] +// D: [e,e,e,e,e,e,e,e] +// E: [e,e,e,e,e,e,e,e] +// F: [e,e,e,e,e,e,e,e] +// --- +// 0: [e,e,f,f,f,f,f,f] +// 1: [f,f,f,f,f,f,f,f] +// 2: [f,f,f,f,f,f,f,f] +// 3: [f,f,f,f,g,g,g,g] +// 4: [g,g,g,g,g,g,g,g] +// 5: [g,g,g,g,g,g,g,g] +// 6: [g,g,g,g,g,g,h,h] +// 7: [h,h,h,h,h,h,h,h] +// 8: [h,h,h,h,h,h,h,h] +// 9: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x17to26bits+432+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x17to26bits+432+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x17to26bits+432+8(SB)/4, $0x09080706 +DATA ยทshuffleInt32x17to26bits+432+12(SB)/4, $0x0C0B0A09 + +DATA ยทshuffleInt32x17to26bits+432+16(SB)/4, $0x800F0E0D +DATA ยทshuffleInt32x17to26bits+432+20(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+432+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x17to26bits+432+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x17to26bits+432+32(SB)/4, $0x00808080 +DATA ยทshuffleInt32x17to26bits+432+36(SB)/4, $0x03020100 +DATA ยทshuffleInt32x17to26bits+432+40(SB)/4, $0x06050403 +DATA ยทshuffleInt32x17to26bits+432+44(SB)/4, $0x09080706 + +// Shuffle masks for unpacking values from bit widths 27 to 31. +// +// The masks are grouped in 80 bytes chunks containing 5 masks of 16 bytes, with +// the following layout: +// +// - The first mask is used to shuffle values from the first 16 bytes of input +// into the lower 16 bytes of output. These values are then shifted RIGHT to +// be aligned on the begining of each 32 bit word. +// +// - The second mask is used to shuffle upper bits of bit-packed values of the +// first 16 bytes of input that spanned across 5 bytes. These extra bits cannot +// be selected by the first mask (which can select at most 4 bytes per word). +// The extra bits are then shifted LEFT to be positioned at the end of the +// words, after the bits extracted by the first mask. +// +// - The third mask selects values from the first 16 bytes of input into the +// upper 16 bytes of output. These values are then shifted RIGHT to be aligned +// on the beginning of each 32 bit word. +// +// - The fourth mask selects values from the second 16 bytes of input into the +// upper 16 bytes of output. These values are then shifted RIGHT to be aligned +// on the beginning of each 32 bit word. +// +// - The fifth mask is used to shuffle upper bits of bit-packed values values of +// second 16 bytes of input that spanned across 5 bytes. These values are then +// shifted LEFT to be aligned on the beginning of each 32 bit word. +// +// The bit width is intended to be used as an index into this array, using this +// formula to convert from the index to a byte offset: +// +// offset = 80 * (bitWidth - 27) +// +GLOBL ยทshuffleInt32x27to31bits(SB), RODATA|NOPTR, $400 + +// 27 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,a,a,b,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,b,b,b,b,c,c] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,c,c,c,c,c] +// A: [c,d,d,d,d,d,d,d] +// B: [d,d,d,d,d,d,d,d] +// C: [d,d,d,d,d,d,d,d] +// D: [d,d,d,d,e,e,e,e] +// E: [e,e,e,e,e,e,e,e] +// F: [e,e,e,e,e,e,e,e] +// --- +// 0: [e,e,e,e,e,e,e,f] +// 1: [f,f,f,f,f,f,f,f] +// 2: [f,f,f,f,f,f,f,f] +// 3: [f,f,f,f,f,f,f,f] +// 4: [f,f,g,g,g,g,g,g] +// 5: [g,g,g,g,g,g,g,g] +// 6: [g,g,g,g,g,g,g,g] +// 7: [g,g,g,g,g,h,h,h] +// 8: [h,h,h,h,h,h,h,h] +// 9: [h,h,h,h,h,h,h,h] +// A: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x27to31bits+0+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x27to31bits+0+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x27to31bits+0+8(SB)/4, $0x09080706 +DATA ยทshuffleInt32x27to31bits+0+12(SB)/4, $0x0D0C0B0A + +DATA ยทshuffleInt32x27to31bits+0+16(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+0+20(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+0+24(SB)/4, $0x0A808080 +DATA ยทshuffleInt32x27to31bits+0+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+0+32(SB)/4, $0x800F0E0D +DATA ยทshuffleInt32x27to31bits+0+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+0+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+0+44(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+0+48(SB)/4, $0x00808080 +DATA ยทshuffleInt32x27to31bits+0+52(SB)/4, $0x03020100 +DATA ยทshuffleInt32x27to31bits+0+56(SB)/4, $0x07060504 +DATA ยทshuffleInt32x27to31bits+0+60(SB)/4, $0x0A090807 + +DATA ยทshuffleInt32x27to31bits+0+64(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+0+68(SB)/4, $0x04808080 +DATA ยทshuffleInt32x27to31bits+0+72(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+0+76(SB)/4, $0x80808080 + +// 28 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,a,a,a,b,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,b,b,b,b,b,b] +// 7: [c,c,c,c,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,c,c,c,c,c] +// A: [c,c,c,c,d,d,d,d] +// B: [d,d,d,d,d,d,d,d] +// C: [d,d,d,d,d,d,d,d] +// D: [d,d,d,d,d,d,d,d] +// E: [e,e,e,e,e,e,e,e] +// F: [e,e,e,e,e,e,e,e] +// --- +// 0: [e,e,e,e,e,e,e,e] +// 1: [e,e,e,e,f,f,f,f] +// 2: [f,f,f,f,f,f,f,f] +// 3: [f,f,f,f,f,f,f,f] +// 4: [f,f,f,f,f,f,f,f] +// 5: [g,g,g,g,g,g,g,g] +// 6: [g,g,g,g,g,g,g,g] +// 7: [g,g,g,g,g,g,g,g] +// 8: [g,g,g,g,h,h,h,h] +// 9: [h,h,h,h,h,h,h,h] +// A: [h,h,h,h,h,h,h,h] +// B: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x27to31bits+80+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x27to31bits+80+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x27to31bits+80+8(SB)/4, $0x0A090807 +DATA ยทshuffleInt32x27to31bits+80+12(SB)/4, $0x0D0C0B0A + +DATA ยทshuffleInt32x27to31bits+80+16(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+20(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+80+32(SB)/4, $0x80800F0E +DATA ยทshuffleInt32x27to31bits+80+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+44(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+80+48(SB)/4, $0x01008080 +DATA ยทshuffleInt32x27to31bits+80+52(SB)/4, $0x04030201 +DATA ยทshuffleInt32x27to31bits+80+56(SB)/4, $0x08070605 +DATA ยทshuffleInt32x27to31bits+80+60(SB)/4, $0x0B0A0908 + +DATA ยทshuffleInt32x27to31bits+80+64(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+68(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+72(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+80+76(SB)/4, $0x80808080 + +// 29 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,a,a,a,a,b,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,b,b,b,b,b,b] +// 7: [b,b,c,c,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,c,c,c,c,c] +// A: [c,c,c,c,c,c,c,d] +// B: [d,d,d,d,d,d,d,d] +// C: [d,d,d,d,d,d,d,d] +// D: [d,d,d,d,d,d,d,d] +// E: [d,d,d,d,e,e,e,e] +// F: [e,e,e,e,e,e,e,e] +// --- +// 0: [e,e,e,e,e,e,e,e] +// 1: [e,e,e,e,e,e,e,e] +// 2: [e,f,f,f,f,f,f,f] +// 3: [f,f,f,f,f,f,f,f] +// 4: [f,f,f,f,f,f,f,f] +// 5: [f,f,f,f,f,f,g,g] +// 6: [g,g,g,g,g,g,g,g] +// 7: [g,g,g,g,g,g,g,g] +// 8: [g,g,g,g,g,g,g,g] +// 9: [g,g,g,h,h,h,h,h] +// A: [h,h,h,h,h,h,h,h] +// B: [h,h,h,h,h,h,h,h] +// C: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x27to31bits+160+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x27to31bits+160+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x27to31bits+160+8(SB)/4, $0x0A090807 +DATA ยทshuffleInt32x27to31bits+160+12(SB)/4, $0x0D0C0B0A + +DATA ยทshuffleInt32x27to31bits+160+16(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+160+20(SB)/4, $0x07808080 +DATA ยทshuffleInt32x27to31bits+160+24(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+160+28(SB)/4, $0x0E808080 + +DATA ยทshuffleInt32x27to31bits+160+32(SB)/4, $0x80800F0E +DATA ยทshuffleInt32x27to31bits+160+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+160+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+160+44(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+160+48(SB)/4, $0x01008080 +DATA ยทshuffleInt32x27to31bits+160+52(SB)/4, $0x05040302 +DATA ยทshuffleInt32x27to31bits+160+56(SB)/4, $0x08070605 +DATA ยทshuffleInt32x27to31bits+160+60(SB)/4, $0x0C0B0A09 + +DATA ยทshuffleInt32x27to31bits+160+64(SB)/4, $0x02808080 +DATA ยทshuffleInt32x27to31bits+160+68(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+160+72(SB)/4, $0x09808080 +DATA ยทshuffleInt32x27to31bits+160+76(SB)/4, $0x80808080 + +// 30 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,a,a,a,a,a,b,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,b,b,b,b,b,b] +// 7: [b,b,b,b,c,c,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,c,c,c,c,c] +// A: [c,c,c,c,c,c,c,c] +// B: [c,c,d,d,d,d,d,d] +// C: [d,d,d,d,d,d,d,d] +// D: [d,d,d,d,d,d,d,d] +// E: [d,d,d,d,d,d,d,d] +// F: [e,e,e,e,e,e,e,e] +// --- +// 0: [e,e,e,e,e,e,e,e] +// 1: [e,e,e,e,e,e,e,e] +// 2: [e,e,e,e,e,e,f,f] +// 3: [f,f,f,f,f,f,f,f] +// 4: [f,f,f,f,f,f,f,f] +// 5: [f,f,f,f,f,f,f,f] +// 6: [f,f,f,f,g,g,g,g] +// 7: [g,g,g,g,g,g,g,g] +// 8: [g,g,g,g,g,g,g,g] +// 9: [g,g,g,g,g,g,g,g] +// A: [g,g,h,h,h,h,h,h] +// B: [h,h,h,h,h,h,h,h] +// C: [h,h,h,h,h,h,h,h] +// D: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x27to31bits+240+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x27to31bits+240+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x27to31bits+240+8(SB)/4, $0x0A090807 +DATA ยทshuffleInt32x27to31bits+240+12(SB)/4, $0x0E0D0C0B + +DATA ยทshuffleInt32x27to31bits+240+16(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+240+20(SB)/4, $0x07808080 +DATA ยทshuffleInt32x27to31bits+240+24(SB)/4, $0x0B808080 +DATA ยทshuffleInt32x27to31bits+240+28(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+240+32(SB)/4, $0x8080800F +DATA ยทshuffleInt32x27to31bits+240+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+240+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+240+44(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+240+48(SB)/4, $0x02010080 +DATA ยทshuffleInt32x27to31bits+240+52(SB)/4, $0x05040302 +DATA ยทshuffleInt32x27to31bits+240+56(SB)/4, $0x09080706 +DATA ยทshuffleInt32x27to31bits+240+60(SB)/4, $0x0D0C0B0A + +DATA ยทshuffleInt32x27to31bits+240+64(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+240+68(SB)/4, $0x06808080 +DATA ยทshuffleInt32x27to31bits+240+72(SB)/4, $0x0A808080 +DATA ยทshuffleInt32x27to31bits+240+76(SB)/4, $0x80808080 + +// 31 bits => 32 bits +// ------------------ +// 0: [a,a,a,a,a,a,a,a] +// 1: [a,a,a,a,a,a,a,a] +// 2: [a,a,a,a,a,a,a,a] +// 3: [a,a,a,a,a,a,a,b] +// 4: [b,b,b,b,b,b,b,b] +// 5: [b,b,b,b,b,b,b,b] +// 6: [b,b,b,b,b,b,b,b] +// 7: [b,b,b,b,b,b,c,c] +// 8: [c,c,c,c,c,c,c,c] +// 9: [c,c,c,c,c,c,c,c] +// A: [c,c,c,c,c,c,c,c] +// B: [c,c,c,c,c,d,d,d] +// C: [d,d,d,d,d,d,d,d] +// D: [d,d,d,d,d,d,d,d] +// E: [d,d,d,d,d,d,d,d] +// F: [d,d,d,d,e,e,e,e] +// --- +// 0: [e,e,e,e,e,e,e,e] +// 1: [e,e,e,e,e,e,e,e] +// 2: [e,e,e,e,e,e,e,e] +// 3: [e,e,e,f,f,f,f,f] +// 4: [f,f,f,f,f,f,f,f] +// 5: [f,f,f,f,f,f,f,f] +// 6: [f,f,f,f,f,f,f,f] +// 7: [f,f,g,g,g,g,g,g] +// 8: [g,g,g,g,g,g,g,g] +// 9: [g,g,g,g,g,g,g,g] +// A: [g,g,g,g,g,g,g,g] +// B: [g,h,h,h,h,h,h,h] +// C: [h,h,h,h,h,h,h,h] +// D: [h,h,h,h,h,h,h,h] +// E: [h,h,h,h,h,h,h,h] +// ... +DATA ยทshuffleInt32x27to31bits+320+0(SB)/4, $0x03020100 +DATA ยทshuffleInt32x27to31bits+320+4(SB)/4, $0x06050403 +DATA ยทshuffleInt32x27to31bits+320+8(SB)/4, $0x0A090807 +DATA ยทshuffleInt32x27to31bits+320+12(SB)/4, $0x0E0D0C0B + +DATA ยทshuffleInt32x27to31bits+320+16(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+320+20(SB)/4, $0x07808080 +DATA ยทshuffleInt32x27to31bits+320+24(SB)/4, $0x0B808080 +DATA ยทshuffleInt32x27to31bits+320+28(SB)/4, $0x0F808080 + +DATA ยทshuffleInt32x27to31bits+320+32(SB)/4, $0x8080800F +DATA ยทshuffleInt32x27to31bits+320+36(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+320+40(SB)/4, $0x80808080 +DATA ยทshuffleInt32x27to31bits+320+44(SB)/4, $0x80808080 + +DATA ยทshuffleInt32x27to31bits+320+48(SB)/4, $0x02010080 +DATA ยทshuffleInt32x27to31bits+320+52(SB)/4, $0x06050403 +DATA ยทshuffleInt32x27to31bits+320+56(SB)/4, $0x0A090807 +DATA ยทshuffleInt32x27to31bits+320+60(SB)/4, $0x0E0D0C0B + +DATA ยทshuffleInt32x27to31bits+320+64(SB)/4, $0x03808080 +DATA ยทshuffleInt32x27to31bits+320+68(SB)/4, $0x07808080 +DATA ยทshuffleInt32x27to31bits+320+72(SB)/4, $0x0B808080 +DATA ยทshuffleInt32x27to31bits+320+76(SB)/4, $0x80808080 + +// The RIGHT shifts to unpack 32 bits integers. +// +// The following formula was determined empirically as the expression which +// generates shift values: +// +// shift[i] = (i * bitWidth) % 8 +// +GLOBL ยทshiftRightInt32(SB), RODATA|NOPTR, $256 + +DATA ยทshiftRightInt32+0+0(SB)/4, $0 +DATA ยทshiftRightInt32+0+4(SB)/4, $1 +DATA ยทshiftRightInt32+0+8(SB)/4, $2 +DATA ยทshiftRightInt32+0+12(SB)/4, $3 +DATA ยทshiftRightInt32+0+16(SB)/4, $4 +DATA ยทshiftRightInt32+0+20(SB)/4, $5 +DATA ยทshiftRightInt32+0+24(SB)/4, $6 +DATA ยทshiftRightInt32+0+28(SB)/4, $7 + +DATA ยทshiftRightInt32+32+0(SB)/4, $0 +DATA ยทshiftRightInt32+32+4(SB)/4, $2 +DATA ยทshiftRightInt32+32+8(SB)/4, $4 +DATA ยทshiftRightInt32+32+12(SB)/4, $6 +DATA ยทshiftRightInt32+32+16(SB)/4, $0 +DATA ยทshiftRightInt32+32+20(SB)/4, $2 +DATA ยทshiftRightInt32+32+24(SB)/4, $4 +DATA ยทshiftRightInt32+32+28(SB)/4, $6 + +DATA ยทshiftRightInt32+64+0(SB)/4, $0 +DATA ยทshiftRightInt32+64+4(SB)/4, $3 +DATA ยทshiftRightInt32+64+8(SB)/4, $6 +DATA ยทshiftRightInt32+64+12(SB)/4, $1 +DATA ยทshiftRightInt32+64+16(SB)/4, $4 +DATA ยทshiftRightInt32+64+20(SB)/4, $7 +DATA ยทshiftRightInt32+64+24(SB)/4, $2 +DATA ยทshiftRightInt32+64+28(SB)/4, $5 + +DATA ยทshiftRightInt32+96+0(SB)/4, $0 +DATA ยทshiftRightInt32+96+4(SB)/4, $4 +DATA ยทshiftRightInt32+96+8(SB)/4, $0 +DATA ยทshiftRightInt32+96+12(SB)/4, $4 +DATA ยทshiftRightInt32+96+16(SB)/4, $0 +DATA ยทshiftRightInt32+96+20(SB)/4, $4 +DATA ยทshiftRightInt32+96+24(SB)/4, $0 +DATA ยทshiftRightInt32+96+28(SB)/4, $4 + +DATA ยทshiftRightInt32+128+0(SB)/4, $0 +DATA ยทshiftRightInt32+128+4(SB)/4, $5 +DATA ยทshiftRightInt32+128+8(SB)/4, $2 +DATA ยทshiftRightInt32+128+12(SB)/4, $7 +DATA ยทshiftRightInt32+128+16(SB)/4, $4 +DATA ยทshiftRightInt32+128+20(SB)/4, $1 +DATA ยทshiftRightInt32+128+24(SB)/4, $6 +DATA ยทshiftRightInt32+128+28(SB)/4, $3 + +DATA ยทshiftRightInt32+160+0(SB)/4, $0 +DATA ยทshiftRightInt32+160+4(SB)/4, $6 +DATA ยทshiftRightInt32+160+8(SB)/4, $4 +DATA ยทshiftRightInt32+160+12(SB)/4, $2 +DATA ยทshiftRightInt32+160+16(SB)/4, $0 +DATA ยทshiftRightInt32+160+20(SB)/4, $6 +DATA ยทshiftRightInt32+160+24(SB)/4, $4 +DATA ยทshiftRightInt32+160+28(SB)/4, $2 + +DATA ยทshiftRightInt32+192+0(SB)/4, $0 +DATA ยทshiftRightInt32+192+4(SB)/4, $7 +DATA ยทshiftRightInt32+192+8(SB)/4, $6 +DATA ยทshiftRightInt32+192+12(SB)/4, $5 +DATA ยทshiftRightInt32+192+16(SB)/4, $4 +DATA ยทshiftRightInt32+192+20(SB)/4, $3 +DATA ยทshiftRightInt32+192+24(SB)/4, $2 +DATA ยทshiftRightInt32+192+28(SB)/4, $1 + +DATA ยทshiftRightInt32+224+0(SB)/4, $0 +DATA ยทshiftRightInt32+224+4(SB)/4, $0 +DATA ยทshiftRightInt32+224+8(SB)/4, $0 +DATA ยทshiftRightInt32+224+12(SB)/4, $0 +DATA ยทshiftRightInt32+224+16(SB)/4, $0 +DATA ยทshiftRightInt32+224+20(SB)/4, $0 +DATA ยทshiftRightInt32+224+24(SB)/4, $0 +DATA ยทshiftRightInt32+224+28(SB)/4, $0 + +// The LEFT shifts to unpack 32 bits integers. +// +// The following formula was determined empirically as the expression which +// generates shift values: +// +// shift[i] = (8 - (i * bitWidth)) % 8 +// +GLOBL ยทshiftLeftInt32(SB), RODATA|NOPTR, $256 + +DATA ยทshiftLeftInt32+0+0(SB)/4, $0 +DATA ยทshiftLeftInt32+0+4(SB)/4, $7 +DATA ยทshiftLeftInt32+0+8(SB)/4, $6 +DATA ยทshiftLeftInt32+0+12(SB)/4, $5 +DATA ยทshiftLeftInt32+0+16(SB)/4, $4 +DATA ยทshiftLeftInt32+0+20(SB)/4, $3 +DATA ยทshiftLeftInt32+0+24(SB)/4, $2 +DATA ยทshiftLeftInt32+0+28(SB)/4, $1 + +DATA ยทshiftLeftInt32+32+0(SB)/4, $0 +DATA ยทshiftLeftInt32+32+4(SB)/4, $6 +DATA ยทshiftLeftInt32+32+8(SB)/4, $4 +DATA ยทshiftLeftInt32+32+12(SB)/4, $2 +DATA ยทshiftLeftInt32+32+16(SB)/4, $0 +DATA ยทshiftLeftInt32+32+20(SB)/4, $6 +DATA ยทshiftLeftInt32+32+24(SB)/4, $4 +DATA ยทshiftLeftInt32+32+28(SB)/4, $2 + +DATA ยทshiftLeftInt32+64+0(SB)/4, $0 +DATA ยทshiftLeftInt32+64+4(SB)/4, $5 +DATA ยทshiftLeftInt32+64+8(SB)/4, $2 +DATA ยทshiftLeftInt32+64+12(SB)/4, $7 +DATA ยทshiftLeftInt32+64+16(SB)/4, $4 +DATA ยทshiftLeftInt32+64+20(SB)/4, $1 +DATA ยทshiftLeftInt32+64+24(SB)/4, $6 +DATA ยทshiftLeftInt32+64+28(SB)/4, $3 + +DATA ยทshiftLeftInt32+96+0(SB)/4, $0 +DATA ยทshiftLeftInt32+96+4(SB)/4, $4 +DATA ยทshiftLeftInt32+96+8(SB)/4, $0 +DATA ยทshiftLeftInt32+96+12(SB)/4, $4 +DATA ยทshiftLeftInt32+96+16(SB)/4, $0 +DATA ยทshiftLeftInt32+96+20(SB)/4, $4 +DATA ยทshiftLeftInt32+96+24(SB)/4, $0 +DATA ยทshiftLeftInt32+96+28(SB)/4, $4 + +DATA ยทshiftLeftInt32+128+0(SB)/4, $0 +DATA ยทshiftLeftInt32+128+4(SB)/4, $3 +DATA ยทshiftLeftInt32+128+8(SB)/4, $6 +DATA ยทshiftLeftInt32+128+12(SB)/4, $1 +DATA ยทshiftLeftInt32+128+16(SB)/4, $4 +DATA ยทshiftLeftInt32+128+20(SB)/4, $7 +DATA ยทshiftLeftInt32+128+24(SB)/4, $2 +DATA ยทshiftLeftInt32+128+28(SB)/4, $5 + +DATA ยทshiftLeftInt32+160+0(SB)/4, $0 +DATA ยทshiftLeftInt32+160+4(SB)/4, $2 +DATA ยทshiftLeftInt32+160+8(SB)/4, $4 +DATA ยทshiftLeftInt32+160+12(SB)/4, $6 +DATA ยทshiftLeftInt32+160+16(SB)/4, $0 +DATA ยทshiftLeftInt32+160+20(SB)/4, $2 +DATA ยทshiftLeftInt32+160+24(SB)/4, $4 +DATA ยทshiftLeftInt32+160+28(SB)/4, $6 + +DATA ยทshiftLeftInt32+192+0(SB)/4, $0 +DATA ยทshiftLeftInt32+192+4(SB)/4, $1 +DATA ยทshiftLeftInt32+192+8(SB)/4, $2 +DATA ยทshiftLeftInt32+192+12(SB)/4, $3 +DATA ยทshiftLeftInt32+192+16(SB)/4, $4 +DATA ยทshiftLeftInt32+192+20(SB)/4, $5 +DATA ยทshiftLeftInt32+192+24(SB)/4, $6 +DATA ยทshiftLeftInt32+192+28(SB)/4, $7 + +DATA ยทshiftLeftInt32+224+0(SB)/4, $0 +DATA ยทshiftLeftInt32+224+4(SB)/4, $0 +DATA ยทshiftLeftInt32+224+8(SB)/4, $0 +DATA ยทshiftLeftInt32+224+12(SB)/4, $0 +DATA ยทshiftLeftInt32+224+16(SB)/4, $0 +DATA ยทshiftLeftInt32+224+20(SB)/4, $0 +DATA ยทshiftLeftInt32+224+24(SB)/4, $0 +DATA ยทshiftLeftInt32+224+28(SB)/4, $0 diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/pack.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/pack.go new file mode 100644 index 0000000000..585202842f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/pack.go @@ -0,0 +1,83 @@ +package bitpack + +import ( + "encoding/binary" +) + +// PackInt32 packs values from src to dst, each value is packed into the given +// bit width regardless of how many bits are needed to represent it. +// +// The function panics if dst is too short to hold the bit packed values. +func PackInt32(dst []byte, src []int32, bitWidth uint) { + assertPack(dst, len(src), bitWidth) + packInt32(dst, src, bitWidth) +} + +func packInt32(dst []byte, src []int32, bitWidth uint) { + n := ByteCount(uint(len(src)) * bitWidth) + b := dst[:n] + + for i := range b { + b[i] = 0 + } + + bitMask := uint32(1<> (32 - j)) + + binary.LittleEndian.PutUint32(dst[(i+0)*4:], lo) + binary.LittleEndian.PutUint32(dst[(i+1)*4:], hi) + + bitOffset += bitWidth + } +} + +// PackInt64 packs values from src to dst, each value is packed into the given +// bit width regardless of how many bits are needed to represent it. +// +// The function panics if dst is too short to hold the bit packed values. +func PackInt64(dst []byte, src []int64, bitWidth uint) { + assertPack(dst, len(src), bitWidth) + packInt64(dst, src, bitWidth) +} + +func packInt64(dst []byte, src []int64, bitWidth uint) { + n := ByteCount(uint(len(src)) * bitWidth) + b := dst[:n] + + for i := range b { + b[i] = 0 + } + + bitMask := uint64(1<> (64 - j)) + + binary.LittleEndian.PutUint64(dst[(i+0)*8:], lo) + binary.LittleEndian.PutUint64(dst[(i+1)*8:], hi) + + bitOffset += bitWidth + } +} + +func assertPack(dst []byte, count int, bitWidth uint) { + _ = dst[:ByteCount(bitWidth*uint(count))] +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack.go new file mode 100644 index 0000000000..752a6f7ff8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack.go @@ -0,0 +1,27 @@ +package bitpack + +// PaddingInt32 is the padding expected to exist after the end of input buffers +// for the UnpackInt32 algorithm to avoid reading beyond the end of the input. +const PaddingInt32 = 16 + +// PaddingInt64 is the padding expected to exist after the end of input buffers +// for the UnpackInt32 algorithm to avoid reading beyond the end of the input. +const PaddingInt64 = 32 + +// UnpackInt32 unpacks 32 bit integers from src to dst. +// +// The function unpacked len(dst) integers, it panics if src is too short to +// contain len(dst) values of the given bit width. +func UnpackInt32(dst []int32, src []byte, bitWidth uint) { + _ = src[:ByteCount(bitWidth*uint(len(dst))+8*PaddingInt32)] + unpackInt32(dst, src, bitWidth) +} + +// UnpackInt64 unpacks 64 bit integers from src to dst. +// +// The function unpacked len(dst) integers, it panics if src is too short to +// contain len(dst) values of the given bit width. +func UnpackInt64(dst []int64, src []byte, bitWidth uint) { + _ = src[:ByteCount(bitWidth*uint(len(dst))+8*PaddingInt64)] + unpackInt64(dst, src, bitWidth) +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.go new file mode 100644 index 0000000000..f393222391 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.go @@ -0,0 +1,36 @@ +//go:build !purego + +package bitpack + +import ( + "github.com/parquet-go/parquet-go/internal/unsafecast" + "golang.org/x/sys/cpu" +) + +//go:noescape +func unpackInt32Default(dst []int32, src []byte, bitWidth uint) + +//go:noescape +func unpackInt32x1to16bitsAVX2(dst []int32, src []byte, bitWidth uint) + +//go:noescape +func unpackInt32x17to26bitsAVX2(dst []int32, src []byte, bitWidth uint) + +//go:noescape +func unpackInt32x27to31bitsAVX2(dst []int32, src []byte, bitWidth uint) + +func unpackInt32(dst []int32, src []byte, bitWidth uint) { + hasAVX2 := cpu.X86.HasAVX2 + switch { + case hasAVX2 && bitWidth <= 16: + unpackInt32x1to16bitsAVX2(dst, src, bitWidth) + case hasAVX2 && bitWidth <= 26: + unpackInt32x17to26bitsAVX2(dst, src, bitWidth) + case hasAVX2 && bitWidth <= 31: + unpackInt32x27to31bitsAVX2(dst, src, bitWidth) + case bitWidth == 32: + copy(dst, unsafecast.Slice[int32](src)) + default: + unpackInt32Default(dst, src, bitWidth) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.s b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.s new file mode 100644 index 0000000000..e0f2bf4a4d --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_amd64.s @@ -0,0 +1,352 @@ +//go:build !purego + +#include "funcdata.h" +#include "textflag.h" + +// func unpackInt32Default(dst []int32, src []byte, bitWidth uint) +TEXT ยทunpackInt32Default(SB), NOSPLIT, $0-56 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), DX + MOVQ src_base+24(FP), BX + MOVQ bitWidth+48(FP), CX + + MOVQ $1, R8 // bitMask = (1 << bitWidth) - 1 + SHLQ CX, R8 + DECQ R8 + MOVQ CX, R9 // bitWidth + + XORQ DI, DI // bitOffset + XORQ SI, SI // index + JMP test +loop: + MOVQ DI, R10 + MOVQ DI, CX + SHRQ $5, R10 // i = bitOffset / 32 + ANDQ $0b11111, CX // j = bitOffset % 32 + + MOVL (BX)(R10*4), R11 + MOVL R8, R12 // d = bitMask + SHLL CX, R12 // d = d << j + ANDL R12, R11 // d = src[i] & d + SHRL CX, R11 // d = d >> j + + MOVL CX, R13 + ADDL R9, R13 + CMPL R13, $32 + JBE next // j+bitWidth <= 32 ? + + MOVL 4(BX)(R10*4), R14 + MOVL CX, R12 + MOVL $32, CX + SUBL R12, CX // k = 32 - j + MOVL R8, R12 // c = bitMask + SHRL CX, R12 // c = c >> k + ANDL R12, R14 // c = src[i+1] & c + SHLL CX, R14 // c = c << k + ORL R14, R11 // d = d | c +next: + MOVL R11, (AX)(SI*4) // dst[n] = d + ADDQ R9, DI // bitOffset += bitWidth + INCQ SI +test: + CMPQ SI, DX + JNE loop + RET + +// ----------------------------------------------------------------------------- +// The unpack* functions below are adaptations of the algorithms +// described in "Decoding billions of integers per second through vectorization" +// from D. Lemire & L. Boytsov, the following changes were made: +// +// - The paper described two methods for decoding integers called "horizontal" +// and "vertical". The "horizontal" version is the one that applies the best +// to the bit packing done in the Parquet delta encoding; however, it also +// differs in some ways, many compression techniques discussed in the paper +// are not implemented in the Parquet format. +// +// - The paper focuses on implementations based on SSE instructions, which +// describes how to use PMULLD to emulate the lack of variable bit shift +// for packed integers. Our version of the bit unpacking algorithms here +// uses AVX2 and can perform variable bit shifts using VPSRLVD, which yields +// better throughput since the instruction latency is a single CPU cycle, +// vs 10 for VPMULLD. +// +// - The reference implementation at https://github.com/lemire/FastPFor/ uses +// specializations for each bit size, resulting in 32 unique functions. +// Our version here are more generic, we provide 3 variations of the +// algorithm for bit widths 1 to 16, 17 to 26, and 27 to 31 (unpacking 32 +// bits values is a simple copy). In that regard, our implementation is +// somewhat an improvement over the reference, since it uses less code and +// less memory to hold the shuffle masks and shift tables. +// +// Technically, each specialization of our functions could be expressed by the +// algorithm used for unpacking values of 27 to 31 bits. However, multiple steps +// of the main loop can be removed for lower bit widths, providing up to ~35% +// better throughput for smaller sizes. Since we expect delta encoding to often +// result in bit packing values to smaller bit widths, the specializations are +// worth the extra complexity. +// +// For more details, see: https://arxiv.org/pdf/1209.2137v5.pdf +// ----------------------------------------------------------------------------- + +// unpackInt32x1to16bitsAVX2 is the implementation of the bit unpacking +// algorithm for inputs of bit width 1 to 16. +// +// In this version of the algorithm, we can perform a single memory load in each +// loop iteration since we know that 8 values will fit in a single XMM register. +// +// func unpackInt32x1to16bitsAVX2(dst []int32, src []byte, bitWidth uint) +TEXT ยทunpackInt32x1to16bitsAVX2(SB), NOSPLIT, $56-56 + NO_LOCAL_POINTERS + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), DX + MOVQ src_base+24(FP), BX + MOVQ bitWidth+48(FP), CX + + CMPQ DX, $8 + JB tail + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + XORQ SI, SI + + MOVQ $1, R8 + SHLQ CX, R8 + DECQ R8 + MOVQ R8, X0 + VPBROADCASTD X0, X0 // bitMask = (1 << bitWidth) - 1 + + MOVQ CX, R9 + DECQ R9 + SHLQ $5, R9 // 32 * (bitWidth - 1) + + MOVQ CX, R10 + DECQ R10 + SHLQ $5, R10 + ANDQ $0xFF, R10 // (32 * (bitWidth - 1)) % 256 + + LEAQ ยทshuffleInt32x1to16bits(SB), R11 + VMOVDQA (R11)(R9*1), X1 + VMOVDQA 16(R11)(R9*1), X2 + + LEAQ ยทshiftRightInt32(SB), R12 + VMOVDQA (R12)(R10*1), X3 + VMOVDQA 16(R12)(R10*1), X4 +loop: + VMOVDQU (BX), X7 + + VPSHUFB X1, X7, X5 + VPSHUFB X2, X7, X6 + + VPSRLVD X3, X5, X5 + VPSRLVD X4, X6, X6 + + VPAND X0, X5, X5 + VPAND X0, X6, X6 + + VMOVDQU X5, (AX)(SI*4) + VMOVDQU X6, 16(AX)(SI*4) + + ADDQ CX, BX + ADDQ $8, SI + CMPQ SI, DI + JNE loop + VZEROUPPER + + CMPQ SI, DX + JE done + LEAQ (AX)(SI*4), AX + SUBQ SI, DX +tail: + MOVQ AX, dst_base-56(SP) + MOVQ DX, dst_len-48(SP) + MOVQ BX, src_base-32(SP) + MOVQ CX, bitWidth-8(SP) + CALL ยทunpackInt32Default(SB) +done: + RET + +// unpackInt32x17to26bitsAVX2 is the implementation of the bit unpacking +// algorithm for inputs of bit width 17 to 26. +// +// In this version of the algorithm, we need to 32 bytes at each loop iteration +// because 8 bit-packed values will span across two XMM registers. +// +// func unpackInt32x17to26bitsAVX2(dst []int32, src []byte, bitWidth uint) +TEXT ยทunpackInt32x17to26bitsAVX2(SB), NOSPLIT, $56-56 + NO_LOCAL_POINTERS + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), DX + MOVQ src_base+24(FP), BX + MOVQ bitWidth+48(FP), CX + + CMPQ DX, $8 + JB tail + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + XORQ SI, SI + + MOVQ $1, R8 + SHLQ CX, R8 + DECQ R8 + MOVQ R8, X0 + VPBROADCASTD X0, X0 + + MOVQ CX, R9 + SUBQ $17, R9 + IMULQ $48, R9 // 48 * (bitWidth - 17) + + MOVQ CX, R10 + DECQ R10 + SHLQ $5, R10 + ANDQ $0xFF, R10 // (32 * (bitWidth - 1)) % 256 + + LEAQ ยทshuffleInt32x17to26bits(SB), R11 + VMOVDQA (R11)(R9*1), X1 + VMOVDQA 16(R11)(R9*1), X2 + VMOVDQA 32(R11)(R9*1), X3 + + LEAQ ยทshiftRightInt32(SB), R12 + VMOVDQA (R12)(R10*1), X4 + VMOVDQA 16(R12)(R10*1), X5 +loop: + VMOVDQU (BX), X6 + VMOVDQU 16(BX), X7 + + VPSHUFB X1, X6, X8 + VPSHUFB X2, X6, X9 + VPSHUFB X3, X7, X10 + VPOR X10, X9, X9 + + VPSRLVD X4, X8, X8 + VPSRLVD X5, X9, X9 + + VPAND X0, X8, X8 + VPAND X0, X9, X9 + + VMOVDQU X8, (AX)(SI*4) + VMOVDQU X9, 16(AX)(SI*4) + + ADDQ CX, BX + ADDQ $8, SI + CMPQ SI, DI + JNE loop + VZEROUPPER + + CMPQ SI, DX + JE done + LEAQ (AX)(SI*4), AX + SUBQ SI, DX +tail: + MOVQ AX, dst_base-56(SP) + MOVQ DX, dst_len-48(SP) + MOVQ BX, src_base-32(SP) + MOVQ CX, bitWidth-8(SP) + CALL ยทunpackInt32Default(SB) +done: + RET + +// unpackInt32x27to31bitsAVX2 is the implementation of the bit unpacking +// algorithm for inputs of bit width 27 to 31. +// +// In this version of the algorithm the bit-packed values may span across up to +// 5 bytes. The simpler approach for smaller bit widths where we could perform a +// single shuffle + shift to unpack the values do not work anymore. +// +// Values are unpacked in two steps: the first one extracts lower bits which are +// shifted RIGHT to align on the beginning of 32 bit words, the second extracts +// upper bits which are shifted LEFT to be moved to the end of the 32 bit words. +// +// The amount of LEFT shifts is always "8 minus the amount of RIGHT shift". +// +// func unpackInt32x27to31bitsAVX2(dst []int32, src []byte, bitWidth uint) +TEXT ยทunpackInt32x27to31bitsAVX2(SB), NOSPLIT, $56-56 + NO_LOCAL_POINTERS + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), DX + MOVQ src_base+24(FP), BX + MOVQ bitWidth+48(FP), CX + + CMPQ DX, $8 + JB tail + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + XORQ SI, SI + + MOVQ $1, R8 + SHLQ CX, R8 + DECQ R8 + MOVQ R8, X0 + VPBROADCASTD X0, X0 + + MOVQ CX, R9 + SUBQ $27, R9 + IMULQ $80, R9 // (80 * (bitWidth - 27)) + + MOVQ CX, R10 + DECQ R10 + SHLQ $5, R10 + ANDQ $0xFF, R10 // (32 * (bitWidth - 1)) % 256 + + LEAQ ยทshuffleInt32x27to31bits(SB), R11 + VMOVDQA (R11)(R9*1), X1 + VMOVDQA 16(R11)(R9*1), X2 + VMOVDQA 32(R11)(R9*1), X3 + VMOVDQA 48(R11)(R9*1), X4 + VMOVDQA 64(R11)(R9*1), X5 + + LEAQ ยทshiftRightInt32(SB), R12 + LEAQ ยทshiftLeftInt32(SB), R13 + VMOVDQA (R12)(R10*1), X6 + VMOVDQA (R13)(R10*1), X7 + VMOVDQA 16(R12)(R10*1), X8 + VMOVDQA 16(R13)(R10*1), X9 +loop: + VMOVDQU (BX), X10 + VMOVDQU 16(BX), X11 + + VPSHUFB X1, X10, X12 + VPSHUFB X2, X10, X13 + VPSHUFB X3, X10, X14 + VPSHUFB X4, X11, X15 + VPSHUFB X5, X11, X11 + + VPSRLVD X6, X12, X12 + VPSLLVD X7, X13, X13 + VPSRLVD X8, X14, X14 + VPSRLVD X8, X15, X15 + VPSLLVD X9, X11, X11 + + VPOR X13, X12, X12 + VPOR X15, X14, X14 + VPOR X11, X14, X14 + + VPAND X0, X12, X12 + VPAND X0, X14, X14 + + VMOVDQU X12, (AX)(SI*4) + VMOVDQU X14, 16(AX)(SI*4) + + ADDQ CX, BX + ADDQ $8, SI + CMPQ SI, DI + JNE loop + VZEROUPPER + + CMPQ SI, DX + JE done + LEAQ (AX)(SI*4), AX + SUBQ SI, DX +tail: + MOVQ AX, dst_base-56(SP) + MOVQ DX, dst_len-48(SP) + MOVQ BX, src_base-32(SP) + MOVQ CX, bitWidth-8(SP) + CALL ยทunpackInt32Default(SB) +done: + RET diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_be.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_be.go new file mode 100644 index 0000000000..0f4ba054c4 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_be.go @@ -0,0 +1,15 @@ +//go:build s390x + +package bitpack + +import "encoding/binary" + +func unsafecastBytesToUint32(src []byte) []uint32 { + out := make([]uint32, len(src)/4) + idx := 0 + for k := range out { + out[k] = binary.LittleEndian.Uint32((src)[idx:(4 + idx)]) + idx += 4 + } + return out +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_le.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_le.go new file mode 100644 index 0000000000..f754e704ff --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_le.go @@ -0,0 +1,9 @@ +//go:build !s390x + +package bitpack + +import "github.com/parquet-go/parquet-go/internal/unsafecast" + +func unsafecastBytesToUint32(src []byte) []uint32 { + return unsafecast.Slice[uint32](src) +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_purego.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_purego.go new file mode 100644 index 0000000000..1e65d8c02b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int32_purego.go @@ -0,0 +1,21 @@ +//go:build purego || !amd64 + +package bitpack + +func unpackInt32(dst []int32, src []byte, bitWidth uint) { + bits := unsafecastBytesToUint32(src) + bitMask := uint32(1<> j + if j+bitWidth > 32 { + k := 32 - j + d |= (bits[i+1] & (bitMask >> k)) << k + } + dst[n] = int32(d) + bitOffset += bitWidth + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.go new file mode 100644 index 0000000000..9314e73c2e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.go @@ -0,0 +1,26 @@ +//go:build !purego + +package bitpack + +import ( + "github.com/parquet-go/parquet-go/internal/unsafecast" + "golang.org/x/sys/cpu" +) + +//go:noescape +func unpackInt64Default(dst []int64, src []byte, bitWidth uint) + +//go:noescape +func unpackInt64x1to32bitsAVX2(dst []int64, src []byte, bitWidth uint) + +func unpackInt64(dst []int64, src []byte, bitWidth uint) { + hasAVX2 := cpu.X86.HasAVX2 + switch { + case hasAVX2 && bitWidth <= 32: + unpackInt64x1to32bitsAVX2(dst, src, bitWidth) + case bitWidth == 64: + copy(dst, unsafecast.Slice[int64](src)) + default: + unpackInt64Default(dst, src, bitWidth) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.s b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.s new file mode 100644 index 0000000000..91d28f680e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_amd64.s @@ -0,0 +1,198 @@ +//go:build !purego + +#include "funcdata.h" +#include "textflag.h" + +// func unpackInt64Default(dst []int64, src []uint32, bitWidth uint) +TEXT ยทunpackInt64Default(SB), NOSPLIT, $0-56 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), DX + MOVQ src_base+24(FP), BX + MOVQ bitWidth+48(FP), CX + + MOVQ $1, R8 // bitMask = (1 << bitWidth) - 1 + SHLQ CX, R8, R8 + DECQ R8 + MOVQ CX, R9 // bitWidth + + XORQ DI, DI // bitOffset + XORQ SI, SI // index + XORQ R10, R10 + XORQ R11, R11 + XORQ R14, R14 + JMP test +loop: + MOVQ DI, R10 + MOVQ DI, CX + SHRQ $5, R10 // i = bitOffset / 32 + ANDQ $0b11111, CX // j = bitOffset % 32 + + MOVLQZX (BX)(R10*4), R11 + MOVQ R8, R12 // d = bitMask + SHLQ CX, R12 // d = d << j + ANDQ R12, R11 // d = src[i] & d + SHRQ CX, R11 // d = d >> j + + MOVQ CX, R13 + ADDQ R9, R13 + CMPQ R13, $32 + JBE next // j+bitWidth <= 32 ? + MOVQ CX, R15 // j + + MOVLQZX 4(BX)(R10*4), R14 + MOVQ $32, CX + SUBQ R15, CX // k = 32 - j + MOVQ R8, R12 // c = bitMask + SHRQ CX, R12 // c = c >> k + ANDQ R12, R14 // c = src[i+1] & c + SHLQ CX, R14 // c = c << k + ORQ R14, R11 // d = d | c + + CMPQ R13, $64 + JBE next + + MOVLQZX 8(BX)(R10*4), R14 + MOVQ $64, CX + SUBQ R15, CX // k = 64 - j + MOVQ R8, R12 // c = bitMask + SHRQ CX, R12 // c = c >> k + ANDQ R12, R14 // c = src[i+2] & c + SHLQ CX, R14 // c = c << k + ORQ R14, R11 // d = d | c +next: + MOVQ R11, (AX)(SI*8) // dst[n] = d + ADDQ R9, DI // bitOffset += bitWidth + INCQ SI +test: + CMPQ SI, DX + JNE loop + RET + +// This bit unpacking function was inspired from the 32 bit version, but +// adapted to account for the fact that eight 64 bit values span across +// two YMM registers, and across lanes of YMM registers. +// +// Because of the two lanes of YMM registers, we cannot use the VPSHUFB +// instruction to dispatch bytes of the input to the registers. Instead we use +// the VPERMD instruction, which has higher latency but supports dispatching +// bytes across register lanes. Measurable throughput gains remain despite the +// algorithm running on a few more CPU cycles per loop. +// +// The initialization phase of this algorithm generates masks for +// permutations and shifts used to decode the bit-packed values. +// +// The permutation masks are written to Y7 and Y8, and contain the results +// of this formula: +// +// temp[i] = (bitWidth * i) / 32 +// mask[i] = temp[i] | ((temp[i] + 1) << 32) +// +// Since VPERMQ only supports reading the permutation combination from an +// immediate value, we use VPERMD and generate permutation for pairs of two +// consecutive 32 bit words, which is why we have the upper part of each 64 +// bit word set with (x+1)<<32. +// +// The masks for right shifts are written to Y5 and Y6, and computed with +// this formula: +// +// shift[i] = (bitWidth * i) - (32 * ((bitWidth * i) / 32)) +// +// The amount to shift by is the number of values previously unpacked, offseted +// by the byte count of 32 bit words that we read from first bits from. +// +// Technically the masks could be precomputed and declared in global tables; +// however, declaring masks for all bit width is tedious and makes code +// maintenance more costly for no measurable benefits on production workloads. +// +// func unpackInt64x1to32bitsAVX2(dst []int64, src []byte, bitWidth uint) +TEXT ยทunpackInt64x1to32bitsAVX2(SB), NOSPLIT, $56-56 + NO_LOCAL_POINTERS + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), DX + MOVQ src_base+24(FP), BX + MOVQ bitWidth+48(FP), CX + + CMPQ DX, $8 + JB tail + + MOVQ DX, DI + SHRQ $3, DI + SHLQ $3, DI + XORQ SI, SI + + MOVQ $1, R8 + SHLQ CX, R8 + DECQ R8 + MOVQ R8, X0 + VPBROADCASTQ X0, Y0 // bitMask = (1 << bitWidth) - 1 + + VPCMPEQQ Y1, Y1, Y1 + VPSRLQ $63, Y1, Y1 // [1,1,1,1] + + MOVQ CX, X2 + VPBROADCASTQ X2, Y2 // [bitWidth] + + VMOVDQU range0n7<>+0(SB), Y3 // [0,1,2,3] + VMOVDQU range0n7<>+32(SB), Y4 // [4,5,6,7] + + VPMULLD Y2, Y3, Y5 // [bitWidth] * [0,1,2,3] + VPMULLD Y2, Y4, Y6 // [bitWidth] * [4,5,6,7] + + VPSRLQ $5, Y5, Y7 // ([bitWidth] * [0,1,2,3]) / 32 + VPSRLQ $5, Y6, Y8 // ([bitWidth] * [4,5,6,7]) / 32 + + VPSLLQ $5, Y7, Y9 // (([bitWidth] * [0,1,2,3]) / 32) * 32 + VPSLLQ $5, Y8, Y10 // (([bitWidth] * [4,5,6,7]) / 32) * 32 + + VPADDQ Y1, Y7, Y11 + VPADDQ Y1, Y8, Y12 + VPSLLQ $32, Y11, Y11 + VPSLLQ $32, Y12, Y12 + VPOR Y11, Y7, Y7 // permutations[i] = [i | ((i + 1) << 32)] + VPOR Y12, Y8, Y8 // permutations[i] = [i | ((i + 1) << 32)] + + VPSUBQ Y9, Y5, Y5 // shifts + VPSUBQ Y10, Y6, Y6 +loop: + VMOVDQU (BX), Y1 + + VPERMD Y1, Y7, Y2 + VPERMD Y1, Y8, Y3 + + VPSRLVQ Y5, Y2, Y2 + VPSRLVQ Y6, Y3, Y3 + + VPAND Y0, Y2, Y2 + VPAND Y0, Y3, Y3 + + VMOVDQU Y2, (AX)(SI*8) + VMOVDQU Y3, 32(AX)(SI*8) + + ADDQ CX, BX + ADDQ $8, SI + CMPQ SI, DI + JNE loop + VZEROUPPER + + CMPQ SI, DX + JE done + LEAQ (AX)(SI*8), AX + SUBQ SI, DX +tail: + MOVQ AX, dst_base-56(SP) + MOVQ DX, dst_len-48(SP) + MOVQ BX, src_base-32(SP) + MOVQ CX, bitWidth-8(SP) + CALL ยทunpackInt64Default(SB) +done: + RET + +GLOBL range0n7<>(SB), RODATA|NOPTR, $64 +DATA range0n7<>+0(SB)/8, $0 +DATA range0n7<>+8(SB)/8, $1 +DATA range0n7<>+16(SB)/8, $2 +DATA range0n7<>+24(SB)/8, $3 +DATA range0n7<>+32(SB)/8, $4 +DATA range0n7<>+40(SB)/8, $5 +DATA range0n7<>+48(SB)/8, $6 +DATA range0n7<>+56(SB)/8, $7 diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_purego.go b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_purego.go new file mode 100644 index 0000000000..7a20882efb --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bitpack/unpack_int64_purego.go @@ -0,0 +1,25 @@ +//go:build purego || !amd64 + +package bitpack + +func unpackInt64(dst []int64, src []byte, bitWidth uint) { + bits := unsafecastBytesToUint32(src) + bitMask := uint64(1<> j + if j+bitWidth > 32 { + k := 32 - j + d |= (uint64(bits[i+1]) & (bitMask >> k)) << k + if j+bitWidth > 64 { + k := 64 - j + d |= (uint64(bits[i+2]) & (bitMask >> k)) << k + } + } + dst[n] = int64(d) + bitOffset += bitWidth + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.go b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.go new file mode 100644 index 0000000000..407f1cfc52 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.go @@ -0,0 +1,17 @@ +//go:build !purego + +package bytealg + +//go:noescape +func broadcastAVX2(dst []byte, src byte) + +// Broadcast writes the src value to all bytes of dst. +func Broadcast(dst []byte, src byte) { + if len(dst) >= 8 && hasAVX2 { + broadcastAVX2(dst, src) + } else { + for i := range dst { + dst[i] = src + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.s b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.s new file mode 100644 index 0000000000..241a658e31 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_amd64.s @@ -0,0 +1,51 @@ +//go:build !purego + +#include "textflag.h" + +// func broadcastAVX2(dst []byte, src byte) +TEXT ยทbroadcastAVX2(SB), NOSPLIT, $0-25 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), BX + MOVBQZX src+24(FP), CX + + CMPQ BX, $8 + JBE test + + CMPQ BX, $64 + JB init8 + + XORQ SI, SI + MOVQ BX, DX + SHRQ $6, DX + SHLQ $6, DX + MOVQ CX, X0 + VPBROADCASTB X0, Y0 +loop64: + VMOVDQU Y0, (AX)(SI*1) + VMOVDQU Y0, 32(AX)(SI*1) + ADDQ $64, SI + CMPQ SI, DX + JNE loop64 + VMOVDQU Y0, -64(AX)(BX*1) + VMOVDQU Y0, -32(AX)(BX*1) + VZEROUPPER + RET + +init8: + MOVQ $0x0101010101010101, R8 + IMULQ R8, CX +loop8: + MOVQ CX, -8(AX)(BX*1) + SUBQ $8, BX + CMPQ BX, $8 + JAE loop8 + MOVQ CX, (AX) + RET + +loop: + MOVB CX, -1(AX)(BX*1) + DECQ BX +test: + CMPQ BX, $0 + JNE loop + RET diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_purego.go b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_purego.go new file mode 100644 index 0000000000..fd858e0a23 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/broadcast_purego.go @@ -0,0 +1,9 @@ +//go:build purego || !amd64 + +package bytealg + +func Broadcast(dst []byte, src byte) { + for i := range dst { + dst[i] = src + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg.go b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg.go new file mode 100644 index 0000000000..0698df22f5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg.go @@ -0,0 +1,2 @@ +// Package bytealg contains optimized algorithms operating on byte slices. +package bytealg diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg_amd64.go b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg_amd64.go new file mode 100644 index 0000000000..edfcb64c74 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/bytealg_amd64.go @@ -0,0 +1,17 @@ +//go:build !purego + +package bytealg + +import "golang.org/x/sys/cpu" + +var ( + hasAVX2 = cpu.X86.HasAVX2 + // These use AVX-512 instructions in the countByte algorithm relies + // operations that are available in the AVX512BW extension: + // * VPCMPUB + // * KMOVQ + // + // Note that the function will fallback to an AVX2 version if those + // instructions are not available. + hasAVX512Count = cpu.X86.HasAVX512VL && cpu.X86.HasAVX512BW +) diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.go b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.go new file mode 100644 index 0000000000..b41d3d8d97 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.go @@ -0,0 +1,26 @@ +//go:build !purego + +package bytealg + +// This function is similar to using the standard bytes.Count function with a +// one-byte separator. However, the implementation makes use of AVX-512 when +// possible, which yields measurable throughput improvements: +// +// name old time/op new time/op delta +// CountByte 82.5ns ยฑ 0% 43.9ns ยฑ 0% -46.74% (p=0.000 n=10+10) +// +// name old speed new speed delta +// CountByte 49.6GB/s ยฑ 0% 93.2GB/s ยฑ 0% +87.74% (p=0.000 n=10+10) +// +// On systems that do not have AVX-512, the AVX2 version of the code is also +// optimized to make use of multiple register lanes, which gives a bit better +// throughput than the standard library function: +// +// name old time/op new time/op delta +// CountByte 82.5ns ยฑ 0% 61.0ns ยฑ 0% -26.04% (p=0.000 n=10+10) +// +// name old speed new speed delta +// CountByte 49.6GB/s ยฑ 0% 67.1GB/s ยฑ 0% +35.21% (p=0.000 n=10+10) +// +//go:noescape +func Count(data []byte, value byte) int diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.s b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.s new file mode 100644 index 0000000000..1a75a0f798 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_amd64.s @@ -0,0 +1,100 @@ +//go:build !purego + +#include "textflag.h" + +// func Count(data []byte, value byte) int +TEXT ยทCount(SB), NOSPLIT, $0-40 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + MOVB value+24(FP), BX + MOVQ CX, DX // len + ADDQ AX, CX // end + XORQ SI, SI // count + + CMPQ DX, $256 + JB test + + CMPB ยทhasAVX2(SB), $0 + JE test + + XORQ R12, R12 + XORQ R13, R13 + XORQ R14, R14 + XORQ DI, DI + + CMPB ยทhasAVX512Count(SB), $0 + JE initAVX2 + + SHRQ $8, DX + SHLQ $8, DX + ADDQ AX, DX + VPBROADCASTB BX, Z0 +loopAVX512: + VMOVDQU64 (AX), Z1 + VMOVDQU64 64(AX), Z2 + VMOVDQU64 128(AX), Z3 + VMOVDQU64 192(AX), Z4 + VPCMPUB $0, Z0, Z1, K1 + VPCMPUB $0, Z0, Z2, K2 + VPCMPUB $0, Z0, Z3, K3 + VPCMPUB $0, Z0, Z4, K4 + KMOVQ K1, R8 + KMOVQ K2, R9 + KMOVQ K3, R10 + KMOVQ K4, R11 + POPCNTQ R8, R8 + POPCNTQ R9, R9 + POPCNTQ R10, R10 + POPCNTQ R11, R11 + ADDQ R8, R12 + ADDQ R9, R13 + ADDQ R10, R14 + ADDQ R11, DI + ADDQ $256, AX + CMPQ AX, DX + JNE loopAVX512 + ADDQ R12, R13 + ADDQ R14, DI + ADDQ R13, SI + ADDQ DI, SI + JMP doneAVX + +initAVX2: + SHRQ $6, DX + SHLQ $6, DX + ADDQ AX, DX + VPBROADCASTB value+24(FP), Y0 +loopAVX2: + VMOVDQU (AX), Y1 + VMOVDQU 32(AX), Y2 + VPCMPEQB Y0, Y1, Y1 + VPCMPEQB Y0, Y2, Y2 + VPMOVMSKB Y1, R12 + VPMOVMSKB Y2, R13 + POPCNTL R12, R12 + POPCNTL R13, R13 + ADDQ R12, R14 + ADDQ R13, DI + ADDQ $64, AX + CMPQ AX, DX + JNE loopAVX2 + ADDQ R14, SI + ADDQ DI, SI + +doneAVX: + VZEROUPPER + JMP test + +loop: + MOVQ SI, DI + INCQ DI + MOVB (AX), R8 + CMPB BX, R8 + CMOVQEQ DI, SI + INCQ AX +test: + CMPQ AX, CX + JNE loop +done: + MOVQ SI, ret+32(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_purego.go b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_purego.go new file mode 100644 index 0000000000..de35ee44a3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/bytealg/count_purego.go @@ -0,0 +1,9 @@ +//go:build purego || !amd64 + +package bytealg + +import "bytes" + +func Count(data []byte, value byte) int { + return bytes.Count(data, []byte{value}) +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/debug/debug.go b/vendor/github.com/parquet-go/parquet-go/internal/debug/debug.go new file mode 100644 index 0000000000..45c12e2a03 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/debug/debug.go @@ -0,0 +1,95 @@ +package debug + +import ( + "encoding/hex" + "fmt" + "io" + "log" + "os" + "strconv" + "strings" +) + +func ReaderAt(reader io.ReaderAt, prefix string) io.ReaderAt { + return &ioReaderAt{ + reader: reader, + prefix: prefix, + } +} + +type ioReaderAt struct { + reader io.ReaderAt + prefix string +} + +func (d *ioReaderAt) ReadAt(b []byte, off int64) (int, error) { + n, err := d.reader.ReadAt(b, off) + fmt.Printf("%s: Read(%d) @%d => %d %v \n%s\n", d.prefix, len(b), off, n, err, hex.Dump(b[:n])) + return n, err +} + +func Reader(reader io.Reader, prefix string) io.Reader { + return &ioReader{ + reader: reader, + prefix: prefix, + } +} + +type ioReader struct { + reader io.Reader + prefix string + offset int64 +} + +func (d *ioReader) Read(b []byte) (int, error) { + n, err := d.reader.Read(b) + fmt.Printf("%s: Read(%d) @%d => %d %v \n%s\n", d.prefix, len(b), d.offset, n, err, hex.Dump(b[:n])) + d.offset += int64(n) + return n, err +} + +func Writer(writer io.Writer, prefix string) io.Writer { + return &ioWriter{ + writer: writer, + prefix: prefix, + } +} + +type ioWriter struct { + writer io.Writer + prefix string + offset int64 +} + +func (d *ioWriter) Write(b []byte) (int, error) { + n, err := d.writer.Write(b) + fmt.Printf("%s: Write(%d) @%d => %d %v \n %q\n", d.prefix, len(b), d.offset, n, err, b[:n]) + d.offset += int64(n) + return n, err +} + +var ( + TRACEBUF int +) + +func init() { + for _, arg := range strings.Split(os.Getenv("PARQUETGODEBUG"), ",") { + k := arg + v := "" + i := strings.IndexByte(arg, '=') + if i >= 0 { + k, v = arg[:i], arg[i+1:] + } + var err error + switch k { + case "": + // ignore empty entries + case "tracebuf": + if TRACEBUF, err = strconv.Atoi(v); err != nil { + log.Printf("PARQUETGODEBUG: invalid value for tracebuf: %q", v) + } + default: + log.Printf("PARQUETGODEBUG: unrecognized debug option: %q", k) + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_off.go b/vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_off.go new file mode 100644 index 0000000000..acabe9cdf6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_off.go @@ -0,0 +1,6 @@ +//go:build debug + +package debug + +// SetFinalizer is a no-op when the debug tag is specified. +func SetFinalizer(interface{}, interface{}) {} diff --git a/vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_on.go b/vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_on.go new file mode 100644 index 0000000000..40edbe481c --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/debug/finalizer_on.go @@ -0,0 +1,7 @@ +//go:build !debug + +package debug + +import "runtime" + +func SetFinalizer(obj, finalizer any) { runtime.SetFinalizer(obj, finalizer) } diff --git a/vendor/github.com/parquet-go/parquet-go/internal/unsafecast/unsafecast.go b/vendor/github.com/parquet-go/parquet-go/internal/unsafecast/unsafecast.go new file mode 100644 index 0000000000..0838fd10e6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/internal/unsafecast/unsafecast.go @@ -0,0 +1,54 @@ +// Package unsafecast exposes functions to bypass the Go type system and perform +// conversions between types that would otherwise not be possible. +// +// The functions of this package are mostly useful as optimizations to avoid +// memory copies when converting between compatible memory layouts; for example, +// casting a [][16]byte to a []byte in order to use functions of the standard +// bytes package on the slices. +// +// With great power comes great responsibility. +package unsafecast + +import "unsafe" + +// The slice type represents the memory layout of slices in Go. It is similar to +// reflect.SliceHeader but uses a unsafe.Pointer instead of uintptr to for the +// backing array to allow the garbage collector to track track the reference. +type slice struct { + ptr unsafe.Pointer + len int + cap int +} + +// Slice converts the data slice of type []From to a slice of type []To sharing +// the same backing array. The length and capacity of the returned slice are +// scaled according to the size difference between the source and destination +// types. +// +// Note that the function does not perform any checks to ensure that the memory +// layouts of the types are compatible, it is possible to cause memory +// corruption if the layouts mismatch (e.g. the pointers in the From are +// different than the pointers in To). +func Slice[To, From any](data []From) []To { + // This function could use unsafe.Slice but it would drop the capacity + // information, so instead we implement the type conversion. + var zf From + var zt To + var s = slice{ + ptr: unsafe.Pointer(unsafe.SliceData(data)), + len: int((uintptr(len(data)) * unsafe.Sizeof(zf)) / unsafe.Sizeof(zt)), + cap: int((uintptr(cap(data)) * unsafe.Sizeof(zf)) / unsafe.Sizeof(zt)), + } + return *(*[]To)(unsafe.Pointer(&s)) +} + +// String converts a byte slice to a string value. The returned string shares +// the backing array of the byte slice. +// +// Programs using this function are responsible for ensuring that the data slice +// is not modified while the returned string is in use, otherwise the guarantee +// of immutability of Go string values will be violated, resulting in undefined +// behavior. +func String(data []byte) string { + return unsafe.String(unsafe.SliceData(data), len(data)) +} diff --git a/vendor/github.com/parquet-go/parquet-go/level.go b/vendor/github.com/parquet-go/parquet-go/level.go new file mode 100644 index 0000000000..0e2576fab5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/level.go @@ -0,0 +1,27 @@ +package parquet + +import "github.com/parquet-go/parquet-go/internal/bytealg" + +func countLevelsEqual(levels []byte, value byte) int { + return bytealg.Count(levels, value) +} + +func countLevelsNotEqual(levels []byte, value byte) int { + return len(levels) - countLevelsEqual(levels, value) +} + +func appendLevel(levels []byte, value byte, count int) []byte { + i := len(levels) + n := len(levels) + count + + if cap(levels) < n { + newLevels := make([]byte, n, 2*n) + copy(newLevels, levels) + levels = newLevels + } else { + levels = levels[:n] + } + + bytealg.Broadcast(levels[i:], value) + return levels +} diff --git a/vendor/github.com/parquet-go/parquet-go/limits.go b/vendor/github.com/parquet-go/parquet-go/limits.go new file mode 100644 index 0000000000..bf877565c9 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/limits.go @@ -0,0 +1,64 @@ +package parquet + +import ( + "fmt" + "math" +) + +const ( + // MaxColumnDepth is the maximum column depth supported by this package. + MaxColumnDepth = math.MaxInt8 + + // MaxColumnIndex is the maximum column index supported by this package. + MaxColumnIndex = math.MaxInt16 + + // MaxRepetitionLevel is the maximum repetition level supported by this + // package. + MaxRepetitionLevel = math.MaxUint8 + + // MaxDefinitionLevel is the maximum definition level supported by this + // package. + MaxDefinitionLevel = math.MaxUint8 + + // MaxRowGroups is the maximum number of row groups which can be contained + // in a single parquet file. + // + // This limit is enforced by the use of 16 bits signed integers in the file + // metadata footer of parquet files. It is part of the parquet specification + // and therefore cannot be changed. + MaxRowGroups = math.MaxInt16 +) + +const ( + estimatedSizeOfByteArrayValues = 20 +) + +func makeRepetitionLevel(i int) byte { + checkIndexRange("repetition level", i, 0, MaxRepetitionLevel) + return byte(i) +} + +func makeDefinitionLevel(i int) byte { + checkIndexRange("definition level", i, 0, MaxDefinitionLevel) + return byte(i) +} + +func makeColumnIndex(i int) int16 { + checkIndexRange("column index", i, 0, MaxColumnIndex) + return int16(i) +} + +func makeNumValues(i int) int32 { + checkIndexRange("number of values", i, 0, math.MaxInt32) + return int32(i) +} + +func checkIndexRange(typ string, i, min, max int) { + if i < min || i > max { + panic(errIndexOutOfRange(typ, i, min, max)) + } +} + +func errIndexOutOfRange(typ string, i, min, max int) error { + return fmt.Errorf("%s out of range: %d not in [%d:%d]", typ, i, min, max) +} diff --git a/vendor/github.com/parquet-go/parquet-go/merge.go b/vendor/github.com/parquet-go/parquet-go/merge.go new file mode 100644 index 0000000000..1655a8375a --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/merge.go @@ -0,0 +1,501 @@ +package parquet + +import ( + "container/heap" + "errors" + "fmt" + "io" + "sync" +) + +// MergeRowGroups constructs a row group which is a merged view of rowGroups. If +// rowGroups are sorted and the passed options include sorting, the merged row +// group will also be sorted. +// +// The function validates the input to ensure that the merge operation is +// possible, ensuring that the schemas match or can be converted to an +// optionally configured target schema passed as argument in the option list. +// +// The sorting columns of each row group are also consulted to determine whether +// the output can be represented. If sorting columns are configured on the merge +// they must be a prefix of sorting columns of all row groups being merged. +func MergeRowGroups(rowGroups []RowGroup, options ...RowGroupOption) (RowGroup, error) { + config, err := NewRowGroupConfig(options...) + if err != nil { + return nil, err + } + + schema := config.Schema + if len(rowGroups) == 0 { + return newEmptyRowGroup(schema), nil + } + if schema == nil { + schema = rowGroups[0].Schema() + + for _, rowGroup := range rowGroups[1:] { + if !EqualNodes(schema, rowGroup.Schema()) { + return nil, ErrRowGroupSchemaMismatch + } + } + } + + mergedRowGroups := make([]RowGroup, len(rowGroups)) + copy(mergedRowGroups, rowGroups) + + for i, rowGroup := range mergedRowGroups { + if rowGroupSchema := rowGroup.Schema(); !EqualNodes(schema, rowGroupSchema) { + conv, err := Convert(schema, rowGroupSchema) + if err != nil { + return nil, fmt.Errorf("cannot merge row groups: %w", err) + } + mergedRowGroups[i] = ConvertRowGroup(rowGroup, conv) + } + } + + m := &mergedRowGroup{sorting: config.Sorting.SortingColumns} + m.init(schema, mergedRowGroups) + + if len(m.sorting) == 0 { + // When the row group has no ordering, use a simpler version of the + // merger which simply concatenates rows from each of the row groups. + // This is preferable because it makes the output deterministic, the + // heap merge may otherwise reorder rows across groups. + return &m.multiRowGroup, nil + } + + for _, rowGroup := range m.rowGroups { + if !sortingColumnsHavePrefix(rowGroup.SortingColumns(), m.sorting) { + return nil, ErrRowGroupSortingColumnsMismatch + } + } + + m.compare = compareRowsFuncOf(schema, m.sorting) + return m, nil +} + +type mergedRowGroup struct { + multiRowGroup + sorting []SortingColumn + compare func(Row, Row) int +} + +func (m *mergedRowGroup) SortingColumns() []SortingColumn { + return m.sorting +} + +func (m *mergedRowGroup) Rows() Rows { + // The row group needs to respect a sorting order; the merged row reader + // uses a heap to merge rows from the row groups. + rows := make([]Rows, len(m.rowGroups)) + for i := range rows { + rows[i] = m.rowGroups[i].Rows() + } + return &mergedRowGroupRows{ + merge: mergedRowReader{ + compare: m.compare, + readers: makeBufferedRowReaders(len(rows), func(i int) RowReader { return rows[i] }), + }, + rows: rows, + schema: m.schema, + } +} + +type mergedRowGroupRows struct { + merge mergedRowReader + rowIndex int64 + seekToRow int64 + rows []Rows + schema *Schema +} + +func (r *mergedRowGroupRows) WriteRowsTo(w RowWriter) (n int64, err error) { + b := newMergeBuffer() + b.setup(r.rows, r.merge.compare) + n, err = b.WriteRowsTo(w) + r.rowIndex += int64(n) + b.release() + return +} + +func (r *mergedRowGroupRows) readInternal(rows []Row) (int, error) { + n, err := r.merge.ReadRows(rows) + r.rowIndex += int64(n) + return n, err +} + +func (r *mergedRowGroupRows) Close() (lastErr error) { + r.merge.close() + r.rowIndex = 0 + r.seekToRow = 0 + + for _, rows := range r.rows { + if err := rows.Close(); err != nil { + lastErr = err + } + } + + return lastErr +} + +func (r *mergedRowGroupRows) ReadRows(rows []Row) (int, error) { + for r.rowIndex < r.seekToRow { + n := min(int(r.seekToRow-r.rowIndex), len(rows)) + n, err := r.readInternal(rows[:n]) + if err != nil { + return 0, err + } + } + + return r.readInternal(rows) +} + +func (r *mergedRowGroupRows) SeekToRow(rowIndex int64) error { + if rowIndex >= r.rowIndex { + r.seekToRow = rowIndex + return nil + } + return fmt.Errorf("SeekToRow: merged row reader cannot seek backward from row %d to %d", r.rowIndex, rowIndex) +} + +func (r *mergedRowGroupRows) Schema() *Schema { + return r.schema +} + +// MergeRowReader constructs a RowReader which creates an ordered sequence of +// all the readers using the given compare function as the ordering predicate. +func MergeRowReaders(readers []RowReader, compare func(Row, Row) int) RowReader { + return &mergedRowReader{ + compare: compare, + readers: makeBufferedRowReaders(len(readers), func(i int) RowReader { return readers[i] }), + } +} + +func makeBufferedRowReaders(numReaders int, readerAt func(int) RowReader) []*bufferedRowReader { + buffers := make([]bufferedRowReader, numReaders) + readers := make([]*bufferedRowReader, numReaders) + + for i := range readers { + buffers[i].rows = readerAt(i) + readers[i] = &buffers[i] + } + + return readers +} + +type mergedRowReader struct { + compare func(Row, Row) int + readers []*bufferedRowReader + initialized bool +} + +func (m *mergedRowReader) initialize() error { + for i, r := range m.readers { + switch err := r.read(); err { + case nil: + case io.EOF: + m.readers[i] = nil + default: + m.readers = nil + return err + } + } + + n := 0 + for _, r := range m.readers { + if r != nil { + m.readers[n] = r + n++ + } + } + + clear := m.readers[n:] + for i := range clear { + clear[i] = nil + } + + m.readers = m.readers[:n] + heap.Init(m) + return nil +} + +func (m *mergedRowReader) close() { + for _, r := range m.readers { + r.close() + } + m.readers = nil +} + +func (m *mergedRowReader) ReadRows(rows []Row) (n int, err error) { + if !m.initialized { + m.initialized = true + + if err := m.initialize(); err != nil { + return 0, err + } + } + + for n < len(rows) && len(m.readers) != 0 { + r := m.readers[0] + if r.end == r.off { // This readers buffer has been exhausted, repopulate it. + if err := r.read(); err != nil { + if err == io.EOF { + heap.Pop(m) + continue + } + return n, err + } else { + heap.Fix(m, 0) + continue + } + } + + rows[n] = append(rows[n][:0], r.head()...) + n++ + + if err := r.next(); err != nil { + if err != io.EOF { + return n, err + } + return n, nil + } else { + heap.Fix(m, 0) + } + } + + if len(m.readers) == 0 { + err = io.EOF + } + + return n, err +} + +func (m *mergedRowReader) Less(i, j int) bool { + return m.compare(m.readers[i].head(), m.readers[j].head()) < 0 +} + +func (m *mergedRowReader) Len() int { + return len(m.readers) +} + +func (m *mergedRowReader) Swap(i, j int) { + m.readers[i], m.readers[j] = m.readers[j], m.readers[i] +} + +func (m *mergedRowReader) Push(x any) { + panic("NOT IMPLEMENTED") +} + +func (m *mergedRowReader) Pop() any { + i := len(m.readers) - 1 + r := m.readers[i] + m.readers = m.readers[:i] + return r +} + +type bufferedRowReader struct { + rows RowReader + off int32 + end int32 + buf [10]Row +} + +func (r *bufferedRowReader) head() Row { + return r.buf[r.off] +} + +func (r *bufferedRowReader) next() error { + if r.off++; r.off == r.end { + r.off = 0 + r.end = 0 + // We need to read more rows, however it is unsafe to do so here because we haven't + // returned the current rows to the caller yet which may cause buffer corruption. + return io.EOF + } + return nil +} + +func (r *bufferedRowReader) read() error { + if r.rows == nil { + return io.EOF + } + n, err := r.rows.ReadRows(r.buf[r.end:]) + if err != nil && n == 0 { + return err + } + r.end += int32(n) + return nil +} + +func (r *bufferedRowReader) close() { + r.rows = nil + r.off = 0 + r.end = 0 +} + +type mergeBuffer struct { + compare func(Row, Row) int + rows []Rows + buffer [][]Row + head []int + len int + copy [mergeBufferSize]Row +} + +const mergeBufferSize = 1 << 10 + +func newMergeBuffer() *mergeBuffer { + return mergeBufferPool.Get().(*mergeBuffer) +} + +var mergeBufferPool = &sync.Pool{ + New: func() any { + return new(mergeBuffer) + }, +} + +func (m *mergeBuffer) setup(rows []Rows, compare func(Row, Row) int) { + m.compare = compare + m.rows = append(m.rows, rows...) + size := len(rows) + if len(m.buffer) < size { + extra := size - len(m.buffer) + b := make([][]Row, extra) + for i := range b { + b[i] = make([]Row, 0, mergeBufferSize) + } + m.buffer = append(m.buffer, b...) + m.head = append(m.head, make([]int, extra)...) + } + m.len = size +} + +func (m *mergeBuffer) reset() { + for i := range m.rows { + m.buffer[i] = m.buffer[i][:0] + m.head[i] = 0 + } + m.rows = m.rows[:0] + m.compare = nil + for i := range m.copy { + m.copy[i] = nil + } + m.len = 0 +} + +func (m *mergeBuffer) release() { + m.reset() + mergeBufferPool.Put(m) +} + +func (m *mergeBuffer) fill() error { + m.len = len(m.rows) + for i := range m.rows { + if m.head[i] < len(m.buffer[i]) { + // There is still rows data in m.buffer[i]. Skip filling the row buffer until + // all rows have been read. + continue + } + m.head[i] = 0 + m.buffer[i] = m.buffer[i][:mergeBufferSize] + n, err := m.rows[i].ReadRows(m.buffer[i]) + if err != nil { + if !errors.Is(err, io.EOF) { + return err + } + } + m.buffer[i] = m.buffer[i][:n] + } + heap.Init(m) + return nil +} + +func (m *mergeBuffer) Less(i, j int) bool { + x := m.buffer[i] + if len(x) == 0 { + return false + } + y := m.buffer[j] + if len(y) == 0 { + return true + } + return m.compare(x[m.head[i]], y[m.head[j]]) == -1 +} + +func (m *mergeBuffer) Pop() any { + m.len-- + // We don't use the popped value. + return nil +} + +func (m *mergeBuffer) Len() int { + return m.len +} + +func (m *mergeBuffer) Swap(i, j int) { + m.buffer[i], m.buffer[j] = m.buffer[j], m.buffer[i] + m.head[i], m.head[j] = m.head[j], m.head[i] +} + +func (m *mergeBuffer) Push(x any) { + panic("NOT IMPLEMENTED") +} + +func (m *mergeBuffer) WriteRowsTo(w RowWriter) (n int64, err error) { + err = m.fill() + if err != nil { + return 0, err + } + var count int + for m.left() { + size := m.read() + if size == 0 { + break + } + count, err = w.WriteRows(m.copy[:size]) + if err != nil { + return + } + n += int64(count) + err = m.fill() + if err != nil { + return + } + } + return +} + +func (m *mergeBuffer) left() bool { + for i := range m.len { + if m.head[i] < len(m.buffer[i]) { + return true + } + } + return false +} + +func (m *mergeBuffer) read() (n int64) { + for n < int64(len(m.copy)) && m.Len() != 0 { + r := m.buffer[:m.len][0] + if len(r) == 0 { + heap.Pop(m) + continue + } + m.copy[n] = append(m.copy[n][:0], r[m.head[0]]...) + m.head[0]++ + n++ + if m.head[0] < len(r) { + // There is still rows in this row group. Adjust the heap + heap.Fix(m, 0) + } else { + heap.Pop(m) + } + } + return +} + +var ( + _ RowReaderWithSchema = (*mergedRowGroupRows)(nil) + _ RowWriterTo = (*mergedRowGroupRows)(nil) + _ heap.Interface = (*mergeBuffer)(nil) + _ RowWriterTo = (*mergeBuffer)(nil) +) diff --git a/vendor/github.com/parquet-go/parquet-go/multi_row_group.go b/vendor/github.com/parquet-go/parquet-go/multi_row_group.go new file mode 100644 index 0000000000..d37767fd56 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/multi_row_group.go @@ -0,0 +1,283 @@ +package parquet + +import ( + "io" +) + +// MultiRowGroup wraps multiple row groups to appear as if it was a single +// RowGroup. RowGroups must have the same schema or it will error. +func MultiRowGroup(rowGroups ...RowGroup) RowGroup { + if len(rowGroups) == 0 { + return &emptyRowGroup{} + } + if len(rowGroups) == 1 { + return rowGroups[0] + } + + schema, err := compatibleSchemaOf(rowGroups) + if err != nil { + panic(err) + } + + rowGroupsCopy := make([]RowGroup, len(rowGroups)) + copy(rowGroupsCopy, rowGroups) + + c := new(multiRowGroup) + c.init(schema, rowGroupsCopy) + return c +} + +func (c *multiRowGroup) init(schema *Schema, rowGroups []RowGroup) error { + columns := make([]multiColumnChunk, len(schema.Columns())) + + rowGroupColumnChunks := make([][]ColumnChunk, len(rowGroups)) + for i, rowGroup := range rowGroups { + rowGroupColumnChunks[i] = rowGroup.ColumnChunks() + } + + for i := range columns { + columns[i].rowGroup = c + columns[i].column = i + columns[i].chunks = make([]ColumnChunk, len(rowGroupColumnChunks)) + + for j, columnChunks := range rowGroupColumnChunks { + columns[i].chunks[j] = columnChunks[i] + } + } + + c.schema = schema + c.rowGroups = rowGroups + c.columns = make([]ColumnChunk, len(columns)) + + for i := range columns { + c.columns[i] = &columns[i] + } + + return nil +} + +func compatibleSchemaOf(rowGroups []RowGroup) (*Schema, error) { + schema := rowGroups[0].Schema() + + // Fast path: Many times all row groups have the exact same schema so a + // pointer comparison is cheaper. + samePointer := true + for _, rowGroup := range rowGroups[1:] { + if rowGroup.Schema() != schema { + samePointer = false + break + } + } + if samePointer { + return schema, nil + } + + // Slow path: The schema pointers are not the same, but they still have to + // be compatible. + for _, rowGroup := range rowGroups[1:] { + if !EqualNodes(schema, rowGroup.Schema()) { + return nil, ErrRowGroupSchemaMismatch + } + } + + return schema, nil +} + +type multiRowGroup struct { + schema *Schema + rowGroups []RowGroup + columns []ColumnChunk +} + +func (c *multiRowGroup) NumRows() (numRows int64) { + for _, rowGroup := range c.rowGroups { + numRows += rowGroup.NumRows() + } + return numRows +} + +func (c *multiRowGroup) ColumnChunks() []ColumnChunk { return c.columns } + +func (c *multiRowGroup) SortingColumns() []SortingColumn { return nil } + +func (c *multiRowGroup) Schema() *Schema { return c.schema } + +func (c *multiRowGroup) Rows() Rows { return NewRowGroupRowReader(c) } + +type multiColumnChunk struct { + rowGroup *multiRowGroup + column int + chunks []ColumnChunk +} + +func (c *multiColumnChunk) Type() Type { + if len(c.chunks) != 0 { + return c.chunks[0].Type() // all chunks should be of the same type + } + return nil +} + +func (c *multiColumnChunk) NumValues() int64 { + n := int64(0) + for i := range c.chunks { + n += c.chunks[i].NumValues() + } + return n +} + +func (c *multiColumnChunk) Column() int { + return c.column +} + +func (c *multiColumnChunk) Pages() Pages { + return &multiPages{column: c} +} + +func (c *multiColumnChunk) ColumnIndex() (ColumnIndex, error) { + // TODO: implement + return nil, nil +} + +func (c *multiColumnChunk) OffsetIndex() (OffsetIndex, error) { + // TODO: implement + return nil, nil +} + +func (c *multiColumnChunk) BloomFilter() BloomFilter { + return multiBloomFilter{c} +} + +type multiBloomFilter struct{ *multiColumnChunk } + +func (f multiBloomFilter) ReadAt(b []byte, off int64) (int, error) { + // TODO: add a test for this function + i := 0 + + for i < len(f.chunks) { + if r := f.chunks[i].BloomFilter(); r != nil { + size := r.Size() + if off < size { + break + } + off -= size + } + i++ + } + + if i == len(f.chunks) { + return 0, io.EOF + } + + rn := int(0) + for len(b) > 0 { + if r := f.chunks[i].BloomFilter(); r != nil { + n, err := r.ReadAt(b, off) + rn += n + if err != nil { + return rn, err + } + if b = b[n:]; len(b) == 0 { + return rn, nil + } + off += int64(n) + } + i++ + } + + if i == len(f.chunks) { + return rn, io.EOF + } + return rn, nil +} + +func (f multiBloomFilter) Size() int64 { + size := int64(0) + for _, c := range f.chunks { + if b := c.BloomFilter(); b != nil { + size += b.Size() + } + } + return size +} + +func (f multiBloomFilter) Check(v Value) (bool, error) { + for _, c := range f.chunks { + if b := c.BloomFilter(); b != nil { + if ok, err := b.Check(v); ok || err != nil { + return ok, err + } + } + } + return false, nil +} + +type multiPages struct { + pages Pages + index int + column *multiColumnChunk +} + +func (m *multiPages) ReadPage() (Page, error) { + for { + if m.pages != nil { + p, err := m.pages.ReadPage() + if err == nil || err != io.EOF { + return p, err + } + if err := m.pages.Close(); err != nil { + return nil, err + } + m.pages = nil + } + + if m.column == nil || m.index == len(m.column.chunks) { + return nil, io.EOF + } + + m.pages = m.column.chunks[m.index].Pages() + m.index++ + } +} + +func (m *multiPages) SeekToRow(rowIndex int64) error { + if m.column == nil { + return io.ErrClosedPipe + } + + if m.pages != nil { + if err := m.pages.Close(); err != nil { + return err + } + } + + rowGroups := m.column.rowGroup.rowGroups + numRows := int64(0) + m.pages = nil + m.index = 0 + + for m.index < len(rowGroups) { + numRows = rowGroups[m.index].NumRows() + if rowIndex < numRows { + break + } + rowIndex -= numRows + m.index++ + } + + if m.index < len(rowGroups) { + m.pages = m.column.chunks[m.index].Pages() + m.index++ + return m.pages.SeekToRow(rowIndex) + } + return nil +} + +func (m *multiPages) Close() (err error) { + if m.pages != nil { + err = m.pages.Close() + } + m.pages = nil + m.index = 0 + m.column = nil + return err +} diff --git a/vendor/github.com/parquet-go/parquet-go/node.go b/vendor/github.com/parquet-go/parquet-go/node.go new file mode 100644 index 0000000000..25603a11f4 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/node.go @@ -0,0 +1,546 @@ +package parquet + +import ( + "reflect" + "slices" + "strings" + "unicode" + "unicode/utf8" + + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" +) + +// Node values represent nodes of a parquet schema. +// +// Nodes carry the type of values, as well as properties like whether the values +// are optional or repeat. Nodes with one or more children represent parquet +// groups and therefore do not have a logical type. +// +// Nodes are immutable values and therefore safe to use concurrently from +// multiple goroutines. +type Node interface { + // The id of this node in its parent node. Zero value is treated as id is not + // set. ID only needs to be unique within its parent context. + // + // This is the same as parquet field_id + ID() int + + // Returns a human-readable representation of the parquet node. + String() string + + // For leaf nodes, returns the type of values of the parquet column. + // + // Calling this method on non-leaf nodes will panic. + Type() Type + + // Returns whether the parquet column is optional. + Optional() bool + + // Returns whether the parquet column is repeated. + Repeated() bool + + // Returns whether the parquet column is required. + Required() bool + + // Returns true if this a leaf node. + Leaf() bool + + // Returns a mapping of the node's fields. + // + // As an optimization, the same slices may be returned by multiple calls to + // this method, programs must treat the returned values as immutable. + // + // This method returns an empty mapping when called on leaf nodes. + Fields() []Field + + // Returns the encoding used by the node. + // + // The method may return nil to indicate that no specific encoding was + // configured on the node, in which case a default encoding might be used. + Encoding() encoding.Encoding + + // Returns compression codec used by the node. + // + // The method may return nil to indicate that no specific compression codec + // was configured on the node, in which case a default compression might be + // used. + Compression() compress.Codec + + // Returns the Go type that best represents the parquet node. + // + // For leaf nodes, this will be one of bool, int32, int64, deprecated.Int96, + // float32, float64, string, []byte, or [N]byte. + // + // For groups, the method returns a struct type. + // + // If the method is called on a repeated node, the method returns a slice of + // the underlying type. + // + // For optional nodes, the method returns a pointer of the underlying type. + // + // For nodes that were constructed from Go values (e.g. using SchemaOf), the + // method returns the original Go type. + GoType() reflect.Type +} + +// Field instances represent fields of a parquet node, which associate a node to +// their name in their parent node. +type Field interface { + Node + + // Returns the name of this field in its parent node. + Name() string + + // Given a reference to the Go value matching the structure of the parent + // node, returns the Go value of the field. + Value(base reflect.Value) reflect.Value +} + +// Encoded wraps the node passed as argument to use the given encoding. +// +// The function panics if it is called on a non-leaf node, or if the +// encoding does not support the node type. +func Encoded(node Node, encoding encoding.Encoding) Node { + if !node.Leaf() { + panic("cannot add encoding to a non-leaf node") + } + if encoding != nil { + kind := node.Type().Kind() + if !canEncode(encoding, kind) { + panic("cannot apply " + encoding.Encoding().String() + " to node of type " + kind.String()) + } + } + return &encodedNode{ + Node: node, + encoding: encoding, + } +} + +type encodedNode struct { + Node + encoding encoding.Encoding +} + +func (n *encodedNode) Encoding() encoding.Encoding { + return n.encoding +} + +// Compressed wraps the node passed as argument to use the given compression +// codec. +// +// If the codec is nil, the node's compression is left unchanged. +// +// The function panics if it is called on a non-leaf node. +func Compressed(node Node, codec compress.Codec) Node { + if !node.Leaf() { + panic("cannot add compression codec to a non-leaf node") + } + return &compressedNode{ + Node: node, + codec: codec, + } +} + +type compressedNode struct { + Node + codec compress.Codec +} + +func (n *compressedNode) Compression() compress.Codec { + return n.codec +} + +// Optional wraps the given node to make it optional. +func Optional(node Node) Node { return &optionalNode{node} } + +type optionalNode struct{ Node } + +func (opt *optionalNode) Optional() bool { return true } +func (opt *optionalNode) Repeated() bool { return false } +func (opt *optionalNode) Required() bool { return false } +func (opt *optionalNode) GoType() reflect.Type { return reflect.PtrTo(opt.Node.GoType()) } + +// FieldID wraps a node to provide node field id +func FieldID(node Node, id int) Node { return &fieldIDNode{Node: node, id: id} } + +type fieldIDNode struct { + Node + id int +} + +func (f *fieldIDNode) ID() int { return f.id } + +// Repeated wraps the given node to make it repeated. +func Repeated(node Node) Node { return &repeatedNode{node} } + +type repeatedNode struct{ Node } + +func (rep *repeatedNode) Optional() bool { return false } +func (rep *repeatedNode) Repeated() bool { return true } +func (rep *repeatedNode) Required() bool { return false } +func (rep *repeatedNode) GoType() reflect.Type { return reflect.SliceOf(rep.Node.GoType()) } + +// Required wraps the given node to make it required. +func Required(node Node) Node { return &requiredNode{node} } + +type requiredNode struct{ Node } + +func (req *requiredNode) Optional() bool { return false } +func (req *requiredNode) Repeated() bool { return false } +func (req *requiredNode) Required() bool { return true } +func (req *requiredNode) GoType() reflect.Type { return req.Node.GoType() } + +type node struct{} + +// Leaf returns a leaf node of the given type. +func Leaf(typ Type) Node { + return &leafNode{typ: typ} +} + +type leafNode struct{ typ Type } + +func (n *leafNode) ID() int { return 0 } + +func (n *leafNode) String() string { return sprint("", n) } + +func (n *leafNode) Type() Type { return n.typ } + +func (n *leafNode) Optional() bool { return false } + +func (n *leafNode) Repeated() bool { return false } + +func (n *leafNode) Required() bool { return true } + +func (n *leafNode) Leaf() bool { return true } + +func (n *leafNode) Fields() []Field { return nil } + +func (n *leafNode) Encoding() encoding.Encoding { return nil } + +func (n *leafNode) Compression() compress.Codec { return nil } + +func (n *leafNode) GoType() reflect.Type { return goTypeOfLeaf(n) } + +var repetitionTypes = [...]format.FieldRepetitionType{ + 0: format.Required, + 1: format.Optional, + 2: format.Repeated, +} + +func fieldRepetitionTypePtrOf(node Node) *format.FieldRepetitionType { + switch { + case node.Required(): + return &repetitionTypes[format.Required] + case node.Optional(): + return &repetitionTypes[format.Optional] + case node.Repeated(): + return &repetitionTypes[format.Repeated] + default: + return nil + } +} + +func fieldRepetitionTypeOf(node Node) format.FieldRepetitionType { + switch { + case node.Optional(): + return format.Optional + case node.Repeated(): + return format.Repeated + default: + return format.Required + } +} + +func applyFieldRepetitionType(t format.FieldRepetitionType, repetitionLevel, definitionLevel byte) (byte, byte) { + switch t { + case format.Optional: + definitionLevel++ + case format.Repeated: + repetitionLevel++ + definitionLevel++ + } + return repetitionLevel, definitionLevel +} + +type Group map[string]Node + +func (g Group) ID() int { return 0 } + +func (g Group) String() string { return sprint("", g) } + +func (g Group) Type() Type { return groupType{} } + +func (g Group) Optional() bool { return false } + +func (g Group) Repeated() bool { return false } + +func (g Group) Required() bool { return true } + +func (g Group) Leaf() bool { return false } + +func (g Group) Fields() []Field { + groupFields := make([]groupField, 0, len(g)) + for name, node := range g { + groupFields = append(groupFields, groupField{ + Node: node, + name: name, + }) + } + slices.SortFunc(groupFields, func(a, b groupField) int { + return strings.Compare(a.name, b.name) + }) + fields := make([]Field, len(groupFields)) + for i := range groupFields { + fields[i] = &groupFields[i] + } + return fields +} + +func (g Group) Encoding() encoding.Encoding { return nil } + +func (g Group) Compression() compress.Codec { return nil } + +func (g Group) GoType() reflect.Type { return goTypeOfGroup(g) } + +type groupField struct { + Node + name string +} + +func (f *groupField) Name() string { return f.name } + +func (f *groupField) Value(base reflect.Value) reflect.Value { + if base.Kind() == reflect.Interface { + if base.IsNil() { + return reflect.ValueOf(nil) + } + if base = base.Elem(); base.Kind() == reflect.Pointer && base.IsNil() { + return reflect.ValueOf(nil) + } + } + return base.MapIndex(reflect.ValueOf(&f.name).Elem()) +} + +func goTypeOf(node Node) reflect.Type { + switch { + case node.Optional(): + return goTypeOfOptional(node) + case node.Repeated(): + return goTypeOfRepeated(node) + default: + return goTypeOfRequired(node) + } +} + +func goTypeOfOptional(node Node) reflect.Type { + return reflect.PtrTo(goTypeOfRequired(node)) +} + +func goTypeOfRepeated(node Node) reflect.Type { + return reflect.SliceOf(goTypeOfRequired(node)) +} + +func goTypeOfRequired(node Node) reflect.Type { + if node.Leaf() { + return goTypeOfLeaf(node) + } else { + return goTypeOfGroup(node) + } +} + +func goTypeOfLeaf(node Node) reflect.Type { + t := node.Type() + if convertibleType, ok := t.(interface{ GoType() reflect.Type }); ok { + return convertibleType.GoType() + } + switch t.Kind() { + case Boolean: + return reflect.TypeOf(false) + case Int32: + return reflect.TypeOf(int32(0)) + case Int64: + return reflect.TypeOf(int64(0)) + case Int96: + return reflect.TypeOf(deprecated.Int96{}) + case Float: + return reflect.TypeOf(float32(0)) + case Double: + return reflect.TypeOf(float64(0)) + case ByteArray: + return reflect.TypeOf(([]byte)(nil)) + case FixedLenByteArray: + return reflect.ArrayOf(t.Length(), reflect.TypeOf(byte(0))) + default: + panic("BUG: parquet type returned an unsupported kind") + } +} + +func goTypeOfGroup(node Node) reflect.Type { + fields := node.Fields() + structFields := make([]reflect.StructField, len(fields)) + for i, field := range fields { + structFields[i].Name = exportedStructFieldName(field.Name()) + structFields[i].Type = field.GoType() + // TODO: can we reconstruct a struct tag that would be valid if a value + // of this type were passed to SchemaOf? + } + return reflect.StructOf(structFields) +} + +func exportedStructFieldName(name string) string { + firstRune, size := utf8.DecodeRuneInString(name) + return string([]rune{unicode.ToUpper(firstRune)}) + name[size:] +} + +func isList(node Node) bool { + logicalType := node.Type().LogicalType() + return logicalType != nil && logicalType.List != nil +} + +func isMap(node Node) bool { + logicalType := node.Type().LogicalType() + return logicalType != nil && logicalType.Map != nil +} + +func numLeafColumnsOf(node Node) int16 { + return makeColumnIndex(numLeafColumns(node, 0)) +} + +func numLeafColumns(node Node, columnIndex int) int { + if node.Leaf() { + return columnIndex + 1 + } + for _, field := range node.Fields() { + columnIndex = numLeafColumns(field, columnIndex) + } + return columnIndex +} + +func listElementOf(node Node) Node { + if !node.Leaf() { + if list := fieldByName(node, "list"); list != nil { + if elem := fieldByName(list, "element"); elem != nil { + return elem + } + // TODO: It should not be named "item", but some versions of pyarrow + // and some versions of polars used that instead of "element". + // https://issues.apache.org/jira/browse/ARROW-11497 + // https://github.com/pola-rs/polars/issues/17100 + if elem := fieldByName(list, "item"); elem != nil { + return elem + } + } + } + panic("node with logical type LIST is not composed of a repeated .list.element") +} + +func mapKeyValueOf(node Node) Node { + if !node.Leaf() && (node.Required() || node.Optional()) { + for _, kv_name := range []string{"key_value", "map"} { + if keyValue := fieldByName(node, kv_name); keyValue != nil && !keyValue.Leaf() && keyValue.Repeated() { + k := fieldByName(keyValue, "key") + v := fieldByName(keyValue, "value") + if k != nil && v != nil && k.Required() { + return keyValue + } + } + } + } + panic("node with logical type MAP is not composed of a repeated .key_value group (or .map group) with key and value fields") +} + +func encodingOf(node Node) encoding.Encoding { + encoding := node.Encoding() + // The parquet-format documentation states that the + // DELTA_LENGTH_BYTE_ARRAY is always preferred to PLAIN when + // encoding BYTE_ARRAY values. We apply it as a default if + // none were explicitly specified, which gives the application + // the opportunity to override this behavior if needed. + // + // https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 + if node.Type().Kind() == ByteArray && encoding == nil { + encoding = &DeltaLengthByteArray + } + if encoding == nil { + encoding = &Plain + } + return encoding +} + +func forEachNodeOf(name string, node Node, do func(string, Node)) { + do(name, node) + + for _, f := range node.Fields() { + forEachNodeOf(f.Name(), f, do) + } +} + +func fieldByName(node Node, name string) Field { + for _, f := range node.Fields() { + if f.Name() == name { + return f + } + } + return nil +} + +// EqualNodes returns true if node1 and node2 are equal. +// +// Nodes that are not of the same repetition type (optional, required, repeated) or +// of the same hierarchical type (leaf, group) are considered not equal. +// Leaf nodes are considered equal if they are of the same data type. +// Group nodes are considered equal if their fields have the same names and are recursively equal. +// +// Note that the encoding and compression of the nodes are not considered by this function. +func EqualNodes(node1, node2 Node) bool { + if node1.Leaf() { + return node2.Leaf() && leafNodesAreEqual(node1, node2) + } else { + return !node2.Leaf() && groupNodesAreEqual(node1, node2) + } +} + +func typesAreEqual(type1, type2 Type) bool { + return type1.Kind() == type2.Kind() && + type1.Length() == type2.Length() && + reflect.DeepEqual(type1.LogicalType(), type2.LogicalType()) +} + +func repetitionsAreEqual(node1, node2 Node) bool { + return node1.Optional() == node2.Optional() && node1.Repeated() == node2.Repeated() +} + +func leafNodesAreEqual(node1, node2 Node) bool { + return typesAreEqual(node1.Type(), node2.Type()) && repetitionsAreEqual(node1, node2) +} + +func groupNodesAreEqual(node1, node2 Node) bool { + fields1 := node1.Fields() + fields2 := node2.Fields() + + if len(fields1) != len(fields2) { + return false + } + + if !repetitionsAreEqual(node1, node2) { + return false + } + + for i := range fields1 { + f1 := fields1[i] + f2 := fields2[i] + + if f1.Name() != f2.Name() { + return false + } + + if !EqualNodes(f1, f2) { + return false + } + } + + return true +} diff --git a/vendor/github.com/parquet-go/parquet-go/null.go b/vendor/github.com/parquet-go/parquet-go/null.go new file mode 100644 index 0000000000..6588580934 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/null.go @@ -0,0 +1,113 @@ +package parquet + +import ( + "reflect" + "unsafe" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/internal/bytealg" + "github.com/parquet-go/parquet-go/internal/unsafecast" + "github.com/parquet-go/parquet-go/sparse" +) + +// nullIndexFunc is the type of functions used to detect null values in rows. +// +// For each value of the rows array, the bitmap passed as first argument is +// populated to indicate whether the values were null (0) or not (1). +// +// The function writes one bit to the output buffer for each row in the input, +// the buffer must be sized accordingly. +type nullIndexFunc func(bits []uint64, rows sparse.Array) + +func nullIndex[T comparable](bits []uint64, rows sparse.Array) { + var zero T + for i := range rows.Len() { + v := *(*T)(rows.Index(i)) + if v != zero { + x := uint(i) / 64 + y := uint(i) % 64 + bits[x] |= 1 << y + } + } +} + +func nullIndexStruct(bits []uint64, rows sparse.Array) { + bytealg.Broadcast(unsafecast.Slice[byte](bits), 0xFF) +} + +func nullIndexFuncOf(t reflect.Type) nullIndexFunc { + switch t { + case reflect.TypeOf(deprecated.Int96{}): + return nullIndex[deprecated.Int96] + } + + switch t.Kind() { + case reflect.Bool: + return nullIndexBool + + case reflect.Int: + return nullIndexInt + + case reflect.Int32: + return nullIndexInt32 + + case reflect.Int64: + return nullIndexInt64 + + case reflect.Uint: + return nullIndexUint + + case reflect.Uint32: + return nullIndexUint32 + + case reflect.Uint64: + return nullIndexUint64 + + case reflect.Float32: + return nullIndexFloat32 + + case reflect.Float64: + return nullIndexFloat64 + + case reflect.String: + return nullIndexString + + case reflect.Slice: + return nullIndexSlice + + case reflect.Map: + return nullIndexPointer + + case reflect.Array: + if t.Elem().Kind() == reflect.Uint8 { + switch size := t.Len(); size { + case 16: + return nullIndexUint128 + default: + return nullIndexFuncOfByteArray(size) + } + } + + case reflect.Pointer: + return nullIndexPointer + + case reflect.Struct: + return nullIndexStruct + } + + panic("cannot convert Go values of type " + typeNameOf(t) + " to parquet value") +} + +func nullIndexFuncOfByteArray(n int) nullIndexFunc { + return func(bits []uint64, rows sparse.Array) { + for i := range rows.Len() { + p := (*byte)(rows.Index(i)) + b := unsafe.Slice(p, n) + if !isZero(b) { + x := uint(i) / 64 + y := uint(i) % 64 + bits[x] |= 1 << y + } + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/null_amd64.go b/vendor/github.com/parquet-go/parquet-go/null_amd64.go new file mode 100644 index 0000000000..aa0cbfffac --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/null_amd64.go @@ -0,0 +1,74 @@ +//go:build !purego + +package parquet + +import "github.com/parquet-go/parquet-go/sparse" + +//go:noescape +func nullIndex8(bits *uint64, rows sparse.Array) + +//go:noescape +func nullIndex32(bits *uint64, rows sparse.Array) + +//go:noescape +func nullIndex64(bits *uint64, rows sparse.Array) + +//go:noescape +func nullIndex128(bits *uint64, rows sparse.Array) + +func nullIndexBool(bits []uint64, rows sparse.Array) { + nullIndex8(&bits[0], rows) +} + +func nullIndexInt(bits []uint64, rows sparse.Array) { + nullIndex64(&bits[0], rows) +} + +func nullIndexInt32(bits []uint64, rows sparse.Array) { + nullIndex32(&bits[0], rows) +} + +func nullIndexInt64(bits []uint64, rows sparse.Array) { + nullIndex64(&bits[0], rows) +} + +func nullIndexUint(bits []uint64, rows sparse.Array) { + nullIndex64(&bits[0], rows) +} + +func nullIndexUint32(bits []uint64, rows sparse.Array) { + nullIndex32(&bits[0], rows) +} + +func nullIndexUint64(bits []uint64, rows sparse.Array) { + nullIndex64(&bits[0], rows) +} + +func nullIndexUint128(bits []uint64, rows sparse.Array) { + nullIndex128(&bits[0], rows) +} + +func nullIndexFloat32(bits []uint64, rows sparse.Array) { + nullIndex32(&bits[0], rows) +} + +func nullIndexFloat64(bits []uint64, rows sparse.Array) { + nullIndex64(&bits[0], rows) +} + +func nullIndexString(bits []uint64, rows sparse.Array) { + // We offset by an extra 8 bytes to test the lengths of string values where + // the first field is the pointer and the second is the length which we want + // to test. + nullIndex64(&bits[0], rows.Offset(8)) +} + +func nullIndexSlice(bits []uint64, rows sparse.Array) { + // Slice values are null if their pointer is nil, which is held in the first + // 8 bytes of the object so we can simply test 64 bits words. + nullIndex64(&bits[0], rows) +} + +func nullIndexPointer(bits []uint64, rows sparse.Array) { + nullIndex64(&bits[0], rows) +} diff --git a/vendor/github.com/parquet-go/parquet-go/null_amd64.s b/vendor/github.com/parquet-go/parquet-go/null_amd64.s new file mode 100644 index 0000000000..52804d3f12 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/null_amd64.s @@ -0,0 +1,227 @@ +//go:build !purego + +#include "textflag.h" + +// func nullIndex8(bits *uint64, rows sparse.Array) +TEXT ยทnullIndex8(SB), NOSPLIT, $0-32 + MOVQ bits+0(FP), AX + MOVQ rows_array_ptr+8(FP), BX + MOVQ rows_array_len+16(FP), DI + MOVQ rows_array_off+24(FP), DX + + MOVQ $1, CX + XORQ SI, SI + + CMPQ DI, $0 + JE done +loop1x1: + XORQ R8, R8 + MOVB (BX), R9 + CMPB R9, $0 + JE next1x1 + + MOVQ SI, R10 + SHRQ $6, R10 + ORQ CX, (AX)(R10*8) +next1x1: + ADDQ DX, BX + ROLQ $1, CX + INCQ SI + CMPQ SI, DI + JNE loop1x1 +done: + RET + +// func nullIndex32(bits *uint64, rows sparse.Array) +TEXT ยทnullIndex32(SB), NOSPLIT, $0-32 + MOVQ bits+0(FP), AX + MOVQ rows_array_ptr+8(FP), BX + MOVQ rows_array_len+16(FP), DI + MOVQ rows_array_off+24(FP), DX + + MOVQ $1, CX + XORQ SI, SI + + CMPQ DI, $0 + JE done + + CMPQ DI, $8 + JB loop1x4 + + CMPB ยทhasAVX2(SB), $0 + JE loop1x4 + + MOVQ DI, R8 + SHRQ $3, R8 + SHLQ $3, R8 + + VPBROADCASTD rows_array_off+24(FP), Y0 + VPMULLD ยทrange0n8(SB), Y0, Y0 + VPCMPEQD Y1, Y1, Y1 + VPCMPEQD Y2, Y2, Y2 + VPXOR Y3, Y3, Y3 +loop8x4: + VPGATHERDD Y1, (BX)(Y0*1), Y4 + VPCMPEQD Y3, Y4, Y4 + VMOVMSKPS Y4, R9 + VMOVDQU Y2, Y1 + + NOTQ R9 + ANDQ $0b11111111, R9 + + MOVQ SI, CX + ANDQ $0b111111, CX + + MOVQ SI, R10 + SHRQ $6, R10 + + SHLQ CX, R9 + ORQ R9, (AX)(R10*8) + + LEAQ (BX)(DX*8), BX + ADDQ $8, SI + CMPQ SI, R8 + JNE loop8x4 + VZEROUPPER + + CMPQ SI, DI + JE done + + MOVQ $1, R8 + MOVQ SI, CX + ANDQ $0b111111, R8 + SHLQ CX, R8 + MOVQ R8, CX + +loop1x4: + MOVL (BX), R8 + CMPL R8, $0 + JE next1x4 + + MOVQ SI, R9 + SHRQ $6, R9 + ORQ CX, (AX)(R9*8) +next1x4: + ADDQ DX, BX + ROLQ $1, CX + INCQ SI + CMPQ SI, DI + JNE loop1x4 +done: + RET + +// func nullIndex64(bits *uint64, rows sparse.Array) +TEXT ยทnullIndex64(SB), NOSPLIT, $0-32 + MOVQ bits+0(FP), AX + MOVQ rows_array_ptr+8(FP), BX + MOVQ rows_array_len+16(FP), DI + MOVQ rows_array_off+24(FP), DX + + MOVQ $1, CX + XORQ SI, SI + + CMPQ DI, $0 + JE done + + CMPQ DI, $4 + JB loop1x8 + + CMPB ยทhasAVX2(SB), $0 + JE loop1x8 + + MOVQ DI, R8 + SHRQ $2, R8 + SHLQ $2, R8 + + VPBROADCASTQ rows_array_off+24(FP), Y0 + VPMULLD scale4x8<>(SB), Y0, Y0 + VPCMPEQQ Y1, Y1, Y1 + VPCMPEQQ Y2, Y2, Y2 + VPXOR Y3, Y3, Y3 +loop4x8: + VPGATHERQQ Y1, (BX)(Y0*1), Y4 + VPCMPEQQ Y3, Y4, Y4 + VMOVMSKPD Y4, R9 + VMOVDQU Y2, Y1 + + NOTQ R9 + ANDQ $0b1111, R9 + + MOVQ SI, CX + ANDQ $0b111111, CX + + MOVQ SI, R10 + SHRQ $6, R10 + + SHLQ CX, R9 + ORQ R9, (AX)(R10*8) + + LEAQ (BX)(DX*4), BX + ADDQ $4, SI + CMPQ SI, R8 + JNE loop4x8 + VZEROUPPER + + CMPQ SI, DI + JE done + + MOVQ $1, R8 + MOVQ SI, CX + ANDQ $0b111111, R8 + SHLQ CX, R8 + MOVQ R8, CX + +loop1x8: + MOVQ (BX), R8 + CMPQ R8, $0 + JE next1x8 + + MOVQ SI, R9 + SHRQ $6, R9 + ORQ CX, (AX)(R9*8) +next1x8: + ADDQ DX, BX + ROLQ $1, CX + INCQ SI + CMPQ SI, DI + JNE loop1x8 +done: + RET + +GLOBL scale4x8<>(SB), RODATA|NOPTR, $32 +DATA scale4x8<>+0(SB)/8, $0 +DATA scale4x8<>+8(SB)/8, $1 +DATA scale4x8<>+16(SB)/8, $2 +DATA scale4x8<>+24(SB)/8, $3 + +// func nullIndex128(bits *uint64, rows sparse.Array) +TEXT ยทnullIndex128(SB), NOSPLIT, $0-32 + MOVQ bits+0(FP), AX + MOVQ rows_array_ptr+8(FP), BX + MOVQ rows_array_len+16(FP), DI + MOVQ rows_array_off+24(FP), DX + + CMPQ DI, $0 + JE done + + MOVQ $1, CX + XORQ SI, SI + PXOR X0, X0 +loop1x16: + MOVOU (BX), X1 + PCMPEQQ X0, X1 + MOVMSKPD X1, R8 + CMPB R8, $0b11 + JE next1x16 + + MOVQ SI, R9 + SHRQ $6, R9 + ORQ CX, (AX)(R9*8) +next1x16: + ADDQ DX, BX + ROLQ $1, CX + INCQ SI + CMPQ SI, DI + JNE loop1x16 +done: + RET diff --git a/vendor/github.com/parquet-go/parquet-go/null_purego.go b/vendor/github.com/parquet-go/parquet-go/null_purego.go new file mode 100644 index 0000000000..fd6c0e52f5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/null_purego.go @@ -0,0 +1,64 @@ +//go:build purego || !amd64 + +package parquet + +import "github.com/parquet-go/parquet-go/sparse" + +func nullIndexBool(bits []uint64, rows sparse.Array) { + nullIndex[bool](bits, rows) +} + +func nullIndexInt(bits []uint64, rows sparse.Array) { + nullIndex[int](bits, rows) +} + +func nullIndexInt32(bits []uint64, rows sparse.Array) { + nullIndex[int32](bits, rows) +} + +func nullIndexInt64(bits []uint64, rows sparse.Array) { + nullIndex[int64](bits, rows) +} + +func nullIndexUint(bits []uint64, rows sparse.Array) { + nullIndex[uint](bits, rows) +} + +func nullIndexUint32(bits []uint64, rows sparse.Array) { + nullIndex[uint32](bits, rows) +} + +func nullIndexUint64(bits []uint64, rows sparse.Array) { + nullIndex[uint64](bits, rows) +} + +func nullIndexUint128(bits []uint64, rows sparse.Array) { + nullIndex[[16]byte](bits, rows) +} + +func nullIndexFloat32(bits []uint64, rows sparse.Array) { + nullIndex[float32](bits, rows) +} + +func nullIndexFloat64(bits []uint64, rows sparse.Array) { + nullIndex[float64](bits, rows) +} + +func nullIndexString(bits []uint64, rows sparse.Array) { + nullIndex[string](bits, rows) +} + +func nullIndexSlice(bits []uint64, rows sparse.Array) { + for i := range rows.Len() { + p := *(**struct{})(rows.Index(i)) + b := uint64(0) + if p != nil { + b = 1 + } + bits[uint(i)/64] |= b << (uint(i) % 64) + } +} + +func nullIndexPointer(bits []uint64, rows sparse.Array) { + nullIndex[*struct{}](bits, rows) +} diff --git a/vendor/github.com/parquet-go/parquet-go/offset_index.go b/vendor/github.com/parquet-go/parquet-go/offset_index.go new file mode 100644 index 0000000000..b499e427cd --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/offset_index.go @@ -0,0 +1,128 @@ +package parquet + +import ( + "github.com/parquet-go/parquet-go/format" +) + +type OffsetIndex interface { + // NumPages returns the number of pages in the offset index. + NumPages() int + + // Offset returns the offset starting from the beginning of the file for the + // page at the given index. + Offset(int) int64 + + // CompressedPageSize returns the size of the page at the given index + // (in bytes). + CompressedPageSize(int) int64 + + // FirstRowIndex returns the the first row in the page at the given index. + // + // The returned row index is based on the row group that the page belongs + // to, the first row has index zero. + FirstRowIndex(int) int64 +} + +type FileOffsetIndex struct { + index *format.OffsetIndex +} + +func (i *FileOffsetIndex) NumPages() int { + return len(i.index.PageLocations) +} + +func (i *FileOffsetIndex) Offset(j int) int64 { + return i.index.PageLocations[j].Offset +} + +func (i *FileOffsetIndex) CompressedPageSize(j int) int64 { + return int64(i.index.PageLocations[j].CompressedPageSize) +} + +func (i *FileOffsetIndex) FirstRowIndex(j int) int64 { + return i.index.PageLocations[j].FirstRowIndex +} + +type emptyOffsetIndex struct{} + +func (emptyOffsetIndex) NumPages() int { return 0 } +func (emptyOffsetIndex) Offset(int) int64 { return 0 } +func (emptyOffsetIndex) CompressedPageSize(int) int64 { return 0 } +func (emptyOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type booleanOffsetIndex struct{ page *booleanPage } + +func (i booleanOffsetIndex) NumPages() int { return 1 } +func (i booleanOffsetIndex) Offset(int) int64 { return 0 } +func (i booleanOffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i booleanOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type int32OffsetIndex struct{ page *int32Page } + +func (i int32OffsetIndex) NumPages() int { return 1 } +func (i int32OffsetIndex) Offset(int) int64 { return 0 } +func (i int32OffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i int32OffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type int64OffsetIndex struct{ page *int64Page } + +func (i int64OffsetIndex) NumPages() int { return 1 } +func (i int64OffsetIndex) Offset(int) int64 { return 0 } +func (i int64OffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i int64OffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type int96OffsetIndex struct{ page *int96Page } + +func (i int96OffsetIndex) NumPages() int { return 1 } +func (i int96OffsetIndex) Offset(int) int64 { return 0 } +func (i int96OffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i int96OffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type floatOffsetIndex struct{ page *floatPage } + +func (i floatOffsetIndex) NumPages() int { return 1 } +func (i floatOffsetIndex) Offset(int) int64 { return 0 } +func (i floatOffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i floatOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type doubleOffsetIndex struct{ page *doublePage } + +func (i doubleOffsetIndex) NumPages() int { return 1 } +func (i doubleOffsetIndex) Offset(int) int64 { return 0 } +func (i doubleOffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i doubleOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type byteArrayOffsetIndex struct{ page *byteArrayPage } + +func (i byteArrayOffsetIndex) NumPages() int { return 1 } +func (i byteArrayOffsetIndex) Offset(int) int64 { return 0 } +func (i byteArrayOffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i byteArrayOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type fixedLenByteArrayOffsetIndex struct{ page *fixedLenByteArrayPage } + +func (i fixedLenByteArrayOffsetIndex) NumPages() int { return 1 } +func (i fixedLenByteArrayOffsetIndex) Offset(int) int64 { return 0 } +func (i fixedLenByteArrayOffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i fixedLenByteArrayOffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type uint32OffsetIndex struct{ page *uint32Page } + +func (i uint32OffsetIndex) NumPages() int { return 1 } +func (i uint32OffsetIndex) Offset(int) int64 { return 0 } +func (i uint32OffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i uint32OffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type uint64OffsetIndex struct{ page *uint64Page } + +func (i uint64OffsetIndex) NumPages() int { return 1 } +func (i uint64OffsetIndex) Offset(int) int64 { return 0 } +func (i uint64OffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i uint64OffsetIndex) FirstRowIndex(int) int64 { return 0 } + +type be128OffsetIndex struct{ page *be128Page } + +func (i be128OffsetIndex) NumPages() int { return 1 } +func (i be128OffsetIndex) Offset(int) int64 { return 0 } +func (i be128OffsetIndex) CompressedPageSize(int) int64 { return i.page.Size() } +func (i be128OffsetIndex) FirstRowIndex(int) int64 { return 0 } diff --git a/vendor/github.com/parquet-go/parquet-go/order.go b/vendor/github.com/parquet-go/parquet-go/order.go new file mode 100644 index 0000000000..b789f1bb65 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/order.go @@ -0,0 +1,102 @@ +package parquet + +import ( + "bytes" + + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +func orderOfBool(data []bool) int { + switch len(data) { + case 0, 1: + return 0 + default: + k := 0 + i := 0 + + if data[0] { // true => false: descending + k = -1 + i = streakOfTrue(data) + if i == len(data) { + k = +1 + } else { + i += streakOfFalse(data[i:]) + } + } else { // false => true: ascending + k = +1 + i = streakOfFalse(data) + i += streakOfTrue(data[i:]) + } + + if i != len(data) { + k = 0 + } + return k + } +} + +func streakOfTrue(data []bool) int { + if i := bytes.IndexByte(unsafecast.Slice[byte](data), 0); i >= 0 { + return i + } + return len(data) +} + +func streakOfFalse(data []bool) int { + if i := bytes.IndexByte(unsafecast.Slice[byte](data), 1); i >= 0 { + return i + } + return len(data) +} + +func orderOfBytes(data [][]byte) int { + switch len(data) { + case 0, 1: + return 0 + } + data = skipBytesStreak(data) + if len(data) < 2 { + return 1 + } + ordering := bytes.Compare(data[0], data[1]) + switch { + case ordering < 0: + if bytesAreInAscendingOrder(data[1:]) { + return +1 + } + case ordering > 0: + if bytesAreInDescendingOrder(data[1:]) { + return -1 + } + } + return 0 +} + +func skipBytesStreak(data [][]byte) [][]byte { + for i := 1; i < len(data); i++ { + if !bytes.Equal(data[i], data[0]) { + return data[i-1:] + } + } + return data[len(data)-1:] +} + +func bytesAreInAscendingOrder(data [][]byte) bool { + for i := len(data) - 1; i > 0; i-- { + k := bytes.Compare(data[i-1], data[i]) + if k > 0 { + return false + } + } + return true +} + +func bytesAreInDescendingOrder(data [][]byte) bool { + for i := len(data) - 1; i > 0; i-- { + k := bytes.Compare(data[i-1], data[i]) + if k < 0 { + return false + } + } + return true +} diff --git a/vendor/github.com/parquet-go/parquet-go/order_amd64.go b/vendor/github.com/parquet-go/parquet-go/order_amd64.go new file mode 100644 index 0000000000..8ac597d62a --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/order_amd64.go @@ -0,0 +1,21 @@ +//go:build !purego + +package parquet + +//go:noescape +func orderOfInt32(data []int32) int + +//go:noescape +func orderOfInt64(data []int64) int + +//go:noescape +func orderOfUint32(data []uint32) int + +//go:noescape +func orderOfUint64(data []uint64) int + +//go:noescape +func orderOfFloat32(data []float32) int + +//go:noescape +func orderOfFloat64(data []float64) int diff --git a/vendor/github.com/parquet-go/parquet-go/order_amd64.s b/vendor/github.com/parquet-go/parquet-go/order_amd64.s new file mode 100644 index 0000000000..651d2510bc --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/order_amd64.s @@ -0,0 +1,547 @@ +//go:build !purego + +#include "textflag.h" + +#define UNDEFINED 0 +#define ASCENDING 1 +#define DESCENDING -1 + +DATA shift1x32<>+0(SB)/4, $1 +DATA shift1x32<>+4(SB)/4, $2 +DATA shift1x32<>+8(SB)/4, $3 +DATA shift1x32<>+12(SB)/4, $4 +DATA shift1x32<>+16(SB)/4, $5 +DATA shift1x32<>+20(SB)/4, $6 +DATA shift1x32<>+24(SB)/4, $7 +DATA shift1x32<>+28(SB)/4, $8 +DATA shift1x32<>+32(SB)/4, $9 +DATA shift1x32<>+36(SB)/4, $10 +DATA shift1x32<>+40(SB)/4, $11 +DATA shift1x32<>+44(SB)/4, $12 +DATA shift1x32<>+48(SB)/4, $13 +DATA shift1x32<>+52(SB)/4, $14 +DATA shift1x32<>+56(SB)/4, $15 +DATA shift1x32<>+60(SB)/4, $15 +GLOBL shift1x32<>(SB), RODATA|NOPTR, $64 + +DATA shift1x64<>+0(SB)/4, $1 +DATA shift1x64<>+8(SB)/4, $2 +DATA shift1x64<>+16(SB)/4, $3 +DATA shift1x64<>+24(SB)/4, $4 +DATA shift1x64<>+32(SB)/4, $5 +DATA shift1x64<>+40(SB)/4, $6 +DATA shift1x64<>+48(SB)/4, $7 +DATA shift1x64<>+56(SB)/4, $7 +GLOBL shift1x64<>(SB), RODATA|NOPTR, $64 + +// func orderOfInt32(data []int32) int +TEXT ยทorderOfInt32(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), R8 + MOVQ data_len+8(FP), R9 + XORQ SI, SI + XORQ DI, DI + + CMPQ R9, $2 + JB undefined + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + CMPQ R9, $16 + JB test + + XORQ DX, DX + MOVQ R9, AX + SHRQ $4, AX + SHLQ $4, AX + MOVQ $15, CX + IDIVQ CX + IMULQ $15, AX + DECQ R9 + + VMOVDQU32 shift1x32<>(SB), Z2 + KXORW K2, K2, K2 +testAscending15: + VMOVDQU32 (R8)(SI*4), Z0 + VMOVDQU32 Z2, Z1 + VPERMI2D Z0, Z0, Z1 + VPCMPD $2, Z1, Z0, K1 + KORTESTW K2, K1 + JNC testDescending15 + ADDQ $15, SI + CMPQ SI, AX + JNE testAscending15 + VZEROUPPER + JMP testAscending +testDescending15: + VMOVDQU32 (R8)(DI*4), Z0 + VMOVDQU32 Z2, Z1 + VPERMI2D Z0, Z0, Z1 + VPCMPD $5, Z1, Z0, K1 + KORTESTW K2, K1 + JNC undefined15 + ADDQ $15, DI + CMPQ DI, AX + JNE testDescending15 + VZEROUPPER + JMP testDescending + +test: + DECQ R9 +testAscending: + CMPQ SI, R9 + JAE ascending + MOVL (R8)(SI*4), BX + MOVL 4(R8)(SI*4), DX + INCQ SI + CMPL BX, DX + JLE testAscending + JMP testDescending +ascending: + MOVQ $ASCENDING, ret+24(FP) + RET +testDescending: + CMPQ DI, R9 + JAE descending + MOVL (R8)(DI*4), BX + MOVL 4(R8)(DI*4), DX + INCQ DI + CMPL BX, DX + JGE testDescending + JMP undefined +descending: + MOVQ $DESCENDING, ret+24(FP) + RET +undefined15: + VZEROUPPER +undefined: + MOVQ $UNDEFINED, ret+24(FP) + RET + +// func orderOfInt64(data []int64) int +TEXT ยทorderOfInt64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), R8 + MOVQ data_len+8(FP), R9 + XORQ SI, SI + XORQ DI, DI + + CMPQ R9, $2 + JB undefined + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + CMPQ R9, $8 + JB test + + XORQ DX, DX + MOVQ R9, AX + SHRQ $3, AX + SHLQ $3, AX + MOVQ $7, CX + IDIVQ CX + IMULQ $7, AX + DECQ R9 + + VMOVDQU64 shift1x64<>(SB), Z2 + KXORB K2, K2, K2 +testAscending7: + VMOVDQU64 (R8)(SI*8), Z0 + VMOVDQU64 Z2, Z1 + VPERMI2Q Z0, Z0, Z1 + VPCMPQ $2, Z1, Z0, K1 + KORTESTB K2, K1 + JNC testDescending7 + ADDQ $7, SI + CMPQ SI, AX + JNE testAscending7 + VZEROUPPER + JMP testAscending +testDescending7: + VMOVDQU64 (R8)(DI*8), Z0 + VMOVDQU64 Z2, Z1 + VPERMI2Q Z0, Z0, Z1 + VPCMPQ $5, Z1, Z0, K1 + KORTESTB K2, K1 + JNC undefined7 + ADDQ $7, DI + CMPQ DI, AX + JNE testDescending7 + VZEROUPPER + JMP testDescending + +test: + DECQ R9 +testAscending: + CMPQ SI, R9 + JAE ascending + MOVQ (R8)(SI*8), BX + MOVQ 8(R8)(SI*8), DX + INCQ SI + CMPQ BX, DX + JLE testAscending + JMP testDescending +ascending: + MOVQ $ASCENDING, ret+24(FP) + RET +testDescending: + CMPQ DI, R9 + JAE descending + MOVQ (R8)(DI*8), BX + MOVQ 8(R8)(DI*8), DX + INCQ DI + CMPQ BX, DX + JGE testDescending + JMP undefined +descending: + MOVQ $DESCENDING, ret+24(FP) + RET +undefined7: + VZEROUPPER +undefined: + MOVQ $UNDEFINED, ret+24(FP) + RET + +// func orderOfUint32(data []uint32) int +TEXT ยทorderOfUint32(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), R8 + MOVQ data_len+8(FP), R9 + XORQ SI, SI + XORQ DI, DI + + CMPQ R9, $2 + JB undefined + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + CMPQ R9, $16 + JB test + + XORQ DX, DX + MOVQ R9, AX + SHRQ $4, AX + SHLQ $4, AX + MOVQ $15, CX + IDIVQ CX + IMULQ $15, AX + DECQ R9 + + VMOVDQU32 shift1x32<>(SB), Z2 + KXORW K2, K2, K2 +testAscending15: + VMOVDQU32 (R8)(SI*4), Z0 + VMOVDQU32 Z2, Z1 + VPERMI2D Z0, Z0, Z1 + VPCMPUD $2, Z1, Z0, K1 + KORTESTW K2, K1 + JNC testDescending15 + ADDQ $15, SI + CMPQ SI, AX + JNE testAscending15 + VZEROUPPER + JMP testAscending +testDescending15: + VMOVDQU32 (R8)(DI*4), Z0 + VMOVDQU32 Z2, Z1 + VPERMI2D Z0, Z0, Z1 + VPCMPUD $5, Z1, Z0, K1 + KORTESTW K2, K1 + JNC undefined15 + ADDQ $15, DI + CMPQ DI, AX + JNE testDescending15 + VZEROUPPER + JMP testDescending + +test: + DECQ R9 +testAscending: + CMPQ SI, R9 + JAE ascending + MOVL (R8)(SI*4), BX + MOVL 4(R8)(SI*4), DX + INCQ SI + CMPL BX, DX + JBE testAscending + JMP testDescending +ascending: + MOVQ $ASCENDING, ret+24(FP) + RET +testDescending: + CMPQ DI, R9 + JAE descending + MOVL (R8)(DI*4), BX + MOVL 4(R8)(DI*4), DX + INCQ DI + CMPL BX, DX + JAE testDescending + JMP undefined +descending: + MOVQ $DESCENDING, ret+24(FP) + RET +undefined15: + VZEROUPPER +undefined: + MOVQ $UNDEFINED, ret+24(FP) + RET + +// func orderOfUint64(data []uint64) int +TEXT ยทorderOfUint64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), R8 + MOVQ data_len+8(FP), R9 + XORQ SI, SI + XORQ DI, DI + + CMPQ R9, $2 + JB undefined + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + CMPQ R9, $8 + JB test + + XORQ DX, DX + MOVQ R9, AX + SHRQ $3, AX + SHLQ $3, AX + MOVQ $7, CX + IDIVQ CX + IMULQ $7, AX + DECQ R9 + + VMOVDQU64 shift1x64<>(SB), Z2 + KXORB K2, K2, K2 +testAscending7: + VMOVDQU64 (R8)(SI*8), Z0 + VMOVDQU64 Z2, Z1 + VPERMI2Q Z0, Z0, Z1 + VPCMPUQ $2, Z1, Z0, K1 + KORTESTB K2, K1 + JNC testDescending7 + ADDQ $7, SI + CMPQ SI, AX + JNE testAscending7 + VZEROUPPER + JMP testAscending +testDescending7: + VMOVDQU64 (R8)(DI*8), Z0 + VMOVDQU64 Z2, Z1 + VPERMI2Q Z0, Z0, Z1 + VPCMPUQ $5, Z1, Z0, K1 + KORTESTB K2, K1 + JNC undefined7 + ADDQ $7, DI + CMPQ DI, AX + JNE testDescending7 + VZEROUPPER + JMP testDescending + +test: + DECQ R9 +testAscending: + CMPQ SI, R9 + JAE ascending + MOVQ (R8)(SI*8), BX + MOVQ 8(R8)(SI*8), DX + INCQ SI + CMPQ BX, DX + JBE testAscending + JMP testDescending +ascending: + MOVQ $ASCENDING, ret+24(FP) + RET +testDescending: + CMPQ DI, R9 + JAE descending + MOVQ (R8)(DI*8), BX + MOVQ 8(R8)(DI*8), DX + INCQ DI + CMPQ BX, DX + JAE testDescending + JMP undefined +descending: + MOVQ $DESCENDING, ret+24(FP) + RET +undefined7: + VZEROUPPER +undefined: + MOVQ $UNDEFINED, ret+24(FP) + RET + +// func orderOfFloat32(data []float32) int +TEXT ยทorderOfFloat32(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), R8 + MOVQ data_len+8(FP), R9 + XORQ SI, SI + XORQ DI, DI + + CMPQ R9, $2 + JB undefined + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + CMPQ R9, $16 + JB test + + XORQ DX, DX + MOVQ R9, AX + SHRQ $4, AX + SHLQ $4, AX + MOVQ $15, CX + IDIVQ CX + IMULQ $15, AX + DECQ R9 + + VMOVDQU32 shift1x32<>(SB), Z2 + KXORW K2, K2, K2 +testAscending15: + VMOVDQU32 (R8)(SI*4), Z0 + VMOVDQU32 Z2, Z1 + VPERMI2D Z0, Z0, Z1 + VCMPPS $2, Z1, Z0, K1 + KORTESTW K2, K1 + JNC testDescending15 + ADDQ $15, SI + CMPQ SI, AX + JNE testAscending15 + VZEROUPPER + JMP testAscending +testDescending15: + VMOVDQU32 (R8)(DI*4), Z0 + VMOVDQU32 Z2, Z1 + VPERMI2D Z0, Z0, Z1 + VCMPPS $5, Z1, Z0, K1 + KORTESTW K2, K1 + JNC undefined15 + ADDQ $15, DI + CMPQ DI, AX + JNE testDescending15 + VZEROUPPER + JMP testDescending + +test: + DECQ R9 +testAscending: + CMPQ SI, R9 + JAE ascending + MOVLQZX (R8)(SI*4), BX + MOVLQZX 4(R8)(SI*4), DX + INCQ SI + MOVQ BX, X0 + MOVQ DX, X1 + UCOMISS X1, X0 + JBE testAscending + JMP testDescending +ascending: + MOVQ $ASCENDING, ret+24(FP) + RET +testDescending: + CMPQ DI, R9 + JAE descending + MOVLQZX (R8)(DI*4), BX + MOVLQZX 4(R8)(DI*4), DX + INCQ DI + MOVQ BX, X0 + MOVQ DX, X1 + UCOMISS X1, X0 + JAE testDescending + JMP undefined +descending: + MOVQ $DESCENDING, ret+24(FP) + RET +undefined15: + VZEROUPPER +undefined: + MOVQ $UNDEFINED, ret+24(FP) + RET + +// func orderOfFloat64(data []uint64) int +TEXT ยทorderOfFloat64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), R8 + MOVQ data_len+8(FP), R9 + XORQ SI, SI + XORQ DI, DI + + CMPQ R9, $2 + JB undefined + + CMPB ยทhasAVX512VL(SB), $0 + JE test + + CMPQ R9, $8 + JB test + + XORQ DX, DX + MOVQ R9, AX + SHRQ $3, AX + SHLQ $3, AX + MOVQ $7, CX + IDIVQ CX + IMULQ $7, AX + DECQ R9 + + VMOVDQU64 shift1x64<>(SB), Z2 + KXORB K2, K2, K2 +testAscending7: + VMOVDQU64 (R8)(SI*8), Z0 + VMOVDQU64 Z2, Z1 + VPERMI2Q Z0, Z0, Z1 + VCMPPD $2, Z1, Z0, K1 + KORTESTB K2, K1 + JNC testDescending7 + ADDQ $7, SI + CMPQ SI, AX + JNE testAscending7 + VZEROUPPER + JMP testAscending +testDescending7: + VMOVDQU64 (R8)(DI*8), Z0 + VMOVDQU64 Z2, Z1 + VPERMI2Q Z0, Z0, Z1 + VCMPPD $5, Z1, Z0, K1 + KORTESTB K2, K1 + JNC undefined7 + ADDQ $7, DI + CMPQ DI, AX + JNE testDescending7 + VZEROUPPER + JMP testDescending + +test: + DECQ R9 +testAscending: + CMPQ SI, R9 + JAE ascending + MOVQ (R8)(SI*8), BX + MOVQ 8(R8)(SI*8), DX + INCQ SI + MOVQ BX, X0 + MOVQ DX, X1 + UCOMISD X1, X0 + JBE testAscending + JMP testDescending +ascending: + MOVQ $ASCENDING, ret+24(FP) + RET +testDescending: + CMPQ DI, R9 + JAE descending + MOVQ (R8)(DI*8), BX + MOVQ 8(R8)(DI*8), DX + INCQ DI + MOVQ BX, X0 + MOVQ DX, X1 + UCOMISD X1, X0 + JAE testDescending + JMP undefined +descending: + MOVQ $DESCENDING, ret+24(FP) + RET +undefined7: + VZEROUPPER +undefined: + MOVQ $UNDEFINED, ret+24(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/order_purego.go b/vendor/github.com/parquet-go/parquet-go/order_purego.go new file mode 100644 index 0000000000..44c4d7905e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/order_purego.go @@ -0,0 +1,42 @@ +//go:build purego || !amd64 + +package parquet + +import "cmp" + +func orderOfInt32(data []int32) int { return orderOf(data) } +func orderOfInt64(data []int64) int { return orderOf(data) } +func orderOfUint32(data []uint32) int { return orderOf(data) } +func orderOfUint64(data []uint64) int { return orderOf(data) } +func orderOfFloat32(data []float32) int { return orderOf(data) } +func orderOfFloat64(data []float64) int { return orderOf(data) } + +func orderOf[T cmp.Ordered](data []T) int { + if len(data) > 1 { + if orderIsAscending(data) { + return +1 + } + if orderIsDescending(data) { + return -1 + } + } + return 0 +} + +func orderIsAscending[T cmp.Ordered](data []T) bool { + for i := len(data) - 1; i > 0; i-- { + if data[i-1] > data[i] { + return false + } + } + return true +} + +func orderIsDescending[T cmp.Ordered](data []T) bool { + for i := len(data) - 1; i > 0; i-- { + if data[i-1] < data[i] { + return false + } + } + return true +} diff --git a/vendor/github.com/parquet-go/parquet-go/page.go b/vendor/github.com/parquet-go/parquet-go/page.go new file mode 100644 index 0000000000..2c31fbe366 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page.go @@ -0,0 +1,1453 @@ +package parquet + +import ( + "bytes" + "fmt" + "io" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/internal/bitpack" + "github.com/parquet-go/parquet-go/internal/debug" +) + +// Page values represent sequences of parquet values. From the Parquet +// documentation: "Column chunks are a chunk of the data for a particular +// column. They live in a particular row group and are guaranteed to be +// contiguous in the file. Column chunks are divided up into pages. A page is +// conceptually an indivisible unit (in terms of compression and encoding). +// There can be multiple page types which are interleaved in a column chunk." +// +// https://github.com/apache/parquet-format#glossary +type Page interface { + // Returns the type of values read from this page. + // + // The returned type can be used to encode the page data, in the case of + // an indexed page (which has a dictionary), the type is configured to + // encode the indexes stored in the page rather than the plain values. + Type() Type + + // Returns the column index that this page belongs to. + Column() int + + // If the page contains indexed values, calling this method returns the + // dictionary in which the values are looked up. Otherwise, the method + // returns nil. + Dictionary() Dictionary + + // Returns the number of rows, values, and nulls in the page. The number of + // rows may be less than the number of values in the page if the page is + // part of a repeated column. + NumRows() int64 + NumValues() int64 + NumNulls() int64 + + // Returns the page's min and max values. + // + // The third value is a boolean indicating whether the page bounds were + // available. Page bounds may not be known if the page contained no values + // or only nulls, or if they were read from a parquet file which had neither + // page statistics nor a page index. + Bounds() (min, max Value, ok bool) + + // Returns the size of the page in bytes (uncompressed). + Size() int64 + + // Returns a reader exposing the values contained in the page. + // + // Depending on the underlying implementation, the returned reader may + // support reading an array of typed Go values by implementing interfaces + // like parquet.Int32Reader. Applications should use type assertions on + // the returned reader to determine whether those optimizations are + // available. + // + // In the data page format version 1, it wasn't specified whether pages + // must start with a new row. Legacy writers have produced parquet files + // where row values were overlapping between two consecutive pages. + // As a result, the values read must not be assumed to start at the + // beginning of a row, unless the program knows that it is only working + // with parquet files that used the data page format version 2 (which is + // the default behavior for parquet-go). + Values() ValueReader + + // Returns a new page which is as slice of the receiver between row indexes + // i and j. + Slice(i, j int64) Page + + // Expose the lists of repetition and definition levels of the page. + // + // The returned slices may be empty when the page has no repetition or + // definition levels. + RepetitionLevels() []byte + DefinitionLevels() []byte + + // Returns the in-memory buffer holding the page values. + // + // The intent is for the returned value to be used as input parameter when + // calling the Encode method of the associated Type. + // + // The slices referenced by the encoding.Values may be the same across + // multiple calls to this method, applications must treat the content as + // immutable. + Data() encoding.Values +} + +// PageReader is an interface implemented by types that support producing a +// sequence of pages. +type PageReader interface { + // Reads and returns the next page from the sequence. When all pages have + // been read, or if the sequence was closed, the method returns io.EOF. + ReadPage() (Page, error) +} + +// PageWriter is an interface implemented by types that support writing pages +// to an underlying storage medium. +type PageWriter interface { + WritePage(Page) (int64, error) +} + +// Pages is an interface implemented by page readers returned by calling the +// Pages method of ColumnChunk instances. +type Pages interface { + PageReader + RowSeeker + io.Closer +} + +// AsyncPages wraps the given Pages instance to perform page reads +// asynchronously in a separate goroutine. +// +// Performing page reads asynchronously is important when the application may +// be reading pages from a high latency backend, and the last +// page read may be processed while initiating reading of the next page. +func AsyncPages(pages Pages) Pages { + read := make(chan asyncPage) + seek := make(chan asyncSeek, 1) + init := make(chan struct{}) + done := make(chan struct{}) + + go readPages(pages, read, seek, init, done) + + p := &asyncPages{ + read: read, + seek: seek, + init: init, + done: done, + } + + // If the pages object gets garbage collected without Close being called, + // this finalizer would ensure that the goroutine is stopped and doesn't + // leak. + debug.SetFinalizer(p, func(p *asyncPages) { p.Close() }) + return p +} + +type asyncPages struct { + read chan asyncPage + seek chan asyncSeek + init chan struct{} + done chan struct{} + version int64 +} + +type asyncPage struct { + page Page + err error + version int64 +} + +type asyncSeek struct { + rowIndex int64 + version int64 +} + +func (pages *asyncPages) Close() (err error) { + if pages.init != nil { + close(pages.init) + pages.init = nil + } + if pages.done != nil { + close(pages.done) + pages.done = nil + } + for p := range pages.read { + // Capture the last error, which is the value returned from closing the + // underlying Pages instance. + err = p.err + } + pages.seek = nil + return err +} + +func (pages *asyncPages) ReadPage() (Page, error) { + pages.start() + for { + p, ok := <-pages.read + if !ok { + return nil, io.EOF + } + // Because calls to SeekToRow might be made concurrently to reading + // pages, it is possible for ReadPage to see pages that were read before + // the last SeekToRow call. + // + // A version number is attached to each page read asynchronously to + // discard outdated pages and ensure that we maintain a consistent view + // of the sequence of pages read. + if p.version == pages.version { + return p.page, p.err + } + } +} + +func (pages *asyncPages) SeekToRow(rowIndex int64) error { + if pages.seek == nil { + return io.ErrClosedPipe + } + // First flush the channel in case SeekToRow is called twice or more in a + // row, otherwise we would block if readPages had already exited. + select { + case <-pages.seek: + default: + pages.version++ + } + // The seek channel has a capacity of 1 to allow the first SeekToRow call to + // be non-blocking. + // + // If SeekToRow calls are performed faster than they can be handled by the + // goroutine reading pages, this path might become a contention point. + pages.seek <- asyncSeek{rowIndex: rowIndex, version: pages.version} + pages.start() + return nil +} + +func (pages *asyncPages) start() { + if pages.init != nil { + close(pages.init) + pages.init = nil + } +} + +func readPages(pages Pages, read chan<- asyncPage, seek <-chan asyncSeek, init, done <-chan struct{}) { + defer func() { + read <- asyncPage{err: pages.Close(), version: -1} + close(read) + }() + + // To avoid reading pages before the first SeekToRow call, we wait for the + // reader to be initialized, which means it either received a call to + // ReadPage, SeekToRow, or Close. + select { + case <-init: + case <-done: + return + } + + // If SeekToRow was invoked before ReadPage, the seek channel contains the + // new position of the reader. + // + // Note that we have a default case in this select because we don't want to + // block if the first call was ReadPage and no values were ever produced to + // the seek channel. + var seekTo asyncSeek + select { + case seekTo = <-seek: + default: + seekTo.rowIndex = -1 + } + + for { + var page Page + var err error + + if seekTo.rowIndex >= 0 { + err = pages.SeekToRow(seekTo.rowIndex) + if err == nil { + seekTo.rowIndex = -1 + continue + } + } else { + page, err = pages.ReadPage() + } + + select { + case read <- asyncPage{ + page: page, + err: err, + version: seekTo.version, + }: + case seekTo = <-seek: + Release(page) + case <-done: + Release(page) + return + } + } +} + +type singlePage struct { + page Page + seek int64 + numRows int64 +} + +func (r *singlePage) ReadPage() (Page, error) { + if r.page != nil { + if r.seek < r.numRows { + seek := r.seek + r.seek = r.numRows + if seek > 0 { + return r.page.Slice(seek, r.numRows), nil + } + return r.page, nil + } + } + return nil, io.EOF +} + +func (r *singlePage) SeekToRow(rowIndex int64) error { + r.seek = rowIndex + return nil +} + +func (r *singlePage) Close() error { + r.page = nil + r.seek = 0 + return nil +} + +func onePage(page Page) Pages { + return &singlePage{page: page, numRows: page.NumRows()} +} + +// CopyPages copies pages from src to dst, returning the number of values that +// were copied. +// +// The function returns any error it encounters reading or writing pages, except +// for io.EOF from the reader which indicates that there were no more pages to +// read. +func CopyPages(dst PageWriter, src PageReader) (numValues int64, err error) { + for { + p, err := src.ReadPage() + if err != nil { + if err == io.EOF { + err = nil + } + return numValues, err + } + n, err := dst.WritePage(p) + numValues += n + if err != nil { + return numValues, err + } + } +} + +// errorPage is an implementation of the Page interface which always errors when +// attempting to read its values. +// +// The error page declares that it contains one value (even if it does not) +// as a way to ensure that it is not ignored due to being empty when written +// to a file. +type errorPage struct { + typ Type + err error + columnIndex int +} + +func newErrorPage(typ Type, columnIndex int, msg string, args ...any) *errorPage { + return &errorPage{ + typ: typ, + err: fmt.Errorf(msg, args...), + columnIndex: columnIndex, + } +} + +func (page *errorPage) Type() Type { return page.typ } +func (page *errorPage) Column() int { return page.columnIndex } +func (page *errorPage) Dictionary() Dictionary { return nil } +func (page *errorPage) NumRows() int64 { return 1 } +func (page *errorPage) NumValues() int64 { return 1 } +func (page *errorPage) NumNulls() int64 { return 0 } +func (page *errorPage) Bounds() (min, max Value, ok bool) { return } +func (page *errorPage) Slice(i, j int64) Page { return page } +func (page *errorPage) Size() int64 { return 1 } +func (page *errorPage) RepetitionLevels() []byte { return nil } +func (page *errorPage) DefinitionLevels() []byte { return nil } +func (page *errorPage) Data() encoding.Values { return encoding.Values{} } +func (page *errorPage) Values() ValueReader { return errorPageValues{page: page} } + +type errorPageValues struct{ page *errorPage } + +func (r errorPageValues) ReadValues([]Value) (int, error) { return 0, r.page.err } +func (r errorPageValues) Close() error { return nil } + +func errPageBoundsOutOfRange(i, j, n int64) error { + return fmt.Errorf("page bounds out of range [%d:%d]: with length %d", i, j, n) +} + +type optionalPage struct { + base Page + maxDefinitionLevel byte + definitionLevels []byte +} + +func newOptionalPage(base Page, maxDefinitionLevel byte, definitionLevels []byte) *optionalPage { + return &optionalPage{ + base: base, + maxDefinitionLevel: maxDefinitionLevel, + definitionLevels: definitionLevels, + } +} + +func (page *optionalPage) Type() Type { return page.base.Type() } + +func (page *optionalPage) Column() int { return page.base.Column() } + +func (page *optionalPage) Dictionary() Dictionary { return page.base.Dictionary() } + +func (page *optionalPage) NumRows() int64 { return int64(len(page.definitionLevels)) } + +func (page *optionalPage) NumValues() int64 { return int64(len(page.definitionLevels)) } + +func (page *optionalPage) NumNulls() int64 { + return int64(countLevelsNotEqual(page.definitionLevels, page.maxDefinitionLevel)) +} + +func (page *optionalPage) Bounds() (min, max Value, ok bool) { return page.base.Bounds() } + +func (page *optionalPage) Size() int64 { return int64(len(page.definitionLevels)) + page.base.Size() } + +func (page *optionalPage) RepetitionLevels() []byte { return nil } + +func (page *optionalPage) DefinitionLevels() []byte { return page.definitionLevels } + +func (page *optionalPage) Data() encoding.Values { return page.base.Data() } + +func (page *optionalPage) Values() ValueReader { + return &optionalPageValues{ + page: page, + values: page.base.Values(), + } +} + +func (page *optionalPage) Slice(i, j int64) Page { + maxDefinitionLevel := page.maxDefinitionLevel + definitionLevels := page.definitionLevels + numNulls1 := int64(countLevelsNotEqual(definitionLevels[:i], maxDefinitionLevel)) + numNulls2 := int64(countLevelsNotEqual(definitionLevels[i:j], maxDefinitionLevel)) + return newOptionalPage( + page.base.Slice(i-numNulls1, j-(numNulls1+numNulls2)), + maxDefinitionLevel, + definitionLevels[i:j:j], + ) +} + +type repeatedPage struct { + base Page + maxRepetitionLevel byte + maxDefinitionLevel byte + definitionLevels []byte + repetitionLevels []byte +} + +func newRepeatedPage(base Page, maxRepetitionLevel, maxDefinitionLevel byte, repetitionLevels, definitionLevels []byte) *repeatedPage { + return &repeatedPage{ + base: base, + maxRepetitionLevel: maxRepetitionLevel, + maxDefinitionLevel: maxDefinitionLevel, + definitionLevels: definitionLevels, + repetitionLevels: repetitionLevels, + } +} + +func (page *repeatedPage) Type() Type { return page.base.Type() } + +func (page *repeatedPage) Column() int { return page.base.Column() } + +func (page *repeatedPage) Dictionary() Dictionary { return page.base.Dictionary() } + +func (page *repeatedPage) NumRows() int64 { return int64(countLevelsEqual(page.repetitionLevels, 0)) } + +func (page *repeatedPage) NumValues() int64 { return int64(len(page.definitionLevels)) } + +func (page *repeatedPage) NumNulls() int64 { + return int64(countLevelsNotEqual(page.definitionLevels, page.maxDefinitionLevel)) +} + +func (page *repeatedPage) Bounds() (min, max Value, ok bool) { return page.base.Bounds() } + +func (page *repeatedPage) Size() int64 { + return int64(len(page.repetitionLevels)) + int64(len(page.definitionLevels)) + page.base.Size() +} + +func (page *repeatedPage) RepetitionLevels() []byte { return page.repetitionLevels } + +func (page *repeatedPage) DefinitionLevels() []byte { return page.definitionLevels } + +func (page *repeatedPage) Data() encoding.Values { return page.base.Data() } + +func (page *repeatedPage) Values() ValueReader { + return &repeatedPageValues{ + page: page, + values: page.base.Values(), + } +} + +func (page *repeatedPage) Slice(i, j int64) Page { + numRows := page.NumRows() + if i < 0 || i > numRows { + panic(errPageBoundsOutOfRange(i, j, numRows)) + } + if j < 0 || j > numRows { + panic(errPageBoundsOutOfRange(i, j, numRows)) + } + if i > j { + panic(errPageBoundsOutOfRange(i, j, numRows)) + } + + maxRepetitionLevel := page.maxRepetitionLevel + maxDefinitionLevel := page.maxDefinitionLevel + repetitionLevels := page.repetitionLevels + definitionLevels := page.definitionLevels + + rowIndex0 := 0 + rowIndex1 := len(repetitionLevels) + rowIndex2 := len(repetitionLevels) + + for k, def := range repetitionLevels { + if def == 0 { + if rowIndex0 == int(i) { + rowIndex1 = k + break + } + rowIndex0++ + } + } + + for k, def := range repetitionLevels[rowIndex1:] { + if def == 0 { + if rowIndex0 == int(j) { + rowIndex2 = rowIndex1 + k + break + } + rowIndex0++ + } + } + + numNulls1 := countLevelsNotEqual(definitionLevels[:rowIndex1], maxDefinitionLevel) + numNulls2 := countLevelsNotEqual(definitionLevels[rowIndex1:rowIndex2], maxDefinitionLevel) + + i = int64(rowIndex1 - numNulls1) + j = int64(rowIndex2 - (numNulls1 + numNulls2)) + + return newRepeatedPage( + page.base.Slice(i, j), + maxRepetitionLevel, + maxDefinitionLevel, + repetitionLevels[rowIndex1:rowIndex2:rowIndex2], + definitionLevels[rowIndex1:rowIndex2:rowIndex2], + ) +} + +type booleanPage struct { + typ Type + bits []byte + offset int32 + numValues int32 + columnIndex int16 +} + +func newBooleanPage(typ Type, columnIndex int16, numValues int32, values encoding.Values) *booleanPage { + return &booleanPage{ + typ: typ, + bits: values.Boolean()[:bitpack.ByteCount(uint(numValues))], + numValues: numValues, + columnIndex: ^columnIndex, + } +} + +func (page *booleanPage) Type() Type { return page.typ } + +func (page *booleanPage) Column() int { return int(^page.columnIndex) } + +func (page *booleanPage) Dictionary() Dictionary { return nil } + +func (page *booleanPage) NumRows() int64 { return int64(page.numValues) } + +func (page *booleanPage) NumValues() int64 { return int64(page.numValues) } + +func (page *booleanPage) NumNulls() int64 { return 0 } + +func (page *booleanPage) Size() int64 { return int64(len(page.bits)) } + +func (page *booleanPage) RepetitionLevels() []byte { return nil } + +func (page *booleanPage) DefinitionLevels() []byte { return nil } + +func (page *booleanPage) Data() encoding.Values { return encoding.BooleanValues(page.bits) } + +func (page *booleanPage) Values() ValueReader { return &booleanPageValues{page: page} } + +func (page *booleanPage) valueAt(i int) bool { + j := uint32(int(page.offset)+i) / 8 + k := uint32(int(page.offset)+i) % 8 + return ((page.bits[j] >> k) & 1) != 0 +} + +func (page *booleanPage) min() bool { + for i := range int(page.numValues) { + if !page.valueAt(i) { + return false + } + } + return page.numValues > 0 +} + +func (page *booleanPage) max() bool { + for i := range int(page.numValues) { + if page.valueAt(i) { + return true + } + } + return false +} + +func (page *booleanPage) bounds() (min, max bool) { + hasFalse, hasTrue := false, false + + for i := range int(page.numValues) { + v := page.valueAt(i) + if v { + hasTrue = true + } else { + hasFalse = true + } + if hasTrue && hasFalse { + break + } + } + + min = !hasFalse + max = hasTrue + return min, max +} + +func (page *booleanPage) Bounds() (min, max Value, ok bool) { + if ok = page.numValues > 0; ok { + minBool, maxBool := page.bounds() + min = page.makeValue(minBool) + max = page.makeValue(maxBool) + } + return min, max, ok +} + +func (page *booleanPage) Slice(i, j int64) Page { + lowWithOffset := i + int64(page.offset) + highWithOffset := j + int64(page.offset) + + off := lowWithOffset / 8 + end := highWithOffset / 8 + + if (highWithOffset % 8) != 0 { + end++ + } + + return &booleanPage{ + typ: page.typ, + bits: page.bits[off:end], + offset: int32(lowWithOffset % 8), + numValues: int32(j - i), + columnIndex: page.columnIndex, + } +} + +func (page *booleanPage) makeValue(v bool) Value { + value := makeValueBoolean(v) + value.columnIndex = page.columnIndex + return value +} + +type int32Page struct { + typ Type + values []int32 + columnIndex int16 +} + +func newInt32Page(typ Type, columnIndex int16, numValues int32, values encoding.Values) *int32Page { + return &int32Page{ + typ: typ, + values: values.Int32()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *int32Page) Type() Type { return page.typ } + +func (page *int32Page) Column() int { return int(^page.columnIndex) } + +func (page *int32Page) Dictionary() Dictionary { return nil } + +func (page *int32Page) NumRows() int64 { return int64(len(page.values)) } + +func (page *int32Page) NumValues() int64 { return int64(len(page.values)) } + +func (page *int32Page) NumNulls() int64 { return 0 } + +func (page *int32Page) Size() int64 { return 4 * int64(len(page.values)) } + +func (page *int32Page) RepetitionLevels() []byte { return nil } + +func (page *int32Page) DefinitionLevels() []byte { return nil } + +func (page *int32Page) Data() encoding.Values { return encoding.Int32Values(page.values) } + +func (page *int32Page) Values() ValueReader { return &int32PageValues{page: page} } + +func (page *int32Page) min() int32 { return minInt32(page.values) } + +func (page *int32Page) max() int32 { return maxInt32(page.values) } + +func (page *int32Page) bounds() (min, max int32) { return boundsInt32(page.values) } + +func (page *int32Page) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minInt32, maxInt32 := page.bounds() + min = page.makeValue(minInt32) + max = page.makeValue(maxInt32) + } + return min, max, ok +} + +func (page *int32Page) Slice(i, j int64) Page { + return &int32Page{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *int32Page) makeValue(v int32) Value { + value := makeValueInt32(v) + value.columnIndex = page.columnIndex + return value +} + +type int64Page struct { + typ Type + values []int64 + columnIndex int16 +} + +func newInt64Page(typ Type, columnIndex int16, numValues int32, values encoding.Values) *int64Page { + return &int64Page{ + typ: typ, + values: values.Int64()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *int64Page) Type() Type { return page.typ } + +func (page *int64Page) Column() int { return int(^page.columnIndex) } + +func (page *int64Page) Dictionary() Dictionary { return nil } + +func (page *int64Page) NumRows() int64 { return int64(len(page.values)) } + +func (page *int64Page) NumValues() int64 { return int64(len(page.values)) } + +func (page *int64Page) NumNulls() int64 { return 0 } + +func (page *int64Page) Size() int64 { return 8 * int64(len(page.values)) } + +func (page *int64Page) RepetitionLevels() []byte { return nil } + +func (page *int64Page) DefinitionLevels() []byte { return nil } + +func (page *int64Page) Data() encoding.Values { return encoding.Int64Values(page.values) } + +func (page *int64Page) Values() ValueReader { return &int64PageValues{page: page} } + +func (page *int64Page) min() int64 { return minInt64(page.values) } + +func (page *int64Page) max() int64 { return maxInt64(page.values) } + +func (page *int64Page) bounds() (min, max int64) { return boundsInt64(page.values) } + +func (page *int64Page) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minInt64, maxInt64 := page.bounds() + min = page.makeValue(minInt64) + max = page.makeValue(maxInt64) + } + return min, max, ok +} + +func (page *int64Page) Slice(i, j int64) Page { + return &int64Page{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *int64Page) makeValue(v int64) Value { + value := makeValueInt64(v) + value.columnIndex = page.columnIndex + return value +} + +type int96Page struct { + typ Type + values []deprecated.Int96 + columnIndex int16 +} + +func newInt96Page(typ Type, columnIndex int16, numValues int32, values encoding.Values) *int96Page { + return &int96Page{ + typ: typ, + values: values.Int96()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *int96Page) Type() Type { return page.typ } + +func (page *int96Page) Column() int { return int(^page.columnIndex) } + +func (page *int96Page) Dictionary() Dictionary { return nil } + +func (page *int96Page) NumRows() int64 { return int64(len(page.values)) } + +func (page *int96Page) NumValues() int64 { return int64(len(page.values)) } + +func (page *int96Page) NumNulls() int64 { return 0 } + +func (page *int96Page) Size() int64 { return 12 * int64(len(page.values)) } + +func (page *int96Page) RepetitionLevels() []byte { return nil } + +func (page *int96Page) DefinitionLevels() []byte { return nil } + +func (page *int96Page) Data() encoding.Values { return encoding.Int96Values(page.values) } + +func (page *int96Page) Values() ValueReader { return &int96PageValues{page: page} } + +func (page *int96Page) min() deprecated.Int96 { return deprecated.MinInt96(page.values) } + +func (page *int96Page) max() deprecated.Int96 { return deprecated.MaxInt96(page.values) } + +func (page *int96Page) bounds() (min, max deprecated.Int96) { + return deprecated.MinMaxInt96(page.values) +} + +func (page *int96Page) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minInt96, maxInt96 := page.bounds() + min = page.makeValue(minInt96) + max = page.makeValue(maxInt96) + } + return min, max, ok +} + +func (page *int96Page) Slice(i, j int64) Page { + return &int96Page{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *int96Page) makeValue(v deprecated.Int96) Value { + value := makeValueInt96(v) + value.columnIndex = page.columnIndex + return value +} + +type floatPage struct { + typ Type + values []float32 + columnIndex int16 +} + +func newFloatPage(typ Type, columnIndex int16, numValues int32, values encoding.Values) *floatPage { + return &floatPage{ + typ: typ, + values: values.Float()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *floatPage) Type() Type { return page.typ } + +func (page *floatPage) Column() int { return int(^page.columnIndex) } + +func (page *floatPage) Dictionary() Dictionary { return nil } + +func (page *floatPage) NumRows() int64 { return int64(len(page.values)) } + +func (page *floatPage) NumValues() int64 { return int64(len(page.values)) } + +func (page *floatPage) NumNulls() int64 { return 0 } + +func (page *floatPage) Size() int64 { return 4 * int64(len(page.values)) } + +func (page *floatPage) RepetitionLevels() []byte { return nil } + +func (page *floatPage) DefinitionLevels() []byte { return nil } + +func (page *floatPage) Data() encoding.Values { return encoding.FloatValues(page.values) } + +func (page *floatPage) Values() ValueReader { return &floatPageValues{page: page} } + +func (page *floatPage) min() float32 { return minFloat32(page.values) } + +func (page *floatPage) max() float32 { return maxFloat32(page.values) } + +func (page *floatPage) bounds() (min, max float32) { return boundsFloat32(page.values) } + +func (page *floatPage) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minFloat32, maxFloat32 := page.bounds() + min = page.makeValue(minFloat32) + max = page.makeValue(maxFloat32) + } + return min, max, ok +} + +func (page *floatPage) Slice(i, j int64) Page { + return &floatPage{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *floatPage) makeValue(v float32) Value { + value := makeValueFloat(v) + value.columnIndex = page.columnIndex + return value +} + +type doublePage struct { + typ Type + values []float64 + columnIndex int16 +} + +func newDoublePage(typ Type, columnIndex int16, numValues int32, values encoding.Values) *doublePage { + return &doublePage{ + typ: typ, + values: values.Double()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *doublePage) Type() Type { return page.typ } + +func (page *doublePage) Column() int { return int(^page.columnIndex) } + +func (page *doublePage) Dictionary() Dictionary { return nil } + +func (page *doublePage) NumRows() int64 { return int64(len(page.values)) } + +func (page *doublePage) NumValues() int64 { return int64(len(page.values)) } + +func (page *doublePage) NumNulls() int64 { return 0 } + +func (page *doublePage) Size() int64 { return 8 * int64(len(page.values)) } + +func (page *doublePage) RepetitionLevels() []byte { return nil } + +func (page *doublePage) DefinitionLevels() []byte { return nil } + +func (page *doublePage) Data() encoding.Values { return encoding.DoubleValues(page.values) } + +func (page *doublePage) Values() ValueReader { return &doublePageValues{page: page} } + +func (page *doublePage) min() float64 { return minFloat64(page.values) } + +func (page *doublePage) max() float64 { return maxFloat64(page.values) } + +func (page *doublePage) bounds() (min, max float64) { return boundsFloat64(page.values) } + +func (page *doublePage) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minFloat64, maxFloat64 := page.bounds() + min = page.makeValue(minFloat64) + max = page.makeValue(maxFloat64) + } + return min, max, ok +} + +func (page *doublePage) Slice(i, j int64) Page { + return &doublePage{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *doublePage) makeValue(v float64) Value { + value := makeValueDouble(v) + value.columnIndex = page.columnIndex + return value +} + +type byteArrayPage struct { + typ Type + values []byte + offsets []uint32 + columnIndex int16 +} + +func newByteArrayPage(typ Type, columnIndex int16, numValues int32, values encoding.Values) *byteArrayPage { + data, offsets := values.ByteArray() + return &byteArrayPage{ + typ: typ, + values: data, + offsets: offsets[:numValues+1], + columnIndex: ^columnIndex, + } +} + +func (page *byteArrayPage) Type() Type { return page.typ } + +func (page *byteArrayPage) Column() int { return int(^page.columnIndex) } + +func (page *byteArrayPage) Dictionary() Dictionary { return nil } + +func (page *byteArrayPage) NumRows() int64 { return int64(page.len()) } + +func (page *byteArrayPage) NumValues() int64 { return int64(page.len()) } + +func (page *byteArrayPage) NumNulls() int64 { return 0 } + +func (page *byteArrayPage) Size() int64 { return int64(len(page.values)) + 4*int64(len(page.offsets)) } + +func (page *byteArrayPage) RepetitionLevels() []byte { return nil } + +func (page *byteArrayPage) DefinitionLevels() []byte { return nil } + +func (page *byteArrayPage) Data() encoding.Values { + return encoding.ByteArrayValues(page.values, page.offsets) +} + +func (page *byteArrayPage) Values() ValueReader { return &byteArrayPageValues{page: page} } + +func (page *byteArrayPage) len() int { return len(page.offsets) - 1 } + +func (page *byteArrayPage) index(i int) []byte { + j := page.offsets[i+0] + k := page.offsets[i+1] + return page.values[j:k:k] +} + +func (page *byteArrayPage) min() (min []byte) { + if n := page.len(); n > 0 { + min = page.index(0) + + for i := 1; i < n; i++ { + v := page.index(i) + + if bytes.Compare(v, min) < 0 { + min = v + } + } + } + return min +} + +func (page *byteArrayPage) max() (max []byte) { + if n := page.len(); n > 0 { + max = page.index(0) + + for i := 1; i < n; i++ { + v := page.index(i) + + if bytes.Compare(v, max) > 0 { + max = v + } + } + } + return max +} + +func (page *byteArrayPage) bounds() (min, max []byte) { + if n := page.len(); n > 0 { + min = page.index(0) + max = min + + for i := 1; i < n; i++ { + v := page.index(i) + + switch { + case bytes.Compare(v, min) < 0: + min = v + case bytes.Compare(v, max) > 0: + max = v + } + } + } + return min, max +} + +func (page *byteArrayPage) Bounds() (min, max Value, ok bool) { + if ok = len(page.offsets) > 1; ok { + minBytes, maxBytes := page.bounds() + min = page.makeValueBytes(minBytes) + max = page.makeValueBytes(maxBytes) + } + return min, max, ok +} + +func (page *byteArrayPage) cloneValues() []byte { + values := make([]byte, len(page.values)) + copy(values, page.values) + return values +} + +func (page *byteArrayPage) cloneOffsets() []uint32 { + offsets := make([]uint32, len(page.offsets)) + copy(offsets, page.offsets) + return offsets +} + +func (page *byteArrayPage) Slice(i, j int64) Page { + return &byteArrayPage{ + typ: page.typ, + values: page.values, + offsets: page.offsets[i : j+1], + columnIndex: page.columnIndex, + } +} + +func (page *byteArrayPage) makeValueBytes(v []byte) Value { + value := makeValueBytes(ByteArray, v) + value.columnIndex = page.columnIndex + return value +} + +func (page *byteArrayPage) makeValueString(v string) Value { + value := makeValueString(ByteArray, v) + value.columnIndex = page.columnIndex + return value +} + +type fixedLenByteArrayPage struct { + typ Type + data []byte + size int + columnIndex int16 +} + +func newFixedLenByteArrayPage(typ Type, columnIndex int16, numValues int32, values encoding.Values) *fixedLenByteArrayPage { + data, size := values.FixedLenByteArray() + return &fixedLenByteArrayPage{ + typ: typ, + data: data[:int(numValues)*size], + size: size, + columnIndex: ^columnIndex, + } +} + +func (page *fixedLenByteArrayPage) Type() Type { return page.typ } + +func (page *fixedLenByteArrayPage) Column() int { return int(^page.columnIndex) } + +func (page *fixedLenByteArrayPage) Dictionary() Dictionary { return nil } + +func (page *fixedLenByteArrayPage) NumRows() int64 { return int64(len(page.data) / page.size) } + +func (page *fixedLenByteArrayPage) NumValues() int64 { return int64(len(page.data) / page.size) } + +func (page *fixedLenByteArrayPage) NumNulls() int64 { return 0 } + +func (page *fixedLenByteArrayPage) Size() int64 { return int64(len(page.data)) } + +func (page *fixedLenByteArrayPage) RepetitionLevels() []byte { return nil } + +func (page *fixedLenByteArrayPage) DefinitionLevels() []byte { return nil } + +func (page *fixedLenByteArrayPage) Data() encoding.Values { + return encoding.FixedLenByteArrayValues(page.data, page.size) +} + +func (page *fixedLenByteArrayPage) Values() ValueReader { + return &fixedLenByteArrayPageValues{page: page} +} + +func (page *fixedLenByteArrayPage) min() []byte { return minFixedLenByteArray(page.data, page.size) } + +func (page *fixedLenByteArrayPage) max() []byte { return maxFixedLenByteArray(page.data, page.size) } + +func (page *fixedLenByteArrayPage) bounds() (min, max []byte) { + return boundsFixedLenByteArray(page.data, page.size) +} + +func (page *fixedLenByteArrayPage) Bounds() (min, max Value, ok bool) { + if ok = len(page.data) > 0; ok { + minBytes, maxBytes := page.bounds() + min = page.makeValueBytes(minBytes) + max = page.makeValueBytes(maxBytes) + } + return min, max, ok +} + +func (page *fixedLenByteArrayPage) Slice(i, j int64) Page { + return &fixedLenByteArrayPage{ + typ: page.typ, + data: page.data[i*int64(page.size) : j*int64(page.size)], + size: page.size, + columnIndex: page.columnIndex, + } +} + +func (page *fixedLenByteArrayPage) makeValueBytes(v []byte) Value { + value := makeValueBytes(FixedLenByteArray, v) + value.columnIndex = page.columnIndex + return value +} + +func (page *fixedLenByteArrayPage) makeValueString(v string) Value { + value := makeValueString(FixedLenByteArray, v) + value.columnIndex = page.columnIndex + return value +} + +type uint32Page struct { + typ Type + values []uint32 + columnIndex int16 +} + +func newUint32Page(typ Type, columnIndex int16, numValues int32, values encoding.Values) *uint32Page { + return &uint32Page{ + typ: typ, + values: values.Uint32()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *uint32Page) Type() Type { return page.typ } + +func (page *uint32Page) Column() int { return int(^page.columnIndex) } + +func (page *uint32Page) Dictionary() Dictionary { return nil } + +func (page *uint32Page) NumRows() int64 { return int64(len(page.values)) } + +func (page *uint32Page) NumValues() int64 { return int64(len(page.values)) } + +func (page *uint32Page) NumNulls() int64 { return 0 } + +func (page *uint32Page) Size() int64 { return 4 * int64(len(page.values)) } + +func (page *uint32Page) RepetitionLevels() []byte { return nil } + +func (page *uint32Page) DefinitionLevels() []byte { return nil } + +func (page *uint32Page) Data() encoding.Values { return encoding.Uint32Values(page.values) } + +func (page *uint32Page) Values() ValueReader { return &uint32PageValues{page: page} } + +func (page *uint32Page) min() uint32 { return minUint32(page.values) } + +func (page *uint32Page) max() uint32 { return maxUint32(page.values) } + +func (page *uint32Page) bounds() (min, max uint32) { return boundsUint32(page.values) } + +func (page *uint32Page) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minUint32, maxUint32 := page.bounds() + min = page.makeValue(minUint32) + max = page.makeValue(maxUint32) + } + return min, max, ok +} + +func (page *uint32Page) Slice(i, j int64) Page { + return &uint32Page{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *uint32Page) makeValue(v uint32) Value { + value := makeValueUint32(v) + value.columnIndex = page.columnIndex + return value +} + +type uint64Page struct { + typ Type + values []uint64 + columnIndex int16 +} + +func newUint64Page(typ Type, columnIndex int16, numValues int32, values encoding.Values) *uint64Page { + return &uint64Page{ + typ: typ, + values: values.Uint64()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *uint64Page) Type() Type { return page.typ } + +func (page *uint64Page) Column() int { return int(^page.columnIndex) } + +func (page *uint64Page) Dictionary() Dictionary { return nil } + +func (page *uint64Page) NumRows() int64 { return int64(len(page.values)) } + +func (page *uint64Page) NumValues() int64 { return int64(len(page.values)) } + +func (page *uint64Page) NumNulls() int64 { return 0 } + +func (page *uint64Page) Size() int64 { return 8 * int64(len(page.values)) } + +func (page *uint64Page) RepetitionLevels() []byte { return nil } + +func (page *uint64Page) DefinitionLevels() []byte { return nil } + +func (page *uint64Page) Data() encoding.Values { return encoding.Uint64Values(page.values) } + +func (page *uint64Page) Values() ValueReader { return &uint64PageValues{page: page} } + +func (page *uint64Page) min() uint64 { return minUint64(page.values) } + +func (page *uint64Page) max() uint64 { return maxUint64(page.values) } + +func (page *uint64Page) bounds() (min, max uint64) { return boundsUint64(page.values) } + +func (page *uint64Page) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minUint64, maxUint64 := page.bounds() + min = page.makeValue(minUint64) + max = page.makeValue(maxUint64) + } + return min, max, ok +} + +func (page *uint64Page) Slice(i, j int64) Page { + return &uint64Page{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *uint64Page) makeValue(v uint64) Value { + value := makeValueUint64(v) + value.columnIndex = page.columnIndex + return value +} + +type be128Page struct { + typ Type + values [][16]byte + columnIndex int16 +} + +func newBE128Page(typ Type, columnIndex int16, numValues int32, values encoding.Values) *be128Page { + return &be128Page{ + typ: typ, + values: values.Uint128()[:numValues], + columnIndex: ^columnIndex, + } +} + +func (page *be128Page) Type() Type { return page.typ } + +func (page *be128Page) Column() int { return int(^page.columnIndex) } + +func (page *be128Page) Dictionary() Dictionary { return nil } + +func (page *be128Page) NumRows() int64 { return int64(len(page.values)) } + +func (page *be128Page) NumValues() int64 { return int64(len(page.values)) } + +func (page *be128Page) NumNulls() int64 { return 0 } + +func (page *be128Page) Size() int64 { return 16 * int64(len(page.values)) } + +func (page *be128Page) RepetitionLevels() []byte { return nil } + +func (page *be128Page) DefinitionLevels() []byte { return nil } + +func (page *be128Page) Data() encoding.Values { return encoding.Uint128Values(page.values) } + +func (page *be128Page) Values() ValueReader { return &be128PageValues{page: page} } + +func (page *be128Page) min() []byte { return minBE128(page.values) } + +func (page *be128Page) max() []byte { return maxBE128(page.values) } + +func (page *be128Page) bounds() (min, max []byte) { return boundsBE128(page.values) } + +func (page *be128Page) Bounds() (min, max Value, ok bool) { + if ok = len(page.values) > 0; ok { + minBytes, maxBytes := page.bounds() + min = page.makeValueBytes(minBytes) + max = page.makeValueBytes(maxBytes) + } + return min, max, ok +} + +func (page *be128Page) Slice(i, j int64) Page { + return &be128Page{ + typ: page.typ, + values: page.values[i:j], + columnIndex: page.columnIndex, + } +} + +func (page *be128Page) makeValue(v *[16]byte) Value { + return page.makeValueBytes(v[:]) +} + +func (page *be128Page) makeValueBytes(v []byte) Value { + value := makeValueBytes(FixedLenByteArray, v) + value.columnIndex = page.columnIndex + return value +} + +func (page *be128Page) makeValueString(v string) Value { + value := makeValueString(FixedLenByteArray, v) + value.columnIndex = page.columnIndex + return value +} + +type nullPage struct { + typ Type + column int + count int +} + +func newNullPage(typ Type, columnIndex int16, numValues int32) *nullPage { + return &nullPage{ + typ: typ, + column: int(columnIndex), + count: int(numValues), + } +} + +func (page *nullPage) Type() Type { return page.typ } +func (page *nullPage) Column() int { return page.column } +func (page *nullPage) Dictionary() Dictionary { return nil } +func (page *nullPage) NumRows() int64 { return int64(page.count) } +func (page *nullPage) NumValues() int64 { return int64(page.count) } +func (page *nullPage) NumNulls() int64 { return int64(page.count) } +func (page *nullPage) Bounds() (min, max Value, ok bool) { return } +func (page *nullPage) Size() int64 { return 1 } +func (page *nullPage) Values() ValueReader { + return &nullPageValues{column: page.column, remain: page.count} +} +func (page *nullPage) Slice(i, j int64) Page { + return &nullPage{column: page.column, count: page.count - int(j-i)} +} +func (page *nullPage) RepetitionLevels() []byte { return nil } +func (page *nullPage) DefinitionLevels() []byte { return nil } +func (page *nullPage) Data() encoding.Values { return encoding.Values{} } diff --git a/vendor/github.com/parquet-go/parquet-go/page_bounds.go b/vendor/github.com/parquet-go/parquet-go/page_bounds.go new file mode 100644 index 0000000000..8278e25ce0 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_bounds.go @@ -0,0 +1,25 @@ +package parquet + +import "bytes" + +func boundsFixedLenByteArray(data []byte, size int) (min, max []byte) { + if len(data) > 0 { + min = data[:size] + max = data[:size] + + for i, j := size, 2*size; j <= len(data); { + item := data[i:j] + + if bytes.Compare(item, min) < 0 { + min = item + } + if bytes.Compare(item, max) > 0 { + max = item + } + + i += size + j += size + } + } + return min, max +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.go b/vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.go new file mode 100644 index 0000000000..9cb513b407 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.go @@ -0,0 +1,129 @@ +//go:build !purego + +package parquet + +// The min-max algorithms combine looking for the min and max values in a single +// pass over the data. While the behavior is the same as calling functions to +// look for the min and max values independently, doing both operations at the +// same time means that we only load the data from memory once. When working on +// large arrays the algorithms are limited by memory bandwidth, computing both +// the min and max together shrinks by half the amount of data read from memory. +// +// The following benchmarks results were highlighting the benefits of combining +// the min-max search, compared to calling the min and max functions separately: +// +// name old time/op new time/op delta +// BoundsInt64/10240KiB 590ยตs ยฑ15% 330ยตs ยฑ10% -44.01% (p=0.000 n=10+10) +// +// name old speed new speed delta +// BoundsInt64/10240KiB 17.9GB/s ยฑ13% 31.8GB/s ยฑ11% +78.13% (p=0.000 n=10+10) +// +// As expected, since the functions are memory-bound in those cases, and load +// half as much data, we see significant improvements. The gains are not 2x because +// running more AVX-512 instructions in the tight loops causes more contention +// on CPU ports. +// +// Optimizations being trade offs, using min/max functions independently appears +// to yield better throughput when the data resides in CPU caches: +// +// name old time/op new time/op delta +// BoundsInt64/4KiB 52.1ns ยฑ 0% 46.2ns ยฑ 1% -12.65% (p=0.000 n=10+10) +// +// name old speed new speed delta +// BoundsInt64/4KiB 78.6GB/s ยฑ 0% 88.6GB/s ยฑ 1% +11.23% (p=0.000 n=10+10) +// +// The probable explanation is that in those cases the algorithms are not +// memory-bound anymore, but limited by contention on CPU ports, and the +// individual min/max functions are able to better parallelize the work due +// to running less instructions per loop. The performance starts to equalize +// around 256KiB, and degrade beyond 1MiB, so we use this threshold to determine +// which approach to prefer. +const combinedBoundsThreshold = 1 * 1024 * 1024 + +//go:noescape +func combinedBoundsBool(data []bool) (min, max bool) + +//go:noescape +func combinedBoundsInt32(data []int32) (min, max int32) + +//go:noescape +func combinedBoundsInt64(data []int64) (min, max int64) + +//go:noescape +func combinedBoundsUint32(data []uint32) (min, max uint32) + +//go:noescape +func combinedBoundsUint64(data []uint64) (min, max uint64) + +//go:noescape +func combinedBoundsFloat32(data []float32) (min, max float32) + +//go:noescape +func combinedBoundsFloat64(data []float64) (min, max float64) + +//go:noescape +func combinedBoundsBE128(data [][16]byte) (min, max []byte) + +func boundsInt32(data []int32) (min, max int32) { + if 4*len(data) >= combinedBoundsThreshold { + return combinedBoundsInt32(data) + } + min = minInt32(data) + max = maxInt32(data) + return +} + +func boundsInt64(data []int64) (min, max int64) { + if 8*len(data) >= combinedBoundsThreshold { + return combinedBoundsInt64(data) + } + min = minInt64(data) + max = maxInt64(data) + return +} + +func boundsUint32(data []uint32) (min, max uint32) { + if 4*len(data) >= combinedBoundsThreshold { + return combinedBoundsUint32(data) + } + min = minUint32(data) + max = maxUint32(data) + return +} + +func boundsUint64(data []uint64) (min, max uint64) { + if 8*len(data) >= combinedBoundsThreshold { + return combinedBoundsUint64(data) + } + min = minUint64(data) + max = maxUint64(data) + return +} + +func boundsFloat32(data []float32) (min, max float32) { + if 4*len(data) >= combinedBoundsThreshold { + return combinedBoundsFloat32(data) + } + min = minFloat32(data) + max = maxFloat32(data) + return +} + +func boundsFloat64(data []float64) (min, max float64) { + if 8*len(data) >= combinedBoundsThreshold { + return combinedBoundsFloat64(data) + } + min = minFloat64(data) + max = maxFloat64(data) + return +} + +func boundsBE128(data [][16]byte) (min, max []byte) { + // TODO: min/max BE128 is really complex to vectorize, and the returns + // were barely better than doing the min and max independently, for all + // input sizes. We should revisit if we find ways to improve the min or + // max algorithms which can be transposed to the combined version. + min = minBE128(data) + max = maxBE128(data) + return +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.s b/vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.s new file mode 100644 index 0000000000..c6d172a5e1 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_bounds_amd64.s @@ -0,0 +1,551 @@ +//go:build !purego + +#include "textflag.h" + +#define bswap128lo 0x08080A0B0C0D0E0F +#define bswap128hi 0x0001020304050607 + +DATA bswap128+0(SB)/8, $bswap128lo +DATA bswap128+8(SB)/8, $bswap128hi +DATA bswap128+16(SB)/8, $bswap128lo +DATA bswap128+24(SB)/8, $bswap128hi +DATA bswap128+32(SB)/8, $bswap128lo +DATA bswap128+40(SB)/8, $bswap128hi +DATA bswap128+48(SB)/8, $bswap128lo +DATA bswap128+56(SB)/8, $bswap128hi +GLOBL bswap128(SB), RODATA|NOPTR, $64 + +DATA indexes128+0(SB)/8, $0 +DATA indexes128+8(SB)/8, $0 +DATA indexes128+16(SB)/8, $1 +DATA indexes128+24(SB)/8, $1 +DATA indexes128+32(SB)/8, $2 +DATA indexes128+40(SB)/8, $2 +DATA indexes128+48(SB)/8, $3 +DATA indexes128+56(SB)/8, $3 +GLOBL indexes128(SB), RODATA|NOPTR, $64 + +DATA swap64+0(SB)/8, $4 +DATA swap64+8(SB)/8, $5 +DATA swap64+16(SB)/8, $6 +DATA swap64+24(SB)/8, $7 +DATA swap64+32(SB)/8, $2 +DATA swap64+40(SB)/8, $3 +DATA swap64+48(SB)/8, $0 +DATA swap64+56(SB)/8, $1 +GLOBL swap64(SB), RODATA|NOPTR, $64 + +DATA swap32+0(SB)/4, $8 +DATA swap32+4(SB)/4, $9 +DATA swap32+8(SB)/4, $10 +DATA swap32+12(SB)/4, $11 +DATA swap32+16(SB)/4, $12 +DATA swap32+20(SB)/4, $13 +DATA swap32+24(SB)/4, $14 +DATA swap32+28(SB)/4, $15 +DATA swap32+32(SB)/4, $4 +DATA swap32+36(SB)/4, $5 +DATA swap32+40(SB)/4, $6 +DATA swap32+44(SB)/4, $7 +DATA swap32+48(SB)/4, $2 +DATA swap32+52(SB)/4, $3 +DATA swap32+56(SB)/4, $0 +DATA swap32+60(SB)/4, $1 +GLOBL swap32(SB), RODATA|NOPTR, $64 + +// func combinedBoundsInt32(data []int32) (min, max int32) +TEXT ยทcombinedBoundsInt32(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ R8, R8 + XORQ R9, R9 + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVLQZX (AX), R8 // min + MOVLQZX (AX), R9 // max + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 + VPBROADCASTD (AX), Z3 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VPMINSD Z1, Z0, Z0 + VPMINSD Z2, Z0, Z0 + VPMAXSD Z1, Z3, Z3 + VPMAXSD Z2, Z3, Z3 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VMOVDQU32 swap32+0(SB), Z2 + VPERMI2D Z0, Z0, Z1 + VPERMI2D Z3, Z3, Z2 + VPMINSD Y1, Y0, Y0 + VPMAXSD Y2, Y3, Y3 + + VMOVDQU32 swap32+32(SB), Y1 + VMOVDQU32 swap32+32(SB), Y2 + VPERMI2D Y0, Y0, Y1 + VPERMI2D Y3, Y3, Y2 + VPMINSD X1, X0, X0 + VPMAXSD X2, X3, X3 + + VMOVDQU32 swap32+48(SB), X1 + VMOVDQU32 swap32+48(SB), X2 + VPERMI2D X0, X0, X1 + VPERMI2D X3, X3, X2 + VPMINSD X1, X0, X0 + VPMAXSD X2, X3, X3 + VZEROUPPER + + MOVQ X0, BX + MOVQ X3, DX + MOVL BX, R8 + MOVL DX, R9 + SHRQ $32, BX + SHRQ $32, DX + CMPL BX, R8 + CMOVLLT BX, R8 + CMPL DX, R9 + CMOVLGT DX, R9 + + CMPQ SI, CX + JE done +loop: + MOVLQZX (AX)(SI*4), DX + CMPL DX, R8 + CMOVLLT DX, R8 + CMPL DX, R9 + CMOVLGT DX, R9 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL R8, min+24(FP) + MOVL R9, max+28(FP) + RET + +// func combinedBoundsInt64(data []int64) (min, max int64) +TEXT ยทcombinedBoundsInt64(SB), NOSPLIT, $-40 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ R8, R8 + XORQ R9, R9 + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVQ (AX), R8 // min + MOVQ (AX), R9 // max + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $16 + JB loop + + MOVQ CX, DI + SHRQ $4, DI + SHLQ $4, DI + VPBROADCASTQ (AX), Z0 + VPBROADCASTQ (AX), Z3 +loop16: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VPMINSQ Z1, Z0, Z0 + VPMINSQ Z2, Z0, Z0 + VPMAXSQ Z1, Z3, Z3 + VPMAXSQ Z2, Z3, Z3 + ADDQ $16, SI + CMPQ SI, DI + JNE loop16 + + VMOVDQU32 swap32+0(SB), Z1 + VMOVDQU32 swap32+0(SB), Z2 + VPERMI2D Z0, Z0, Z1 + VPERMI2D Z3, Z3, Z2 + VPMINSQ Y1, Y0, Y0 + VPMAXSQ Y2, Y3, Y3 + + VMOVDQU32 swap32+32(SB), Y1 + VMOVDQU32 swap32+32(SB), Y2 + VPERMI2D Y0, Y0, Y1 + VPERMI2D Y3, Y3, Y2 + VPMINSQ X1, X0, X0 + VPMAXSQ X2, X3, X3 + + VMOVDQU32 swap32+48(SB), X1 + VMOVDQU32 swap32+48(SB), X2 + VPERMI2D X0, X0, X1 + VPERMI2D X3, X3, X2 + VPMINSQ X1, X0, X0 + VPMAXSQ X2, X3, X3 + VZEROUPPER + + MOVQ X0, R8 + MOVQ X3, R9 + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + CMPQ DX, R8 + CMOVQLT DX, R8 + CMPQ DX, R9 + CMOVQGT DX, R9 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ R8, min+24(FP) + MOVQ R9, max+32(FP) + RET + +// func combinedBoundsUint32(data []uint32) (min, max uint32) +TEXT ยทcombinedBoundsUint32(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ R8, R8 + XORQ R9, R9 + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVLQZX (AX), R8 // min + MOVLQZX (AX), R9 // max + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 + VPBROADCASTD (AX), Z3 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VPMINUD Z1, Z0, Z0 + VPMINUD Z2, Z0, Z0 + VPMAXUD Z1, Z3, Z3 + VPMAXUD Z2, Z3, Z3 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VMOVDQU32 swap32+0(SB), Z2 + VPERMI2D Z0, Z0, Z1 + VPERMI2D Z3, Z3, Z2 + VPMINUD Y1, Y0, Y0 + VPMAXUD Y2, Y3, Y3 + + VMOVDQU32 swap32+32(SB), Y1 + VMOVDQU32 swap32+32(SB), Y2 + VPERMI2D Y0, Y0, Y1 + VPERMI2D Y3, Y3, Y2 + VPMINUD X1, X0, X0 + VPMAXUD X2, X3, X3 + + VMOVDQU32 swap32+48(SB), X1 + VMOVDQU32 swap32+48(SB), X2 + VPERMI2D X0, X0, X1 + VPERMI2D X3, X3, X2 + VPMINUD X1, X0, X0 + VPMAXUD X2, X3, X3 + VZEROUPPER + + MOVQ X0, BX + MOVQ X3, DX + MOVL BX, R8 + MOVL DX, R9 + SHRQ $32, BX + SHRQ $32, DX + CMPL BX, R8 + CMOVLCS BX, R8 + CMPL DX, R9 + CMOVLHI DX, R9 + + CMPQ SI, CX + JE done +loop: + MOVLQZX (AX)(SI*4), DX + CMPL DX, R8 + CMOVLCS DX, R8 + CMPL DX, R9 + CMOVLHI DX, R9 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL R8, min+24(FP) + MOVL R9, max+28(FP) + RET + +// func combinedBoundsUint64(data []uint64) (min, max uint64) +TEXT ยทcombinedBoundsUint64(SB), NOSPLIT, $-40 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ R8, R8 + XORQ R9, R9 + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVQ (AX), R8 // min + MOVQ (AX), R9 // max + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $16 + JB loop + + MOVQ CX, DI + SHRQ $4, DI + SHLQ $4, DI + VPBROADCASTQ (AX), Z0 + VPBROADCASTQ (AX), Z3 +loop16: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VPMINUQ Z1, Z0, Z0 + VPMINUQ Z2, Z0, Z0 + VPMAXUQ Z1, Z3, Z3 + VPMAXUQ Z2, Z3, Z3 + ADDQ $16, SI + CMPQ SI, DI + JNE loop16 + + VMOVDQU32 swap32+0(SB), Z1 + VMOVDQU32 swap32+0(SB), Z2 + VPERMI2D Z0, Z0, Z1 + VPERMI2D Z3, Z3, Z2 + VPMINUQ Y1, Y0, Y0 + VPMAXUQ Y2, Y3, Y3 + + VMOVDQU32 swap32+32(SB), Y1 + VMOVDQU32 swap32+32(SB), Y2 + VPERMI2D Y0, Y0, Y1 + VPERMI2D Y3, Y3, Y2 + VPMINUQ X1, X0, X0 + VPMAXUQ X2, X3, X3 + + VMOVDQU32 swap32+48(SB), X1 + VMOVDQU32 swap32+48(SB), X2 + VPERMI2D X0, X0, X1 + VPERMI2D X3, X3, X2 + VPMINUQ X1, X0, X0 + VPMAXUQ X2, X3, X3 + VZEROUPPER + + MOVQ X0, R8 + MOVQ X3, R9 + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + CMPQ DX, R8 + CMOVQCS DX, R8 + CMPQ DX, R9 + CMOVQHI DX, R9 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ R8, min+24(FP) + MOVQ R9, max+32(FP) + RET + +// func combinedBoundsFloat32(data []float32) (min, max float32) +TEXT ยทcombinedBoundsFloat32(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ R8, R8 + XORQ R9, R9 + + CMPQ CX, $0 + JE done + XORPS X0, X0 + XORPS X1, X1 + XORQ SI, SI + MOVLQZX (AX), R8 // min + MOVLQZX (AX), R9 // max + MOVQ R8, X0 + MOVQ R9, X1 + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 + VPBROADCASTD (AX), Z3 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VMINPS Z1, Z0, Z0 + VMINPS Z2, Z0, Z0 + VMAXPS Z1, Z3, Z3 + VMAXPS Z2, Z3, Z3 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VMOVDQU32 swap32+0(SB), Z2 + VPERMI2D Z0, Z0, Z1 + VPERMI2D Z3, Z3, Z2 + VMINPS Y1, Y0, Y0 + VMAXPS Y2, Y3, Y3 + + VMOVDQU32 swap32+32(SB), Y1 + VMOVDQU32 swap32+32(SB), Y2 + VPERMI2D Y0, Y0, Y1 + VPERMI2D Y3, Y3, Y2 + VMINPS X1, X0, X0 + VMAXPS X2, X3, X3 + + VMOVDQU32 swap32+48(SB), X1 + VMOVDQU32 swap32+48(SB), X2 + VPERMI2D X0, X0, X1 + VPERMI2D X3, X3, X2 + VMINPS X1, X0, X0 + VMAXPS X2, X3, X3 + VZEROUPPER + + MOVAPS X0, X1 + MOVAPS X3, X2 + + PSRLQ $32, X1 + MOVQ X0, R8 + MOVQ X1, R10 + UCOMISS X0, X1 + CMOVLCS R10, R8 + + PSRLQ $32, X2 + MOVQ X3, R9 + MOVQ X2, R11 + UCOMISS X3, X2 + CMOVLHI R11, R9 + + CMPQ SI, CX + JE done + MOVQ R8, X0 + MOVQ R9, X1 +loop: + MOVLQZX (AX)(SI*4), DX + MOVQ DX, X2 + UCOMISS X0, X2 + CMOVLCS DX, R8 + UCOMISS X1, X2 + CMOVLHI DX, R9 + MOVQ R8, X0 + MOVQ R9, X1 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL R8, min+24(FP) + MOVL R9, max+28(FP) + RET + +// func combinedBoundsFloat64(data []float64) (min, max float64) +TEXT ยทcombinedBoundsFloat64(SB), NOSPLIT, $-40 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ R8, R8 + XORQ R9, R9 + + CMPQ CX, $0 + JE done + XORPD X0, X0 + XORPD X1, X1 + XORQ SI, SI + MOVQ (AX), R8 // min + MOVQ (AX), R9 // max + MOVQ R8, X0 + MOVQ R9, X1 + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $16 + JB loop + + MOVQ CX, DI + SHRQ $4, DI + SHLQ $4, DI + VPBROADCASTQ (AX), Z0 + VPBROADCASTQ (AX), Z3 +loop16: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMINPD Z1, Z0, Z0 + VMINPD Z2, Z0, Z0 + VMAXPD Z1, Z3, Z3 + VMAXPD Z2, Z3, Z3 + ADDQ $16, SI + CMPQ SI, DI + JNE loop16 + + VMOVDQU64 swap32+0(SB), Z1 + VMOVDQU64 swap32+0(SB), Z2 + VPERMI2D Z0, Z0, Z1 + VPERMI2D Z3, Z3, Z2 + VMINPD Y1, Y0, Y0 + VMAXPD Y2, Y3, Y3 + + VMOVDQU64 swap32+32(SB), Y1 + VMOVDQU64 swap32+32(SB), Y2 + VPERMI2D Y0, Y0, Y1 + VPERMI2D Y3, Y3, Y2 + VMINPD X1, X0, X0 + VMAXPD X2, X3, X3 + + VMOVDQU64 swap32+48(SB), X1 + VMOVDQU64 swap32+48(SB), X2 + VPERMI2D X0, X0, X1 + VPERMI2D X3, X3, X2 + VMINPD X1, X0, X0 + VMAXPD X2, X3, X1 + VZEROUPPER + + MOVQ X0, R8 + MOVQ X1, R9 + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + MOVQ DX, X2 + UCOMISD X0, X2 + CMOVQCS DX, R8 + UCOMISD X1, X2 + CMOVQHI DX, R9 + MOVQ R8, X0 + MOVQ R9, X1 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ R8, min+24(FP) + MOVQ R9, max+32(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/page_bounds_purego.go b/vendor/github.com/parquet-go/parquet-go/page_bounds_purego.go new file mode 100644 index 0000000000..0b5a2474f8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_bounds_purego.go @@ -0,0 +1,143 @@ +//go:build purego || !amd64 + +package parquet + +import ( + "encoding/binary" +) + +func boundsInt32(data []int32) (min, max int32) { + if len(data) > 0 { + min = data[0] + max = data[0] + + for _, v := range data[1:] { + if v < min { + min = v + } + if v > max { + max = v + } + } + } + return min, max +} + +func boundsInt64(data []int64) (min, max int64) { + if len(data) > 0 { + min = data[0] + max = data[0] + + for _, v := range data[1:] { + if v < min { + min = v + } + if v > max { + max = v + } + } + } + return min, max +} + +func boundsUint32(data []uint32) (min, max uint32) { + if len(data) > 0 { + min = data[0] + max = data[0] + + for _, v := range data[1:] { + if v < min { + min = v + } + if v > max { + max = v + } + } + } + return min, max +} + +func boundsUint64(data []uint64) (min, max uint64) { + if len(data) > 0 { + min = data[0] + max = data[0] + + for _, v := range data[1:] { + if v < min { + min = v + } + if v > max { + max = v + } + } + } + return min, max +} + +func boundsFloat32(data []float32) (min, max float32) { + if len(data) > 0 { + min = data[0] + max = data[0] + + for _, v := range data[1:] { + if v < min { + min = v + } + if v > max { + max = v + } + } + } + return min, max +} + +func boundsFloat64(data []float64) (min, max float64) { + if len(data) > 0 { + min = data[0] + max = data[0] + + for _, v := range data[1:] { + if v < min { + min = v + } + if v > max { + max = v + } + } + } + return min, max +} + +func boundsBE128(data [][16]byte) (min, max []byte) { + if len(data) > 0 { + minHi := binary.BigEndian.Uint64(data[0][:8]) + maxHi := minHi + minIndex := 0 + maxIndex := 0 + for i := 1; i < len(data); i++ { + hi := binary.BigEndian.Uint64(data[i][:8]) + lo := binary.BigEndian.Uint64(data[i][8:]) + switch { + case hi < minHi: + minHi, minIndex = hi, i + case hi == minHi: + minLo := binary.BigEndian.Uint64(data[minIndex][8:]) + if lo < minLo { + minHi, minIndex = hi, i + } + } + switch { + case hi > maxHi: + maxHi, maxIndex = hi, i + case hi == maxHi: + maxLo := binary.BigEndian.Uint64(data[maxIndex][8:]) + if lo > maxLo { + maxHi, maxIndex = hi, i + } + } + } + min = data[minIndex][:] + max = data[maxIndex][:] + } + return min, max +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_header.go b/vendor/github.com/parquet-go/parquet-go/page_header.go new file mode 100644 index 0000000000..14f912b11b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_header.go @@ -0,0 +1,221 @@ +package parquet + +import ( + "fmt" + + "github.com/parquet-go/parquet-go/format" +) + +// PageHeader is an interface implemented by parquet page headers. +type PageHeader interface { + // Returns the number of values in the page (including nulls). + NumValues() int64 + + // Returns the page encoding. + Encoding() format.Encoding + + // Returns the parquet format page type. + PageType() format.PageType +} + +// DataPageHeader is a specialization of the PageHeader interface implemented by +// data pages. +type DataPageHeader interface { + PageHeader + + // Returns the encoding of the repetition level section. + RepetitionLevelEncoding() format.Encoding + + // Returns the encoding of the definition level section. + DefinitionLevelEncoding() format.Encoding + + // Returns the number of null values in the page. + NullCount() int64 + + // Returns the minimum value in the page based on the ordering rules of the + // column's logical type. + // + // As an optimization, the method may return the same slice across multiple + // calls. Programs must treat the returned value as immutable to prevent + // unpredictable behaviors. + // + // If the page only contains only null values, an empty slice is returned. + MinValue() []byte + + // Returns the maximum value in the page based on the ordering rules of the + // column's logical type. + // + // As an optimization, the method may return the same slice across multiple + // calls. Programs must treat the returned value as immutable to prevent + // unpredictable behaviors. + // + // If the page only contains only null values, an empty slice is returned. + MaxValue() []byte +} + +// DictionaryPageHeader is an implementation of the PageHeader interface +// representing dictionary pages. +type DictionaryPageHeader struct { + header *format.DictionaryPageHeader +} + +func (dict DictionaryPageHeader) NumValues() int64 { + return int64(dict.header.NumValues) +} + +func (dict DictionaryPageHeader) Encoding() format.Encoding { + return dict.header.Encoding +} + +func (dict DictionaryPageHeader) PageType() format.PageType { + return format.DictionaryPage +} + +func (dict DictionaryPageHeader) IsSorted() bool { + return dict.header.IsSorted +} + +func (dict DictionaryPageHeader) String() string { + return fmt.Sprintf("DICTIONARY_PAGE_HEADER{NumValues=%d,Encoding=%s,IsSorted=%t}", + dict.header.NumValues, + dict.header.Encoding, + dict.header.IsSorted) +} + +// DataPageHeaderV1 is an implementation of the DataPageHeader interface +// representing data pages version 1. +type DataPageHeaderV1 struct { + header *format.DataPageHeader +} + +func (v1 DataPageHeaderV1) NumValues() int64 { + return int64(v1.header.NumValues) +} + +func (v1 DataPageHeaderV1) RepetitionLevelEncoding() format.Encoding { + return v1.header.RepetitionLevelEncoding +} + +func (v1 DataPageHeaderV1) DefinitionLevelEncoding() format.Encoding { + return v1.header.DefinitionLevelEncoding +} + +func (v1 DataPageHeaderV1) Encoding() format.Encoding { + return v1.header.Encoding +} + +func (v1 DataPageHeaderV1) PageType() format.PageType { + return format.DataPage +} + +func (v1 DataPageHeaderV1) NullCount() int64 { + return v1.header.Statistics.NullCount +} + +func (v1 DataPageHeaderV1) MinValue() []byte { + return v1.header.Statistics.MinValue +} + +func (v1 DataPageHeaderV1) MaxValue() []byte { + return v1.header.Statistics.MaxValue +} + +func (v1 DataPageHeaderV1) String() string { + return fmt.Sprintf("DATA_PAGE_HEADER{NumValues=%d,Encoding=%s}", + v1.header.NumValues, + v1.header.Encoding) +} + +// DataPageHeaderV2 is an implementation of the DataPageHeader interface +// representing data pages version 2. +type DataPageHeaderV2 struct { + header *format.DataPageHeaderV2 +} + +func (v2 DataPageHeaderV2) NumValues() int64 { + return int64(v2.header.NumValues) +} + +func (v2 DataPageHeaderV2) NumNulls() int64 { + return int64(v2.header.NumNulls) +} + +func (v2 DataPageHeaderV2) NumRows() int64 { + return int64(v2.header.NumRows) +} + +func (v2 DataPageHeaderV2) RepetitionLevelsByteLength() int64 { + return int64(v2.header.RepetitionLevelsByteLength) +} + +func (v2 DataPageHeaderV2) DefinitionLevelsByteLength() int64 { + return int64(v2.header.DefinitionLevelsByteLength) +} + +func (v2 DataPageHeaderV2) RepetitionLevelEncoding() format.Encoding { + return format.RLE +} + +func (v2 DataPageHeaderV2) DefinitionLevelEncoding() format.Encoding { + return format.RLE +} + +func (v2 DataPageHeaderV2) Encoding() format.Encoding { + return v2.header.Encoding +} + +func (v2 DataPageHeaderV2) PageType() format.PageType { + return format.DataPageV2 +} + +func (v2 DataPageHeaderV2) NullCount() int64 { + return v2.header.Statistics.NullCount +} + +func (v2 DataPageHeaderV2) MinValue() []byte { + return v2.header.Statistics.MinValue +} + +func (v2 DataPageHeaderV2) MaxValue() []byte { + return v2.header.Statistics.MaxValue +} + +func (v2 DataPageHeaderV2) IsCompressed() bool { + return v2.header.IsCompressed == nil || *v2.header.IsCompressed +} + +func (v2 DataPageHeaderV2) String() string { + return fmt.Sprintf("DATA_PAGE_HEADER_V2{NumValues=%d,NumNulls=%d,NumRows=%d,Encoding=%s,IsCompressed=%t}", + v2.header.NumValues, + v2.header.NumNulls, + v2.header.NumRows, + v2.header.Encoding, + v2.IsCompressed()) +} + +type unknownPageHeader struct { + header *format.PageHeader +} + +func (u unknownPageHeader) NumValues() int64 { + return 0 +} + +func (u unknownPageHeader) Encoding() format.Encoding { + return -1 +} + +func (u unknownPageHeader) PageType() format.PageType { + return u.header.Type +} + +func (u unknownPageHeader) String() string { + return fmt.Sprintf("UNKNOWN_PAGE_HEADER{Type=%d}", u.header.Type) +} + +var ( + _ PageHeader = DictionaryPageHeader{} + _ DataPageHeader = DataPageHeaderV1{} + _ DataPageHeader = DataPageHeaderV2{} + _ PageHeader = unknownPageHeader{} +) diff --git a/vendor/github.com/parquet-go/parquet-go/page_max.go b/vendor/github.com/parquet-go/parquet-go/page_max.go new file mode 100644 index 0000000000..f6afecf6a6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_max.go @@ -0,0 +1,23 @@ +package parquet + +import ( + "bytes" +) + +func maxFixedLenByteArray(data []byte, size int) (max []byte) { + if len(data) > 0 { + max = data[:size] + + for i, j := size, 2*size; j <= len(data); { + item := data[i:j] + + if bytes.Compare(item, max) > 0 { + max = item + } + + i += size + j += size + } + } + return max +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_max_amd64.go b/vendor/github.com/parquet-go/parquet-go/page_max_amd64.go new file mode 100644 index 0000000000..2ac1de2f06 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_max_amd64.go @@ -0,0 +1,24 @@ +//go:build !purego + +package parquet + +//go:noescape +func maxInt32(data []int32) int32 + +//go:noescape +func maxInt64(data []int64) int64 + +//go:noescape +func maxUint32(data []uint32) uint32 + +//go:noescape +func maxUint64(data []uint64) uint64 + +//go:noescape +func maxFloat32(data []float32) float32 + +//go:noescape +func maxFloat64(data []float64) float64 + +//go:noescape +func maxBE128(data [][16]byte) []byte diff --git a/vendor/github.com/parquet-go/parquet-go/page_max_amd64.s b/vendor/github.com/parquet-go/parquet-go/page_max_amd64.s new file mode 100644 index 0000000000..8159583d1f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_max_amd64.s @@ -0,0 +1,598 @@ +//go:build !purego + +#include "textflag.h" + +// func maxInt32(data []int32) int32 +TEXT ยทmaxInt32(SB), NOSPLIT, $-28 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVLQZX (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VPMAXSD Z1, Z0, Z0 + VPMAXSD Z2, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMAXSD Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMAXSD X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMAXSD X1, X0, X0 + VZEROUPPER + + MOVQ X0, DX + MOVL DX, BX + SHRQ $32, DX + CMPL DX, BX + CMOVLGT DX, BX + + CMPQ SI, CX + JE done +loop: + MOVLQZX (AX)(SI*4), DX + CMPL DX, BX + CMOVLGT DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL BX, ret+24(FP) + RET + +// func maxInt64(data []int64) int64 +TEXT ยทmaxInt64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVQ (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTQ (AX), Z0 +loop32: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMOVDQU64 128(AX)(SI*8), Z3 + VMOVDQU64 192(AX)(SI*8), Z4 + VPMAXSQ Z1, Z2, Z5 + VPMAXSQ Z3, Z4, Z6 + VPMAXSQ Z5, Z6, Z1 + VPMAXSQ Z1, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMAXSQ Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMAXSQ X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMAXSQ X1, X0, X0 + VZEROUPPER + + MOVQ X0, BX + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + CMPQ DX, BX + CMOVQGT DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ BX, ret+24(FP) + RET + +// func maxUint32(data []int32) int32 +TEXT ยทmaxUint32(SB), NOSPLIT, $-28 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVLQZX (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VPMAXUD Z1, Z0, Z0 + VPMAXUD Z2, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMAXUD Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMAXUD X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMAXUD X1, X0, X0 + VZEROUPPER + + MOVQ X0, DX + MOVL DX, BX + SHRQ $32, DX + CMPL DX, BX + CMOVLHI DX, BX + + CMPQ SI, CX + JE done +loop: + MOVLQZX (AX)(SI*4), DX + CMPL DX, BX + CMOVLHI DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL BX, ret+24(FP) + RET + +// func maxUint64(data []uint64) uint64 +TEXT ยทmaxUint64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVQ (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTQ (AX), Z0 +loop32: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMOVDQU64 128(AX)(SI*8), Z3 + VMOVDQU64 192(AX)(SI*8), Z4 + VPMAXUQ Z1, Z2, Z5 + VPMAXUQ Z3, Z4, Z6 + VPMAXUQ Z5, Z6, Z1 + VPMAXUQ Z1, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMAXUQ Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMAXUQ X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMAXUQ X1, X0, X0 + VZEROUPPER + + MOVQ X0, BX + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + CMPQ DX, BX + CMOVQHI DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ BX, ret+24(FP) + RET + +// func maxFloat32(data []float32) float32 +TEXT ยทmaxFloat32(SB), NOSPLIT, $-28 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORPS X0, X0 + XORPS X1, X1 + XORQ SI, SI + MOVLQZX (AX), BX + MOVQ BX, X0 + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $64 + JB loop + + MOVQ CX, DI + SHRQ $6, DI + SHLQ $6, DI + VPBROADCASTD (AX), Z0 +loop64: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VMOVDQU32 128(AX)(SI*4), Z3 + VMOVDQU32 192(AX)(SI*4), Z4 + VMAXPS Z1, Z2, Z5 + VMAXPS Z3, Z4, Z6 + VMAXPS Z5, Z6, Z1 + VMAXPS Z1, Z0, Z0 + ADDQ $64, SI + CMPQ SI, DI + JNE loop64 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VMAXPS Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VMAXPS X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VMAXPS X1, X0, X0 + VZEROUPPER + + MOVAPS X0, X1 + PSRLQ $32, X1 + MOVQ X0, BX + MOVQ X1, DX + UCOMISS X0, X1 + CMOVLHI DX, BX + + CMPQ SI, CX + JE done + MOVQ BX, X0 +loop: + MOVLQZX (AX)(SI*4), DX + MOVQ DX, X1 + UCOMISS X0, X1 + CMOVLHI DX, BX + MOVQ BX, X0 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL BX, ret+24(FP) + RET + +// func maxFloat64(data []float64) float64 +TEXT ยทmaxFloat64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORPD X0, X0 + XORPD X1, X1 + XORQ SI, SI + MOVQ (AX), BX + MOVQ BX, X0 + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTQ (AX), Z0 +loop32: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMOVDQU64 128(AX)(SI*8), Z3 + VMOVDQU64 192(AX)(SI*8), Z4 + VMAXPD Z1, Z2, Z5 + VMAXPD Z3, Z4, Z6 + VMAXPD Z5, Z6, Z1 + VMAXPD Z1, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU64 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VMAXPD Y1, Y0, Y0 + + VMOVDQU64 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VMAXPD X1, X0, X0 + + VMOVDQU64 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VMAXPD X1, X0, X0 + VZEROUPPER + + MOVQ X0, BX + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + MOVQ DX, X1 + UCOMISD X0, X1 + CMOVQHI DX, BX + MOVQ BX, X0 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ BX, ret+24(FP) + RET + +// vpmaxu128 is a macro comparing unsigned 128 bits values held in the +// `srcValues` and `maxValues` vectors. The `srcIndexes` and `maxIndexes` +// vectors contain the indexes of elements in the value vectors. Remaining +// K and R arguments are mask and general purpose registers needed to hold +// temporary values during the computation. The last M argument is a mask +// generated by vpmaxu128mask. +// +// The routine uses AVX-512 instructions (VPCMPUQ, VPBLENDMQ) to implement +// the comparison of 128 bits values. The values are expected to be stored +// in the vectors as a little-endian pair of two consecutive quad words. +// +// The results are written to the `maxValues` and `maxIndexes` vectors, +// overwriting the inputs. `srcValues` and `srcIndexes` are read-only +// parameters. +// +// At a high level, for two pairs of quad words formaxg two 128 bits values +// A and B, the test implemented by this macro is: +// +// A[1] > B[1] || (A[1] == B[1] && A[0] > B[0]) +// +// Values in the source vector that evaluate to true on this expression are +// written to the vector of maximum values, and their indexes are written to +// the vector of indexes. +#define vpmaxu128(srcValues, srcIndexes, maxValues, maxIndexes, K1, K2, R1, R2, R3, M) \ + VPCMPUQ $0, maxValues, srcValues, K1 \ + VPCMPUQ $6, maxValues, srcValues, K2 \ + KMOVB K1, R1 \ + KMOVB K2, R2 \ + MOVB R2, R3 \ + SHLB $1, R3 \ + ANDB R3, R1 \ + ORB R2, R1 \ + ANDB M, R1 \ + MOVB R1, R2 \ + SHRB $1, R2 \ + ORB R2, R1 \ + KMOVB R1, K1 \ + VPBLENDMQ srcValues, maxValues, K1, maxValues \ + VPBLENDMQ srcIndexes, maxIndexes, K1, maxIndexes + +// vpmaxu128mask is a macro used to initialize the mask passed as last argument +// to vpmaxu128. The argument M is intended to be a general purpose register. +// +// The bit mask is used to merge the results of the "greater than" and "equal" +// comparison that are performed on each lane of maximum vectors. The upper bits +// are used to compute results of the operation to determine which of the pairs +// of quad words representing the 128 bits elements are the maximums. +#define vpmaxu128mask(M) MOVB $0b10101010, M + +// func maxBE128(data [][16]byte) []byte +TEXT ยทmaxBE128(SB), NOSPLIT, $-48 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + CMPQ CX, $0 + JE null + + SHLQ $4, CX + MOVQ CX, DX // len + MOVQ AX, BX // max + ADDQ AX, CX // end + + CMPQ DX, $256 + JB loop + + CMPB ยทhasAVX512MinMaxBE128(SB), $0 + JE loop + + // Z19 holds a vector of the count by which we increment the vectors of + // swap at each loop iteration. + MOVQ $16, DI + VPBROADCASTQ DI, Z19 + + // Z31 holds the shuffle mask used to convert 128 bits elements from big to + // little endian so we can apply vectorized comparison instructions. + VMOVDQU64 bswap128(SB), Z31 + + // These vectors hold four lanes of maximum values found in the input. + VBROADCASTI64X2 (AX), Z0 + VPSHUFB Z31, Z0, Z0 + VMOVDQU64 Z0, Z5 + VMOVDQU64 Z0, Z10 + VMOVDQU64 Z0, Z15 + + // These vectors hold four lanes of swap of maximum values. + // + // We initialize them at zero because we broadcast the first value of the + // input in the vectors that track the maximums of each lane; in other + // words, we assume the maximum value is at the first offset and work our + // way up from there. + VPXORQ Z2, Z2, Z2 + VPXORQ Z7, Z7, Z7 + VPXORQ Z12, Z12, Z12 + VPXORQ Z17, Z17, Z17 + + // These vectors are used to compute the swap of maximum values held + // in [Z1, Z5, Z10, Z15]. Each vector holds a contiguous sequence of + // swap; for example, Z3 is initialized with [0, 1, 2, 3]. At each + // loop iteration, the swap are incremented by the number of elements + // consumed from the input (4x4=16). + VMOVDQU64 indexes128(SB), Z3 + VPXORQ Z8, Z8, Z8 + VPXORQ Z13, Z13, Z13 + VPXORQ Z18, Z18, Z18 + MOVQ $4, DI + VPBROADCASTQ DI, Z1 + VPADDQ Z1, Z3, Z8 + VPADDQ Z1, Z8, Z13 + VPADDQ Z1, Z13, Z18 + + // This bit mask is used to merge the results of the "less than" and "equal" + // comparison that we perform on each lane of maximum vectors. We use the + // upper bits to compute four results of the operation which determines + // which of the pair of quad words representing the 128 bits elements is the + // maximum. + vpmaxu128mask(DI) + SHRQ $8, DX + SHLQ $8, DX + ADDQ AX, DX +loop16: + // Compute 4x4 maximum values in vector registers, along with their swap + // in the input array. + VMOVDQU64 (AX), Z1 + VMOVDQU64 64(AX), Z6 + VMOVDQU64 128(AX), Z11 + VMOVDQU64 192(AX), Z16 + VPSHUFB Z31, Z1, Z1 + VPSHUFB Z31, Z6, Z6 + VPSHUFB Z31, Z11, Z11 + VPSHUFB Z31, Z16, Z16 + vpmaxu128(Z1, Z3, Z0, Z2, K1, K2, R8, R9, R10, DI) + vpmaxu128(Z6, Z8, Z5, Z7, K3, K4, R11, R12, R13, DI) + vpmaxu128(Z11, Z13, Z10, Z12, K1, K2, R8, R9, R10, DI) + vpmaxu128(Z16, Z18, Z15, Z17, K3, K4, R11, R12, R13, DI) + VPADDQ Z19, Z3, Z3 + VPADDQ Z19, Z8, Z8 + VPADDQ Z19, Z13, Z13 + VPADDQ Z19, Z18, Z18 + ADDQ $256, AX + CMPQ AX, DX + JB loop16 + + // After the loop completed, we need to merge the lanes that each contain + // 4 maximum values (so 16 total candidate at this stage). The results are + // reduced into 4 candidates in Z0, with their swap in Z2. + vpmaxu128(Z10, Z12, Z0, Z2, K1, K2, R8, R9, R10, DI) + vpmaxu128(Z15, Z17, Z5, Z7, K3, K4, R11, R12, R13, DI) + vpmaxu128(Z5, Z7, Z0, Z2, K1, K2, R8, R9, R10, DI) + + // Further reduce the results by swapping the upper and lower parts of the + // vector registers, and comparing them to determaxe which values are the + // smallest. We compare 2x2 values at this step, then 2x1 values at the next + // to find the index of the maximum. + VMOVDQU64 swap64+0(SB), Z1 + VMOVDQU64 swap64+0(SB), Z3 + VPERMI2Q Z0, Z0, Z1 + VPERMI2Q Z2, Z2, Z3 + vpmaxu128(Y1, Y3, Y0, Y2, K1, K2, R8, R9, R10, DI) + + VMOVDQU64 swap64+32(SB), Y1 + VMOVDQU64 swap64+32(SB), Y3 + VPERMI2Q Y0, Y0, Y1 + VPERMI2Q Y2, Y2, Y3 + vpmaxu128(X1, X3, X0, X2, K1, K2, R8, R9, R10, DI) + VZEROUPPER + + // Extract the index of the maximum value computed in the lower 64 bits of + // X2 and position the BX pointer at the index of the maximum value. + MOVQ X2, DX + SHLQ $4, DX + ADDQ DX, BX + CMPQ AX, CX + JE done + + // Unless the input was aligned on 256 bytes, we need to perform a few more + // iterations on the remaining elements. + // + // This loop is also taken if the CPU has no support for AVX-512. +loop: + MOVQ (AX), R8 + MOVQ (BX), R9 + BSWAPQ R8 + BSWAPQ R9 + CMPQ R8, R9 + JA more + JB next + MOVQ 8(AX), R8 + MOVQ 8(BX), R9 + BSWAPQ R8 + BSWAPQ R9 + CMPQ R8, R9 + JBE next +more: + MOVQ AX, BX +next: + ADDQ $16, AX + CMPQ AX, CX + JB loop +done: + MOVQ BX, ret_base+24(FP) + MOVQ $16, ret_len+32(FP) + MOVQ $16, ret_cap+40(FP) + RET +null: + XORQ BX, BX + MOVQ BX, ret_base+24(FP) + MOVQ BX, ret_len+32(FP) + MOVQ BX, ret_cap+40(FP) + RET + diff --git a/vendor/github.com/parquet-go/parquet-go/page_max_purego.go b/vendor/github.com/parquet-go/parquet-go/page_max_purego.go new file mode 100644 index 0000000000..e3f828811f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_max_purego.go @@ -0,0 +1,110 @@ +//go:build purego || !amd64 + +package parquet + +import "encoding/binary" + +// ----------------------------------------------------------------------------- +// TODO: use generics versions of the these functions to reduce the amount of +// code to maintain when we drop compatilibty with Go version older than 1.18. +// ----------------------------------------------------------------------------- + +func maxInt32(data []int32) (max int32) { + if len(data) > 0 { + max = data[0] + + for _, value := range data { + if value > max { + max = value + } + } + } + return max +} + +func maxInt64(data []int64) (max int64) { + if len(data) > 0 { + max = data[0] + + for _, value := range data { + if value > max { + max = value + } + } + } + return max +} + +func maxUint32(data []uint32) (max uint32) { + if len(data) > 0 { + max = data[0] + + for _, value := range data { + if value > max { + max = value + } + } + } + return max +} + +func maxUint64(data []uint64) (max uint64) { + if len(data) > 0 { + max = data[0] + + for _, value := range data { + if value > max { + max = value + } + } + } + return max +} + +func maxFloat32(data []float32) (max float32) { + if len(data) > 0 { + max = data[0] + + for _, value := range data { + if value > max { + max = value + } + } + } + return max +} + +func maxFloat64(data []float64) (max float64) { + if len(data) > 0 { + max = data[0] + + for _, value := range data { + if value > max { + max = value + } + } + } + return max +} + +func maxBE128(data [][16]byte) (min []byte) { + if len(data) > 0 { + m := binary.BigEndian.Uint64(data[0][:8]) + j := 0 + for i := 1; i < len(data); i++ { + x := binary.BigEndian.Uint64(data[i][:8]) + switch { + case x > m: + m, j = x, i + case x == m: + y := binary.BigEndian.Uint64(data[i][8:]) + n := binary.BigEndian.Uint64(data[j][8:]) + if y > n { + m, j = x, i + } + } + } + min = data[j][:] + } + return min +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_min.go b/vendor/github.com/parquet-go/parquet-go/page_min.go new file mode 100644 index 0000000000..2bc185ac28 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_min.go @@ -0,0 +1,23 @@ +package parquet + +import ( + "bytes" +) + +func minFixedLenByteArray(data []byte, size int) (min []byte) { + if len(data) > 0 { + min = data[:size] + + for i, j := size, 2*size; j <= len(data); { + item := data[i:j] + + if bytes.Compare(item, min) < 0 { + min = item + } + + i += size + j += size + } + } + return min +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_min_amd64.go b/vendor/github.com/parquet-go/parquet-go/page_min_amd64.go new file mode 100644 index 0000000000..c9fd654753 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_min_amd64.go @@ -0,0 +1,24 @@ +//go:build !purego + +package parquet + +//go:noescape +func minInt32(data []int32) int32 + +//go:noescape +func minInt64(data []int64) int64 + +//go:noescape +func minUint32(data []uint32) uint32 + +//go:noescape +func minUint64(data []uint64) uint64 + +//go:noescape +func minFloat32(data []float32) float32 + +//go:noescape +func minFloat64(data []float64) float64 + +//go:noescape +func minBE128(data [][16]byte) []byte diff --git a/vendor/github.com/parquet-go/parquet-go/page_min_amd64.s b/vendor/github.com/parquet-go/parquet-go/page_min_amd64.s new file mode 100644 index 0000000000..7bd2e3a872 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_min_amd64.s @@ -0,0 +1,592 @@ +//go:build !purego + +#include "textflag.h" + +// func minInt32(data []int32) int32 +TEXT ยทminInt32(SB), NOSPLIT, $-28 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVLQZX (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VPMINSD Z1, Z0, Z0 + VPMINSD Z2, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMINSD Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMINSD X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMINSD X1, X0, X0 + VZEROUPPER + + MOVQ X0, DX + MOVL DX, BX + SHRQ $32, DX + CMPL DX, BX + CMOVLLT DX, BX + + CMPQ SI, CX + JE done +loop: + MOVLQZX (AX)(SI*4), DX + CMPL DX, BX + CMOVLLT DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL BX, ret+24(FP) + RET + +// func minInt64(data []int64) int64 +TEXT ยทminInt64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVQ (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTQ (AX), Z0 +loop32: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMOVDQU64 128(AX)(SI*8), Z3 + VMOVDQU64 192(AX)(SI*8), Z4 + VPMINSQ Z1, Z2, Z5 + VPMINSQ Z3, Z4, Z6 + VPMINSQ Z5, Z6, Z1 + VPMINSQ Z1, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMINSQ Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMINSQ X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMINSQ X1, X0, X0 + VZEROUPPER + + MOVQ X0, BX + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + CMPQ DX, BX + CMOVQLT DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ BX, ret+24(FP) + RET + +// func minUint32(data []int32) int32 +TEXT ยทminUint32(SB), NOSPLIT, $-28 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVLQZX (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTD (AX), Z0 +loop32: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VPMINUD Z1, Z0, Z0 + VPMINUD Z2, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMINUD Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMINUD X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMINUD X1, X0, X0 + VZEROUPPER + + MOVQ X0, DX + MOVL DX, BX + SHRQ $32, DX + CMPL DX, BX + CMOVLCS DX, BX + + CMPQ SI, CX + JE done +loop: + MOVLQZX (AX)(SI*4), DX + CMPL DX, BX + CMOVLCS DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL BX, ret+24(FP) + RET + +// func minUint64(data []uint64) uint64 +TEXT ยทminUint64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORQ SI, SI + MOVQ (AX), BX + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTQ (AX), Z0 +loop32: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMOVDQU64 128(AX)(SI*8), Z3 + VMOVDQU64 192(AX)(SI*8), Z4 + VPMINUQ Z1, Z2, Z5 + VPMINUQ Z3, Z4, Z6 + VPMINUQ Z5, Z6, Z1 + VPMINUQ Z1, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VPMINUQ Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VPMINUQ X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VPMINUQ X1, X0, X0 + VZEROUPPER + + MOVQ X0, BX + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + CMPQ DX, BX + CMOVQCS DX, BX + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ BX, ret+24(FP) + RET + +// func minFloat32(data []float32) float32 +TEXT ยทminFloat32(SB), NOSPLIT, $-28 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORPS X0, X0 + XORPS X1, X1 + XORQ SI, SI + MOVLQZX (AX), BX + MOVQ BX, X0 + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $64 + JB loop + + MOVQ CX, DI + SHRQ $6, DI + SHLQ $6, DI + VPBROADCASTD (AX), Z0 +loop64: + VMOVDQU32 (AX)(SI*4), Z1 + VMOVDQU32 64(AX)(SI*4), Z2 + VMOVDQU32 128(AX)(SI*4), Z3 + VMOVDQU32 192(AX)(SI*4), Z4 + VMINPS Z1, Z2, Z5 + VMINPS Z3, Z4, Z6 + VMINPS Z5, Z6, Z1 + VMINPS Z1, Z0, Z0 + ADDQ $64, SI + CMPQ SI, DI + JNE loop64 + + VMOVDQU32 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VMINPS Y1, Y0, Y0 + + VMOVDQU32 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VMINPS X1, X0, X0 + + VMOVDQU32 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VMINPS X1, X0, X0 + VZEROUPPER + + MOVAPS X0, X1 + PSRLQ $32, X1 + MOVQ X0, BX + MOVQ X1, DX + UCOMISS X0, X1 + CMOVLCS DX, BX + + CMPQ SI, CX + JE done + MOVQ BX, X0 +loop: + MOVLQZX (AX)(SI*4), DX + MOVQ DX, X1 + UCOMISS X0, X1 + CMOVLCS DX, BX + MOVQ BX, X0 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVL BX, ret+24(FP) + RET + +// func minFloat64(data []float64) float64 +TEXT ยทminFloat64(SB), NOSPLIT, $-32 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + XORQ BX, BX + + CMPQ CX, $0 + JE done + XORPD X0, X0 + XORPD X1, X1 + XORQ SI, SI + MOVQ (AX), BX + MOVQ BX, X0 + + CMPB ยทhasAVX512VL(SB), $0 + JE loop + + CMPQ CX, $32 + JB loop + + MOVQ CX, DI + SHRQ $5, DI + SHLQ $5, DI + VPBROADCASTQ (AX), Z0 +loop32: + VMOVDQU64 (AX)(SI*8), Z1 + VMOVDQU64 64(AX)(SI*8), Z2 + VMOVDQU64 128(AX)(SI*8), Z3 + VMOVDQU64 192(AX)(SI*8), Z4 + VMINPD Z1, Z2, Z5 + VMINPD Z3, Z4, Z6 + VMINPD Z5, Z6, Z1 + VMINPD Z1, Z0, Z0 + ADDQ $32, SI + CMPQ SI, DI + JNE loop32 + + VMOVDQU64 swap32+0(SB), Z1 + VPERMI2D Z0, Z0, Z1 + VMINPD Y1, Y0, Y0 + + VMOVDQU64 swap32+32(SB), Y1 + VPERMI2D Y0, Y0, Y1 + VMINPD X1, X0, X0 + + VMOVDQU64 swap32+48(SB), X1 + VPERMI2D X0, X0, X1 + VMINPD X1, X0, X0 + VZEROUPPER + + MOVQ X0, BX + CMPQ SI, CX + JE done +loop: + MOVQ (AX)(SI*8), DX + MOVQ DX, X1 + UCOMISD X0, X1 + CMOVQCS DX, BX + MOVQ BX, X0 + INCQ SI + CMPQ SI, CX + JNE loop +done: + MOVQ BX, ret+24(FP) + RET + +// vpminu128 is a macro comparing unsigned 128 bits values held in the +// `srcValues` and `minValues` vectors. The `srcIndexes` and `minIndexes` +// vectors contain the indexes of elements in the value vectors. Remaining +// K and R arguments are mask and general purpose registers needed to hold +// temporary values during the computation. The last M argument is a mask +// generated by vpminu128mask. +// +// The routine uses AVX-512 instructions (VPCMPUQ, VPBLENDMQ) to implement +// the comparison of 128 bits values. The values are expected to be stored +// in the vectors as a little-endian pair of two consecutive quad words. +// +// The results are written to the `minValues` and `minIndexes` vectors, +// overwriting the inputs. `srcValues` and `srcIndexes` are read-only +// parameters. +// +// At a high level, for two pairs of quad words forming two 128 bits values +// A and B, the test implemented by this macro is: +// +// A[1] < B[1] || (A[1] == B[1] && A[0] < B[0]) +// +// Values in the source vector that evalute to true on this expression are +// written to the vector of minimum values, and their indexes are written to +// the vector of indexes. +#define vpminu128(srcValues, srcIndexes, minValues, minIndexes, K1, K2, R1, R2, R3, M) \ + VPCMPUQ $0, minValues, srcValues, K1 \ + VPCMPUQ $1, minValues, srcValues, K2 \ + KMOVB K1, R1 \ + KMOVB K2, R2 \ + MOVB R2, R3 \ + SHLB $1, R3 \ + ANDB R3, R1 \ + ORB R2, R1 \ + ANDB M, R1 \ + MOVB R1, R2 \ + SHRB $1, R2 \ + ORB R2, R1 \ + KMOVB R1, K1 \ + VPBLENDMQ srcValues, minValues, K1, minValues \ + VPBLENDMQ srcIndexes, minIndexes, K1, minIndexes + +// vpminu128mask is a macro used to initialize the mask passed as last argument +// to vpminu128. The argument M is intended to be a general purpose register. +// +// The bit mask is used to merge the results of the "less than" and "equal" +// comparison that are performed on each lane of minimum vectors. The upper bits +// are used to compute results of the operation to determines which of the pairs +// of quad words representing the 128 bits elements are the minimums. +#define vpminu128mask(M) MOVB $0b10101010, M + +// func minBE128(data [][16]byte) []byte +TEXT ยทminBE128(SB), NOSPLIT, $-48 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + CMPQ CX, $0 + JE null + + SHLQ $4, CX + MOVQ CX, DX // len + MOVQ AX, BX // min + ADDQ AX, CX // end + + CMPQ DX, $256 + JB loop + + CMPB ยทhasAVX512MinMaxBE128(SB), $0 + JE loop + + // Z19 holds a vector of the count by which we increment the vectors of + // swap at each loop iteration. + MOVQ $16, DI + VPBROADCASTQ DI, Z19 + + // Z31 holds the shuffle mask used to convert 128 bits elements from big to + // little endian so we can apply vectorized comparison instructions. + VMOVDQU64 bswap128(SB), Z31 + + // These vectors hold four lanes of minimum values found in the input. + VBROADCASTI64X2 (AX), Z0 + VPSHUFB Z31, Z0, Z0 + VMOVDQU64 Z0, Z5 + VMOVDQU64 Z0, Z10 + VMOVDQU64 Z0, Z15 + + // These vectors hold four lanes of swap of minimum values. + // + // We initialize them at zero because we broadcast the first value of the + // input in the vectors that track the minimums of each lane; in other + // words, we assume the minimum value is at the first offset and work our + // way up from there. + VPXORQ Z2, Z2, Z2 + VPXORQ Z7, Z7, Z7 + VPXORQ Z12, Z12, Z12 + VPXORQ Z17, Z17, Z17 + + // These vectors are used to compute the swap of minimum values held + // in [Z1, Z5, Z10, Z15]. Each vector holds a contiguous sequence of + // swap; for example, Z3 is initialized with [0, 1, 2, 3]. At each + // loop iteration, the swap are incremented by the number of elements + // consumed from the input (4x4=16). + VMOVDQU64 indexes128(SB), Z3 + VPXORQ Z8, Z8, Z8 + VPXORQ Z13, Z13, Z13 + VPXORQ Z18, Z18, Z18 + MOVQ $4, DI + VPBROADCASTQ DI, Z1 + VPADDQ Z1, Z3, Z8 + VPADDQ Z1, Z8, Z13 + VPADDQ Z1, Z13, Z18 + + vpminu128mask(DI) + SHRQ $8, DX + SHLQ $8, DX + ADDQ AX, DX +loop16: + // Compute 4x4 minimum values in vector registers, along with their swap + // in the input array. + VMOVDQU64 (AX), Z1 + VMOVDQU64 64(AX), Z6 + VMOVDQU64 128(AX), Z11 + VMOVDQU64 192(AX), Z16 + VPSHUFB Z31, Z1, Z1 + VPSHUFB Z31, Z6, Z6 + VPSHUFB Z31, Z11, Z11 + VPSHUFB Z31, Z16, Z16 + vpminu128(Z1, Z3, Z0, Z2, K1, K2, R8, R9, R10, DI) + vpminu128(Z6, Z8, Z5, Z7, K3, K4, R11, R12, R13, DI) + vpminu128(Z11, Z13, Z10, Z12, K1, K2, R8, R9, R10, DI) + vpminu128(Z16, Z18, Z15, Z17, K3, K4, R11, R12, R13, DI) + VPADDQ Z19, Z3, Z3 + VPADDQ Z19, Z8, Z8 + VPADDQ Z19, Z13, Z13 + VPADDQ Z19, Z18, Z18 + ADDQ $256, AX + CMPQ AX, DX + JB loop16 + + // After the loop completed, we need to merge the lanes that each contain + // 4 minimum values (so 16 total candidate at this stage). The results are + // reduced into 4 candidates in Z0, with their swap in Z2. + vpminu128(Z10, Z12, Z0, Z2, K1, K2, R8, R9, R10, DI) + vpminu128(Z15, Z17, Z5, Z7, K3, K4, R11, R12, R13, DI) + vpminu128(Z5, Z7, Z0, Z2, K1, K2, R8, R9, R10, DI) + + // Further reduce the results by swapping the upper and lower parts of the + // vector registers, and comparing them to determine which values are the + // smallest. We compare 2x2 values at this step, then 2x1 values at the next + // to find the index of the minimum. + VMOVDQU64 swap64+0(SB), Z1 + VMOVDQU64 swap64+0(SB), Z3 + VPERMI2Q Z0, Z0, Z1 + VPERMI2Q Z2, Z2, Z3 + vpminu128(Y1, Y3, Y0, Y2, K1, K2, R8, R9, R10, DI) + + VMOVDQU64 swap64+32(SB), Y1 + VMOVDQU64 swap64+32(SB), Y3 + VPERMI2Q Y0, Y0, Y1 + VPERMI2Q Y2, Y2, Y3 + vpminu128(X1, X3, X0, X2, K1, K2, R8, R9, R10, DI) + VZEROUPPER + + // Extract the index of the minimum value computed in the lower 64 bits of + // X2 and position the BX pointer at the index of the minimum value. + MOVQ X2, DX + SHLQ $4, DX + ADDQ DX, BX + CMPQ AX, CX + JE done + + // Unless the input was aligned on 256 bytes, we need to perform a few more + // iterations on the remaining elements. + // + // This loop is also taken if the CPU has no support for AVX-512. +loop: + MOVQ (AX), R8 + MOVQ (BX), R9 + BSWAPQ R8 + BSWAPQ R9 + CMPQ R8, R9 + JB less + JA next + MOVQ 8(AX), R8 + MOVQ 8(BX), R9 + BSWAPQ R8 + BSWAPQ R9 + CMPQ R8, R9 + JAE next +less: + MOVQ AX, BX +next: + ADDQ $16, AX + CMPQ AX, CX + JB loop +done: + MOVQ BX, ret_base+24(FP) + MOVQ $16, ret_len+32(FP) + MOVQ $16, ret_cap+40(FP) + RET +null: + XORQ BX, BX + MOVQ BX, ret_base+24(FP) + MOVQ BX, ret_len+32(FP) + MOVQ BX, ret_cap+40(FP) + RET diff --git a/vendor/github.com/parquet-go/parquet-go/page_min_purego.go b/vendor/github.com/parquet-go/parquet-go/page_min_purego.go new file mode 100644 index 0000000000..e68afbb6d8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_min_purego.go @@ -0,0 +1,110 @@ +//go:build purego || !amd64 + +package parquet + +import "encoding/binary" + +// ----------------------------------------------------------------------------- +// TODO: use generics versions of the these functions to reduce the amount of +// code to maintain when we drop compatilibty with Go version older than 1.18. +// ----------------------------------------------------------------------------- + +func minInt32(data []int32) (min int32) { + if len(data) > 0 { + min = data[0] + + for _, value := range data { + if value < min { + min = value + } + } + } + return min +} + +func minInt64(data []int64) (min int64) { + if len(data) > 0 { + min = data[0] + + for _, value := range data { + if value < min { + min = value + } + } + } + return min +} + +func minUint32(data []uint32) (min uint32) { + if len(data) > 0 { + min = data[0] + + for _, value := range data { + if value < min { + min = value + } + } + } + return min +} + +func minUint64(data []uint64) (min uint64) { + if len(data) > 0 { + min = data[0] + + for _, value := range data { + if value < min { + min = value + } + } + } + return min +} + +func minFloat32(data []float32) (min float32) { + if len(data) > 0 { + min = data[0] + + for _, value := range data { + if value < min { + min = value + } + } + } + return min +} + +func minFloat64(data []float64) (min float64) { + if len(data) > 0 { + min = data[0] + + for _, value := range data { + if value < min { + min = value + } + } + } + return min +} + +func minBE128(data [][16]byte) (min []byte) { + if len(data) > 0 { + m := binary.BigEndian.Uint64(data[0][:8]) + j := 0 + for i := 1; i < len(data); i++ { + x := binary.BigEndian.Uint64(data[i][:8]) + switch { + case x < m: + m, j = x, i + case x == m: + y := binary.BigEndian.Uint64(data[i][8:]) + n := binary.BigEndian.Uint64(data[j][8:]) + if y < n { + m, j = x, i + } + } + } + min = data[j][:] + } + return min +} diff --git a/vendor/github.com/parquet-go/parquet-go/page_values.go b/vendor/github.com/parquet-go/parquet-go/page_values.go new file mode 100644 index 0000000000..ecbdffb0c8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/page_values.go @@ -0,0 +1,487 @@ +package parquet + +import ( + "io" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding/plain" + "github.com/parquet-go/parquet-go/internal/unsafecast" +) + +type optionalPageValues struct { + page *optionalPage + values ValueReader + offset int +} + +func (r *optionalPageValues) ReadValues(values []Value) (n int, err error) { + maxDefinitionLevel := r.page.maxDefinitionLevel + definitionLevels := r.page.definitionLevels + columnIndex := ^int16(r.page.Column()) + + for n < len(values) && r.offset < len(definitionLevels) { + for n < len(values) && r.offset < len(definitionLevels) && definitionLevels[r.offset] != maxDefinitionLevel { + values[n] = Value{ + definitionLevel: definitionLevels[r.offset], + columnIndex: columnIndex, + } + r.offset++ + n++ + } + + i := n + j := r.offset + for i < len(values) && j < len(definitionLevels) && definitionLevels[j] == maxDefinitionLevel { + i++ + j++ + } + + if n < i { + for j, err = r.values.ReadValues(values[n:i]); j > 0; j-- { + values[n].definitionLevel = maxDefinitionLevel + r.offset++ + n++ + } + // Do not return on an io.EOF here as we may still have null values to read. + if err != nil && err != io.EOF { + return n, err + } + err = nil + } + } + + if r.offset == len(definitionLevels) { + err = io.EOF + } + return n, err +} + +type repeatedPageValues struct { + page *repeatedPage + values ValueReader + offset int +} + +func (r *repeatedPageValues) ReadValues(values []Value) (n int, err error) { + maxDefinitionLevel := r.page.maxDefinitionLevel + definitionLevels := r.page.definitionLevels + repetitionLevels := r.page.repetitionLevels + columnIndex := ^int16(r.page.Column()) + + // While we haven't exceeded the output buffer and we haven't exceeded the page size. + for n < len(values) && r.offset < len(definitionLevels) { + + // While we haven't exceeded the output buffer and we haven't exceeded the + // page size AND the current element's definitionLevel is not the + // maxDefinitionLevel (this is a null value), Create the zero values to be + // returned in this run. + for n < len(values) && r.offset < len(definitionLevels) && definitionLevels[r.offset] != maxDefinitionLevel { + values[n] = Value{ + repetitionLevel: repetitionLevels[r.offset], + definitionLevel: definitionLevels[r.offset], + columnIndex: columnIndex, + } + r.offset++ + n++ + } + + i := n + j := r.offset + // Get the length of the run of non-zero values to be copied. + for i < len(values) && j < len(definitionLevels) && definitionLevels[j] == maxDefinitionLevel { + i++ + j++ + } + + // Copy all the non-zero values in this run. + if n < i { + for j, err = r.values.ReadValues(values[n:i]); j > 0; j-- { + values[n].repetitionLevel = repetitionLevels[r.offset] + values[n].definitionLevel = maxDefinitionLevel + r.offset++ + n++ + } + if err != nil && err != io.EOF { + return n, err + } + err = nil + } + } + + if r.offset == len(definitionLevels) { + err = io.EOF + } + return n, err +} + +type booleanPageValues struct { + page *booleanPage + offset int +} + +func (r *booleanPageValues) ReadBooleans(values []bool) (n int, err error) { + for n < len(values) && r.offset < int(r.page.numValues) { + values[n] = r.page.valueAt(r.offset) + r.offset++ + n++ + } + if r.offset == int(r.page.numValues) { + err = io.EOF + } + return n, err +} + +func (r *booleanPageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < int(r.page.numValues) { + values[n] = r.page.makeValue(r.page.valueAt(r.offset)) + r.offset++ + n++ + } + if r.offset == int(r.page.numValues) { + err = io.EOF + } + return n, err +} + +type int32PageValues struct { + page *int32Page + offset int +} + +func (r *int32PageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadInt32s(unsafecast.Slice[int32](b)) + return 4 * n, err +} + +func (r *int32PageValues) ReadInt32s(values []int32) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *int32PageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type int64PageValues struct { + page *int64Page + offset int +} + +func (r *int64PageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadInt64s(unsafecast.Slice[int64](b)) + return 8 * n, err +} + +func (r *int64PageValues) ReadInt64s(values []int64) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *int64PageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type int96PageValues struct { + page *int96Page + offset int +} + +func (r *int96PageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadInt96s(unsafecast.Slice[deprecated.Int96](b)) + return 12 * n, err +} + +func (r *int96PageValues) ReadInt96s(values []deprecated.Int96) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *int96PageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type floatPageValues struct { + page *floatPage + offset int +} + +func (r *floatPageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadFloats(unsafecast.Slice[float32](b)) + return 4 * n, err +} + +func (r *floatPageValues) ReadFloats(values []float32) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *floatPageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type doublePageValues struct { + page *doublePage + offset int +} + +func (r *doublePageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadDoubles(unsafecast.Slice[float64](b)) + return 8 * n, err +} + +func (r *doublePageValues) ReadDoubles(values []float64) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *doublePageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type byteArrayPageValues struct { + page *byteArrayPage + offset int +} + +func (r *byteArrayPageValues) Read(b []byte) (int, error) { + _, n, err := r.readByteArrays(b) + return n, err +} + +func (r *byteArrayPageValues) ReadRequired(values []byte) (int, error) { + return r.ReadByteArrays(values) +} + +func (r *byteArrayPageValues) ReadByteArrays(values []byte) (int, error) { + n, _, err := r.readByteArrays(values) + return n, err +} + +func (r *byteArrayPageValues) readByteArrays(values []byte) (c, n int, err error) { + numValues := r.page.len() + for r.offset < numValues { + b := r.page.index(r.offset) + k := plain.ByteArrayLengthSize + len(b) + if k > (len(values) - n) { + break + } + plain.PutByteArrayLength(values[n:], len(b)) + n += plain.ByteArrayLengthSize + n += copy(values[n:], b) + r.offset++ + c++ + } + if r.offset == numValues { + err = io.EOF + } else if n == 0 && len(values) > 0 { + err = io.ErrShortBuffer + } + return c, n, err +} + +func (r *byteArrayPageValues) ReadValues(values []Value) (n int, err error) { + numValues := r.page.len() + for n < len(values) && r.offset < numValues { + values[n] = r.page.makeValueBytes(r.page.index(r.offset)) + r.offset++ + n++ + } + if r.offset == numValues { + err = io.EOF + } + return n, err +} + +type fixedLenByteArrayPageValues struct { + page *fixedLenByteArrayPage + offset int +} + +func (r *fixedLenByteArrayPageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadFixedLenByteArrays(b) + return n * r.page.size, err +} + +func (r *fixedLenByteArrayPageValues) ReadRequired(values []byte) (int, error) { + return r.ReadFixedLenByteArrays(values) +} + +func (r *fixedLenByteArrayPageValues) ReadFixedLenByteArrays(values []byte) (n int, err error) { + n = copy(values, r.page.data[r.offset:]) / r.page.size + r.offset += n * r.page.size + if r.offset == len(r.page.data) { + err = io.EOF + } else if n == 0 && len(values) > 0 { + err = io.ErrShortBuffer + } + return n, err +} + +func (r *fixedLenByteArrayPageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.data) { + values[n] = r.page.makeValueBytes(r.page.data[r.offset : r.offset+r.page.size]) + r.offset += r.page.size + n++ + } + if r.offset == len(r.page.data) { + err = io.EOF + } + return n, err +} + +type uint32PageValues struct { + page *uint32Page + offset int +} + +func (r *uint32PageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadUint32s(unsafecast.Slice[uint32](b)) + return 4 * n, err +} + +func (r *uint32PageValues) ReadUint32s(values []uint32) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *uint32PageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type uint64PageValues struct { + page *uint64Page + offset int +} + +func (r *uint64PageValues) Read(b []byte) (n int, err error) { + n, err = r.ReadUint64s(unsafecast.Slice[uint64](b)) + return 8 * n, err +} + +func (r *uint64PageValues) ReadUint64s(values []uint64) (n int, err error) { + n = copy(values, r.page.values[r.offset:]) + r.offset += n + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +func (r *uint64PageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type be128PageValues struct { + page *be128Page + offset int +} + +func (r *be128PageValues) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.offset < len(r.page.values) { + values[n] = r.page.makeValue(&r.page.values[r.offset]) + r.offset++ + n++ + } + if r.offset == len(r.page.values) { + err = io.EOF + } + return n, err +} + +type nullPageValues struct { + column int + remain int +} + +func (r *nullPageValues) ReadValues(values []Value) (n int, err error) { + columnIndex := ^int16(r.column) + values = values[:min(r.remain, len(values))] + for i := range values { + values[i] = Value{columnIndex: columnIndex} + } + r.remain -= len(values) + if r.remain == 0 { + err = io.EOF + } + return len(values), err +} diff --git a/vendor/github.com/parquet-go/parquet-go/parquet.go b/vendor/github.com/parquet-go/parquet-go/parquet.go new file mode 100644 index 0000000000..666b16eb18 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/parquet.go @@ -0,0 +1,122 @@ +// Copyright 2022 Twilio Inc. + +// Package parquet is a library for working with parquet files. For an overview +// of Parquet's qualities as a storage format, see this blog post: +// https://blog.twitter.com/engineering/en_us/a/2013/dremel-made-simple-with-parquet +// +// Or see the Parquet documentation: https://parquet.apache.org/docs/ +package parquet + +import ( + "io" + "os" + "reflect" +) + +// Read reads and returns rows from the parquet file in the given reader. +// +// The type T defines the type of rows read from r. T must be compatible with +// the file's schema or an error will be returned. The row type might represent +// a subset of the full schema, in which case only a subset of the columns will +// be loaded from r. +// +// This function is provided for convenience to facilitate reading of parquet +// files from arbitrary locations in cases where the data set fit in memory. +func Read[T any](r io.ReaderAt, size int64, options ...ReaderOption) (rows []T, err error) { + config, err := NewReaderConfig(options...) + if err != nil { + return nil, err + } + file, err := OpenFile(r, size) + if err != nil { + return nil, err + } + rows = make([]T, file.NumRows()) + reader := NewGenericReader[T](file, config) + n, err := reader.Read(rows) + if err == io.EOF { + err = nil + } + reader.Close() + return rows[:n], err +} + +// ReadFile reads rows of the parquet file at the given path. +// +// The type T defines the type of rows read from r. T must be compatible with +// the file's schema or an error will be returned. The row type might represent +// a subset of the full schema, in which case only a subset of the columns will +// be loaded from the file. +// +// This function is provided for convenience to facilitate reading of parquet +// files from the file system in cases where the data set fit in memory. +func ReadFile[T any](path string, options ...ReaderOption) (rows []T, err error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + s, err := f.Stat() + if err != nil { + return nil, err + } + return Read[T](f, s.Size()) +} + +// Write writes the given list of rows to a parquet file written to w. +// +// This function is provided for convenience to facilitate the creation of +// parquet files. +func Write[T any](w io.Writer, rows []T, options ...WriterOption) error { + config, err := NewWriterConfig(options...) + if err != nil { + return err + } + writer := NewGenericWriter[T](w, config) + if _, err := writer.Write(rows); err != nil { + return err + } + return writer.Close() +} + +// Write writes the given list of rows to a parquet file written to w. +// +// This function is provided for convenience to facilitate writing parquet +// files to the file system. +func WriteFile[T any](path string, rows []T, options ...WriterOption) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + return Write(f, rows, options...) +} + +func atLeastOne(size int) int { + return atLeast(size, 1) +} + +func atLeast(size, least int) int { + if size < least { + return least + } + return size +} + +func typeNameOf(t reflect.Type) string { + s1 := t.String() + s2 := t.Kind().String() + if s1 == s2 { + return s1 + } + return s1 + " (" + s2 + ")" +} + +func isZero(b []byte) bool { + for _, c := range b { + if c != 0 { + return false + } + } + return true +} diff --git a/vendor/github.com/parquet-go/parquet-go/parquet_amd64.go b/vendor/github.com/parquet-go/parquet-go/parquet_amd64.go new file mode 100644 index 0000000000..64165571d3 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/parquet_amd64.go @@ -0,0 +1,18 @@ +//go:build !purego + +package parquet + +import "golang.org/x/sys/cpu" + +var ( + // This variable is used in x86 assembly source files to gate the use of + // AVX2 instructions depending on whether the CPU supports it. + hasAVX2 = cpu.X86.HasAVX2 + hasAVX512F = cpu.X86.HasAVX512F + hasAVX512VL = cpu.X86.HasAVX512F && cpu.X86.HasAVX512VL + // For min/max functions over big-endian 128 bits values, we need the + // follwing instructions from the DQ set: + // * VPBROADCASTQ (with 64 bits source register) + // * VBROADCASTI64X2 + hasAVX512MinMaxBE128 = cpu.X86.HasAVX512F && cpu.X86.HasAVX512DQ +) diff --git a/vendor/github.com/parquet-go/parquet-go/print.go b/vendor/github.com/parquet-go/parquet-go/print.go new file mode 100644 index 0000000000..746c900ce6 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/print.go @@ -0,0 +1,339 @@ +package parquet + +import ( + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/olekukonko/tablewriter" +) + +func PrintSchema(w io.Writer, name string, node Node) error { + return PrintSchemaIndent(w, name, node, "\t", "\n") +} + +func PrintSchemaIndent(w io.Writer, name string, node Node, pattern, newline string) error { + pw := &printWriter{writer: w} + pi := &printIndent{} + + if node.Leaf() { + printSchemaWithIndent(pw, "", node, pi) + } else { + pw.WriteString("message ") + + if name == "" { + pw.WriteString("{") + } else { + pw.WriteString(name) + pw.WriteString(" {") + } + + pi.pattern = pattern + pi.newline = newline + pi.repeat = 1 + pi.writeNewLine(pw) + + for _, field := range node.Fields() { + printSchemaWithIndent(pw, field.Name(), field, pi) + pi.writeNewLine(pw) + } + + pw.WriteString("}") + } + + return pw.err +} + +func printSchemaWithIndent(w io.StringWriter, name string, node Node, indent *printIndent) { + indent.writeTo(w) + + switch { + case node.Optional(): + w.WriteString("optional ") + case node.Repeated(): + w.WriteString("repeated ") + default: + w.WriteString("required ") + } + + if node.Leaf() { + t := node.Type() + switch t.Kind() { + case Boolean: + w.WriteString("boolean") + case Int32: + w.WriteString("int32") + case Int64: + w.WriteString("int64") + case Int96: + w.WriteString("int96") + case Float: + w.WriteString("float") + case Double: + w.WriteString("double") + case ByteArray: + w.WriteString("binary") + case FixedLenByteArray: + w.WriteString("fixed_len_byte_array(") + w.WriteString(strconv.Itoa(t.Length())) + w.WriteString(")") + default: + w.WriteString("") + } + + if name != "" { + w.WriteString(" ") + w.WriteString(name) + } + + if annotation := annotationOf(node); annotation != "" { + w.WriteString(" (") + w.WriteString(annotation) + w.WriteString(")") + } + + if id := node.ID(); id != 0 { + w.WriteString(" = ") + w.WriteString(strconv.Itoa(id)) + } + + w.WriteString(";") + } else { + w.WriteString("group") + + if name != "" { + w.WriteString(" ") + w.WriteString(name) + } + + if annotation := annotationOf(node); annotation != "" { + w.WriteString(" (") + w.WriteString(annotation) + w.WriteString(")") + } + + if id := node.ID(); id != 0 { + w.WriteString(" = ") + w.WriteString(strconv.Itoa(id)) + } + + w.WriteString(" {") + indent.writeNewLine(w) + indent.push() + + for _, field := range node.Fields() { + printSchemaWithIndent(w, field.Name(), field, indent) + indent.writeNewLine(w) + } + + indent.pop() + indent.writeTo(w) + w.WriteString("}") + } +} + +func annotationOf(node Node) string { + if logicalType := node.Type().LogicalType(); logicalType != nil { + return logicalType.String() + } + return "" +} + +type printIndent struct { + pattern string + newline string + repeat int +} + +func (i *printIndent) push() { + i.repeat++ +} + +func (i *printIndent) pop() { + i.repeat-- +} + +func (i *printIndent) writeTo(w io.StringWriter) { + if i.pattern != "" { + for n := i.repeat; n > 0; n-- { + w.WriteString(i.pattern) + } + } +} + +func (i *printIndent) writeNewLine(w io.StringWriter) { + if i.newline != "" { + w.WriteString(i.newline) + } +} + +type printWriter struct { + writer io.Writer + err error +} + +func (w *printWriter) Write(b []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + n, err := w.writer.Write(b) + if err != nil { + w.err = err + } + return n, err +} + +func (w *printWriter) WriteString(s string) (int, error) { + if w.err != nil { + return 0, w.err + } + n, err := io.WriteString(w.writer, s) + if err != nil { + w.err = err + } + return n, err +} + +var ( + _ io.StringWriter = (*printWriter)(nil) +) + +func sprint(name string, node Node) string { + s := new(strings.Builder) + PrintSchema(s, name, node) + return s.String() +} + +func PrintRowGroup(w io.Writer, rowGroup RowGroup) error { + schema := rowGroup.Schema() + pw := &printWriter{writer: w} + tw := tablewriter.NewWriter(pw) + + columns := schema.Columns() + header := make([]string, len(columns)) + footer := make([]string, len(columns)) + alignment := make([]int, len(columns)) + + for i, column := range columns { + leaf, _ := schema.Lookup(column...) + columnType := leaf.Node.Type() + + header[i] = strings.Join(column, ".") + footer[i] = columnType.String() + + switch columnType.Kind() { + case ByteArray: + alignment[i] = tablewriter.ALIGN_LEFT + default: + alignment[i] = tablewriter.ALIGN_RIGHT + } + } + + rowbuf := make([]Row, defaultRowBufferSize) + cells := make([]string, 0, len(columns)) + rows := rowGroup.Rows() + defer rows.Close() + + for { + n, err := rows.ReadRows(rowbuf) + + for _, row := range rowbuf[:n] { + cells = cells[:0] + + for _, value := range row { + columnIndex := value.Column() + + for len(cells) <= columnIndex { + cells = append(cells, "") + } + + if cells[columnIndex] == "" { + cells[columnIndex] = value.String() + } else { + cells[columnIndex] += "," + value.String() + alignment[columnIndex] = tablewriter.ALIGN_LEFT + } + } + + tw.Append(cells) + } + + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return err + } + } + + tw.SetAutoFormatHeaders(false) + tw.SetColumnAlignment(alignment) + tw.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + tw.SetFooterAlignment(tablewriter.ALIGN_LEFT) + tw.SetHeader(header) + tw.SetFooter(footer) + tw.Render() + + fmt.Fprintf(pw, "%d rows\n\n", rowGroup.NumRows()) + return pw.err +} + +func PrintColumnChunk(w io.Writer, columnChunk ColumnChunk) error { + pw := &printWriter{writer: w} + pw.WriteString(columnChunk.Type().String()) + pw.WriteString("\n--------------------------------------------------------------------------------\n") + + values := [42]Value{} + pages := columnChunk.Pages() + numPages, numValues := int64(0), int64(0) + + defer pages.Close() + for { + p, err := pages.ReadPage() + if err != nil { + if !errors.Is(err, io.EOF) { + return err + } + break + } + + numPages++ + n := p.NumValues() + if n == 0 { + fmt.Fprintf(pw, "*** page %d, no values ***\n", numPages) + } else { + fmt.Fprintf(pw, "*** page %d, values %d to %d ***\n", numPages, numValues+1, numValues+n) + printPage(w, p, values[:], numValues+1) + numValues += n + } + + pw.WriteString("\n") + } + + return pw.err +} + +func PrintPage(w io.Writer, page Page) error { + return printPage(w, page, make([]Value, 42), 0) +} + +func printPage(w io.Writer, page Page, values []Value, numValues int64) error { + r := page.Values() + for { + n, err := r.ReadValues(values[:]) + for i, v := range values[:n] { + _, err := fmt.Fprintf(w, "value %d: %+v\n", numValues+int64(i), v) + if err != nil { + return err + } + } + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + return err + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/reader.go b/vendor/github.com/parquet-go/parquet-go/reader.go new file mode 100644 index 0000000000..c4277f786e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/reader.go @@ -0,0 +1,671 @@ +package parquet + +import ( + "errors" + "fmt" + "io" + "reflect" + + "github.com/parquet-go/parquet-go/format" +) + +// GenericReader is similar to a Reader but uses a type parameter to define the +// Go type representing the schema of rows being read. +// +// See GenericWriter for details about the benefits over the classic Reader API. +type GenericReader[T any] struct { + base Reader + read readFunc[T] +} + +// NewGenericReader is like NewReader but returns GenericReader[T] suited to write +// rows of Go type T. +// +// The type parameter T should be a map, struct, or any. Any other types will +// cause a panic at runtime. Type checking is a lot more effective when the +// generic parameter is a struct type, using map and interface types is somewhat +// similar to using a Writer. +// +// If the option list may explicitly declare a schema, it must be compatible +// with the schema generated from T. +func NewGenericReader[T any](input io.ReaderAt, options ...ReaderOption) *GenericReader[T] { + c, err := NewReaderConfig(options...) + if err != nil { + panic(err) + } + + f, err := openFile(input) + if err != nil { + panic(err) + } + + rowGroup := fileRowGroupOf(f) + + t := typeOf[T]() + if c.Schema == nil { + if t == nil { + c.Schema = rowGroup.Schema() + } else { + c.Schema = schemaOf(dereference(t)) + } + } + + r := &GenericReader[T]{ + base: Reader{ + file: reader{ + file: f, + schema: c.Schema, + rowGroup: rowGroup, + }, + }, + } + + if !EqualNodes(c.Schema, f.schema) { + r.base.file.rowGroup = convertRowGroupTo(r.base.file.rowGroup, c.Schema) + } + + r.base.read.init(r.base.file.schema, r.base.file.rowGroup) + r.read = readFuncOf[T](t, r.base.file.schema) + return r +} + +func NewGenericRowGroupReader[T any](rowGroup RowGroup, options ...ReaderOption) *GenericReader[T] { + c, err := NewReaderConfig(options...) + if err != nil { + panic(err) + } + + t := typeOf[T]() + if c.Schema == nil { + if t == nil { + c.Schema = rowGroup.Schema() + } else { + c.Schema = schemaOf(dereference(t)) + } + } + + r := &GenericReader[T]{ + base: Reader{ + file: reader{ + schema: c.Schema, + rowGroup: rowGroup, + }, + }, + } + + if !EqualNodes(c.Schema, rowGroup.Schema()) { + r.base.file.rowGroup = convertRowGroupTo(r.base.file.rowGroup, c.Schema) + } + + r.base.read.init(r.base.file.schema, r.base.file.rowGroup) + r.read = readFuncOf[T](t, r.base.file.schema) + return r +} + +func (r *GenericReader[T]) Reset() { + r.base.Reset() +} + +// Read reads the next rows from the reader into the given rows slice up to len(rows). +// +// The returned values are safe to reuse across Read calls and do not share +// memory with the reader's underlying page buffers. +// +// The method returns the number of rows read and io.EOF when no more rows +// can be read from the reader. +func (r *GenericReader[T]) Read(rows []T) (int, error) { + return r.read(r, rows) +} + +func (r *GenericReader[T]) ReadRows(rows []Row) (int, error) { + return r.base.ReadRows(rows) +} + +func (r *GenericReader[T]) Schema() *Schema { + return r.base.Schema() +} + +func (r *GenericReader[T]) NumRows() int64 { + return r.base.NumRows() +} + +func (r *GenericReader[T]) SeekToRow(rowIndex int64) error { + return r.base.SeekToRow(rowIndex) +} + +func (r *GenericReader[T]) Close() error { + return r.base.Close() +} + +// File returns a FileView of the underlying parquet file. +func (r *GenericReader[T]) File() FileView { + return r.base.File() +} + +// readRows reads the next rows from the reader into the given rows slice up to len(rows). +// +// The returned values are safe to reuse across readRows calls and do not share +// memory with the reader's underlying page buffers. +// +// The method returns the number of rows read and io.EOF when no more rows +// can be read from the reader. +func (r *GenericReader[T]) readRows(rows []T) (int, error) { + nRequest := len(rows) + if cap(r.base.rowbuf) < nRequest { + r.base.rowbuf = make([]Row, nRequest) + } else { + r.base.rowbuf = r.base.rowbuf[:nRequest] + } + + var n, nTotal int + var err error + for { + // ReadRows reads the minimum remaining rows in a column page across all columns + // of the underlying reader, unless the length of the slice passed to it is smaller. + // In that case, ReadRows will read the number of rows equal to the length of the + // given slice argument. We limit that length to never be more than requested + // because sequential reads can cross page boundaries. + n, err = r.base.ReadRows(r.base.rowbuf[:nRequest-nTotal]) + if n > 0 { + schema := r.base.Schema() + + for i, row := range r.base.rowbuf[:n] { + if err2 := schema.Reconstruct(&rows[nTotal+i], row); err2 != nil { + return nTotal + i, err2 + } + } + } + nTotal += n + if n == 0 || nTotal == nRequest || err != nil { + break + } + } + + return nTotal, err +} + +var ( + _ Rows = (*GenericReader[any])(nil) + _ RowReaderWithSchema = (*Reader)(nil) + + _ Rows = (*GenericReader[struct{}])(nil) + _ RowReaderWithSchema = (*GenericReader[struct{}])(nil) + + _ Rows = (*GenericReader[map[struct{}]struct{}])(nil) + _ RowReaderWithSchema = (*GenericReader[map[struct{}]struct{}])(nil) +) + +type readFunc[T any] func(*GenericReader[T], []T) (int, error) + +func readFuncOf[T any](t reflect.Type, schema *Schema) readFunc[T] { + if t == nil { + return (*GenericReader[T]).readRows + } + switch t.Kind() { + case reflect.Interface, reflect.Map: + return (*GenericReader[T]).readRows + + case reflect.Struct: + return (*GenericReader[T]).readRows + + case reflect.Pointer: + if e := t.Elem(); e.Kind() == reflect.Struct { + return (*GenericReader[T]).readRows + } + } + panic("cannot create reader for values of type " + t.String()) +} + +// Deprecated: A Reader reads Go values from parquet files. +// +// This example showcases a typical use of parquet readers: +// +// reader := parquet.NewReader(file) +// rows := []RowType{} +// for { +// row := RowType{} +// err := reader.Read(&row) +// if err != nil { +// if err == io.EOF { +// break +// } +// ... +// } +// rows = append(rows, row) +// } +// if err := reader.Close(); err != nil { +// ... +// } +// +// For programs building with Go 1.18 or later, the GenericReader[T] type +// supersedes this one. +type Reader struct { + seen reflect.Type + file reader + read reader + rowIndex int64 + rowbuf []Row +} + +// NewReader constructs a parquet reader reading rows from the given +// io.ReaderAt. +// +// In order to read parquet rows, the io.ReaderAt must be converted to a +// parquet.File. If r is already a parquet.File it is used directly; otherwise, +// the io.ReaderAt value is expected to either have a `Size() int64` method or +// implement io.Seeker in order to determine its size. +// +// The function panics if the reader configuration is invalid. Programs that +// cannot guarantee the validity of the options passed to NewReader should +// construct the reader configuration independently prior to calling this +// function: +// +// config, err := parquet.NewReaderConfig(options...) +// if err != nil { +// // handle the configuration error +// ... +// } else { +// // this call to create a reader is guaranteed not to panic +// reader := parquet.NewReader(input, config) +// ... +// } +func NewReader(input io.ReaderAt, options ...ReaderOption) *Reader { + c, err := NewReaderConfig(options...) + if err != nil { + panic(err) + } + + f, err := openFile(input) + if err != nil { + panic(err) + } + + r := &Reader{ + file: reader{ + file: f, + schema: f.schema, + rowGroup: fileRowGroupOf(f), + }, + } + + if c.Schema != nil { + r.file.schema = c.Schema + r.file.rowGroup = convertRowGroupTo(r.file.rowGroup, c.Schema) + } + + r.read.init(r.file.schema, r.file.rowGroup) + return r +} + +func openFile(input io.ReaderAt) (*File, error) { + f, _ := input.(*File) + if f != nil { + return f, nil + } + n, err := sizeOf(input) + if err != nil { + return nil, err + } + return OpenFile(input, n) +} + +func fileRowGroupOf(f *File) RowGroup { + switch rowGroups := f.RowGroups(); len(rowGroups) { + case 0: + return newEmptyRowGroup(f.Schema()) + case 1: + return rowGroups[0] + default: + // TODO: should we attempt to merge the row groups via MergeRowGroups + // to preserve the global order of sorting columns within the file? + return MultiRowGroup(rowGroups...) + } +} + +// NewRowGroupReader constructs a new Reader which reads rows from the RowGroup +// passed as argument. +func NewRowGroupReader(rowGroup RowGroup, options ...ReaderOption) *Reader { + c, err := NewReaderConfig(options...) + if err != nil { + panic(err) + } + + if c.Schema != nil { + rowGroup = convertRowGroupTo(rowGroup, c.Schema) + } + + r := &Reader{ + file: reader{ + schema: rowGroup.Schema(), + rowGroup: rowGroup, + }, + } + + r.read.init(r.file.schema, r.file.rowGroup) + return r +} + +func convertRowGroupTo(rowGroup RowGroup, schema *Schema) RowGroup { + if rowGroupSchema := rowGroup.Schema(); !EqualNodes(schema, rowGroupSchema) { + conv, err := Convert(schema, rowGroupSchema) + if err != nil { + // TODO: this looks like something we should not be panicking on, + // but the current NewReader API does not offer a mechanism to + // report errors. + panic(err) + } + rowGroup = ConvertRowGroup(rowGroup, conv) + } + return rowGroup +} + +func sizeOf(r io.ReaderAt) (int64, error) { + switch f := r.(type) { + case interface{ Size() int64 }: + return f.Size(), nil + case io.Seeker: + off, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + end, err := f.Seek(0, io.SeekEnd) + if err != nil { + return 0, err + } + _, err = f.Seek(off, io.SeekStart) + return end, err + default: + return 0, fmt.Errorf("cannot determine length of %T", r) + } +} + +// Reset repositions the reader at the beginning of the underlying parquet file. +func (r *Reader) Reset() { + r.file.Reset() + r.read.Reset() + r.rowIndex = 0 + clearRows(r.rowbuf) +} + +// Read reads the next row from r. The type of the row must match the schema +// of the underlying parquet file or an error will be returned. +// +// The method returns io.EOF when no more rows can be read from r. +func (r *Reader) Read(row any) error { + if rowType := dereference(reflect.TypeOf(row)); rowType.Kind() == reflect.Struct { + if r.seen != rowType { + if err := r.updateReadSchema(rowType); err != nil { + return fmt.Errorf("cannot read parquet row into go value of type %T: %w", row, err) + } + } + } + + if err := r.read.SeekToRow(r.rowIndex); err != nil { + if errors.Is(err, io.ErrClosedPipe) { + return io.EOF + } + return fmt.Errorf("seeking reader to row %d: %w", r.rowIndex, err) + } + + if cap(r.rowbuf) == 0 { + r.rowbuf = make([]Row, 1) + } else { + r.rowbuf = r.rowbuf[:1] + } + + n, err := r.read.ReadRows(r.rowbuf[:]) + if n == 0 { + return err + } + + r.rowIndex++ + return r.read.schema.Reconstruct(row, r.rowbuf[0]) +} + +func (r *Reader) updateReadSchema(rowType reflect.Type) error { + schema := schemaOf(rowType) + + if EqualNodes(schema, r.file.schema) { + r.read.init(schema, r.file.rowGroup) + } else { + conv, err := Convert(schema, r.file.schema) + if err != nil { + return err + } + r.read.init(schema, ConvertRowGroup(r.file.rowGroup, conv)) + } + + r.seen = rowType + return nil +} + +// ReadRows reads the next rows from r into the given Row buffer. +// +// The returned values are laid out in the order expected by the +// parquet.(*Schema).Reconstruct method. +// +// The method returns io.EOF when no more rows can be read from r. +func (r *Reader) ReadRows(rows []Row) (int, error) { + if err := r.file.SeekToRow(r.rowIndex); err != nil { + return 0, err + } + n, err := r.file.ReadRows(rows) + r.rowIndex += int64(n) + return n, err +} + +// Schema returns the schema of rows read by r. +func (r *Reader) Schema() *Schema { return r.file.schema } + +// NumRows returns the number of rows that can be read from r. +func (r *Reader) NumRows() int64 { return r.file.rowGroup.NumRows() } + +// SeekToRow positions r at the given row index. +func (r *Reader) SeekToRow(rowIndex int64) error { + if err := r.file.SeekToRow(rowIndex); err != nil { + return err + } + r.rowIndex = rowIndex + return nil +} + +// Close closes the reader, preventing more rows from being read. +func (r *Reader) Close() error { + if err := r.read.Close(); err != nil { + return err + } + if err := r.file.Close(); err != nil { + return err + } + return nil +} + +// reader is a subtype used in the implementation of Reader to support the two +// use cases of either reading rows calling the ReadRow method (where full rows +// are read from the underlying parquet file), or calling the Read method to +// read rows into Go values, potentially doing partial reads on a subset of the +// columns due to using a converted row group view. +type reader struct { + file *File + schema *Schema + rowGroup RowGroup + rows Rows + rowIndex int64 +} + +func (r *reader) init(schema *Schema, rowGroup RowGroup) { + r.schema = schema + r.rowGroup = rowGroup + r.Reset() +} + +func (r *reader) Reset() { + r.rowIndex = 0 + + if rows, ok := r.rows.(interface{ Reset() }); ok { + // This optimization works for the common case where the underlying type + // of the Rows instance is rowGroupRows, which should be true in most + // cases since even external implementations of the RowGroup interface + // can construct values of this type via the NewRowGroupRowReader + // function. + // + // Foreign implementations of the Rows interface may also define a Reset + // method in order to participate in this optimization. + rows.Reset() + return + } + + if r.rows != nil { + r.rows.Close() + r.rows = nil + } +} + +func (r *reader) ReadRows(rows []Row) (int, error) { + if r.rowGroup == nil { + return 0, io.EOF + } + if r.rows == nil { + r.rows = r.rowGroup.Rows() + if r.rowIndex > 0 { + if err := r.rows.SeekToRow(r.rowIndex); err != nil { + return 0, err + } + } + } + n, err := r.rows.ReadRows(rows) + r.rowIndex += int64(n) + return n, err +} + +func (r *reader) SeekToRow(rowIndex int64) error { + if r.rowGroup == nil { + return io.ErrClosedPipe + } + if rowIndex != r.rowIndex { + if r.rows != nil { + if err := r.rows.SeekToRow(rowIndex); err != nil { + return err + } + } + r.rowIndex = rowIndex + } + return nil +} + +func (r *reader) Close() (err error) { + r.rowGroup = nil + if r.rows != nil { + err = r.rows.Close() + } + return err +} + +var ( + _ Rows = (*Reader)(nil) + _ RowReaderWithSchema = (*Reader)(nil) + + _ RowReader = (*reader)(nil) + _ RowSeeker = (*reader)(nil) +) + +type readerFileView struct { + reader *reader + schema *Schema +} + +// File returns a FileView of the parquet file being read. +// Only available if Reader was created with a File. +func (r *Reader) File() FileView { + if r.file.schema == nil || r.file.file == nil { + return nil + } + return &readerFileView{ + &r.file, + r.file.schema, + } +} + +func (r *readerFileView) Metadata() *format.FileMetaData { + if r.reader.file != nil { + return r.reader.file.Metadata() + } + return nil +} + +func (r *readerFileView) Schema() *Schema { + return r.schema +} + +func (r *readerFileView) NumRows() int64 { + return r.reader.rowGroup.NumRows() +} + +func (r *readerFileView) Lookup(key string) (string, bool) { + if meta := r.Metadata(); meta != nil { + return lookupKeyValueMetadata(meta.KeyValueMetadata, key) + } + return "", false +} + +func (r *readerFileView) Size() int64 { + if r.reader.file != nil { + return r.reader.file.Size() + } + return 0 +} + +func (r *readerFileView) ColumnIndexes() []format.ColumnIndex { + if r.reader.file != nil { + return r.reader.file.ColumnIndexes() + } + return nil +} + +func (r *readerFileView) OffsetIndexes() []format.OffsetIndex { + if r.reader.file != nil { + return r.reader.file.OffsetIndexes() + } + return nil +} + +func (r *readerFileView) Root() *Column { + if meta := r.Metadata(); meta != nil { + root, _ := openColumns(nil, meta, r.ColumnIndexes(), r.OffsetIndexes()) + return root + } + return nil +} + +func (r *readerFileView) RowGroups() []RowGroup { + file := r.reader.file + if file == nil { + return nil + } + columns := makeLeafColumns(r.Root()) + fileRowGroups := makeFileRowGroups(file, columns) + return makeRowGroups(fileRowGroups) +} + +func makeLeafColumns(root *Column) []*Column { + columns := make([]*Column, 0, numLeafColumnsOf(root)) + root.forEachLeaf(func(c *Column) { columns = append(columns, c) }) + return columns +} + +func makeFileRowGroups(file *File, columns []*Column) []FileRowGroup { + rowGroups := file.metadata.RowGroups + fileRowGroups := make([]FileRowGroup, len(rowGroups)) + for i := range fileRowGroups { + fileRowGroups[i].init(file, columns, &rowGroups[i]) + } + return fileRowGroups +} + +func makeRowGroups(fileRowGroups []FileRowGroup) []RowGroup { + rowGroups := make([]RowGroup, len(fileRowGroups)) + for i := range fileRowGroups { + rowGroups[i] = &fileRowGroups[i] + } + return rowGroups +} diff --git a/vendor/github.com/parquet-go/parquet-go/row.go b/vendor/github.com/parquet-go/parquet-go/row.go new file mode 100644 index 0000000000..a885eaa265 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/row.go @@ -0,0 +1,848 @@ +package parquet + +import ( + "errors" + "fmt" + "io" + "reflect" +) + +const ( + defaultRowBufferSize = 42 +) + +// Row represents a parquet row as a slice of values. +// +// Each value should embed a column index, repetition level, and definition +// level allowing the program to determine how to reconstruct the original +// object from the row. +type Row []Value + +// MakeRow constructs a Row from a list of column values. +// +// The function panics if the column indexes of values in each column do not +// match their position in the argument list. +func MakeRow(columns ...[]Value) Row { return AppendRow(nil, columns...) } + +// AppendRow appends to row the given list of column values. +// +// AppendRow can be used to construct a Row value from columns, while retaining +// the underlying memory buffer to avoid reallocation; for example: +// +// The function panics if the column indexes of values in each column do not +// match their position in the argument list. +func AppendRow(row Row, columns ...[]Value) Row { + numValues := 0 + + for expectedColumnIndex, column := range columns { + numValues += len(column) + + for _, value := range column { + if value.columnIndex != ^int16(expectedColumnIndex) { + panic(fmt.Sprintf("value of column %d has column index %d", expectedColumnIndex, value.Column())) + } + } + } + + if capacity := cap(row) - len(row); capacity < numValues { + row = append(make(Row, 0, len(row)+numValues), row...) + } + + return appendRow(row, columns) +} + +func appendRow(row Row, columns [][]Value) Row { + for _, column := range columns { + row = append(row, column...) + } + return row +} + +// Clone creates a copy of the row which shares no pointers. +// +// This method is useful to capture rows after a call to RowReader.ReadRows when +// values need to be retained before the next call to ReadRows or after the lifespan +// of the reader. +func (row Row) Clone() Row { + clone := make(Row, len(row)) + for i := range row { + clone[i] = row[i].Clone() + } + return clone +} + +// Equal returns true if row and other contain the same sequence of values. +func (row Row) Equal(other Row) bool { + if len(row) != len(other) { + return false + } + for i := range row { + if !Equal(row[i], other[i]) { + return false + } + if row[i].repetitionLevel != other[i].repetitionLevel { + return false + } + if row[i].definitionLevel != other[i].definitionLevel { + return false + } + if row[i].columnIndex != other[i].columnIndex { + return false + } + } + return true +} + +// Range calls f for each column of row. +func (row Row) Range(f func(columnIndex int, columnValues []Value) bool) { + columnIndex := 0 + + for i := 0; i < len(row); { + j := i + 1 + + for j < len(row) && row[j].columnIndex == ^int16(columnIndex) { + j++ + } + + if !f(columnIndex, row[i:j:j]) { + break + } + + columnIndex++ + i = j + } +} + +// RowSeeker is an interface implemented by readers of parquet rows which can be +// positioned at a specific row index. +type RowSeeker interface { + // Positions the stream on the given row index. + // + // Some implementations of the interface may only allow seeking forward. + // + // The method returns io.ErrClosedPipe if the stream had already been closed. + SeekToRow(int64) error +} + +// RowReader reads a sequence of parquet rows. +type RowReader interface { + // ReadRows reads rows from the reader, returning the number of rows read + // into the buffer, and any error that occurred. + // + // When all rows have been read, the reader returns io.EOF to indicate the + // end of the sequence. It is valid for the reader to return both a non-zero + // number of rows and a non-nil error (including io.EOF). + // + // The buffer of rows passed as argument will be used to store values of + // each row read from the reader. If the rows are not nil, the backing array + // of the slices will be used as an optimization to avoid re-allocating new + // arrays. + // + // The application is expected to handle the case where ReadRows returns + // less rows than requested and no error, by looking at the first returned + // value from ReadRows, which is the number of rows that were read. + ReadRows([]Row) (int, error) +} + +// RowReaderFrom reads parquet rows from reader. +type RowReaderFrom interface { + ReadRowsFrom(RowReader) (int64, error) +} + +// RowReaderWithSchema is an extension of the RowReader interface which +// advertises the schema of rows returned by ReadRow calls. +type RowReaderWithSchema interface { + RowReader + Schema() *Schema +} + +// RowReadSeeker is an interface implemented by row readers which support +// seeking to arbitrary row positions. +type RowReadSeeker interface { + RowReader + RowSeeker +} + +// RowReadCloser is an interface implemented by row readers which require +// closing when done. +type RowReadCloser interface { + RowReader + io.Closer +} + +// RowReadSeekCloser is an interface implemented by row readers which support +// seeking to arbitrary row positions and required closing the reader when done. +type RowReadSeekCloser interface { + RowReader + RowSeeker + io.Closer +} + +// RowWriter writes parquet rows to an underlying medium. +type RowWriter interface { + // Writes rows to the writer, returning the number of rows written and any + // error that occurred. + // + // Because columnar operations operate on independent columns of values, + // writes of rows may not be atomic operations, and could result in some + // rows being partially written. The method returns the number of rows that + // were successfully written, but if an error occurs, values of the row(s) + // that failed to be written may have been partially committed to their + // columns. For that reason, applications should consider a write error as + // fatal and assume that they need to discard the state, they cannot retry + // the write nor recover the underlying file. + WriteRows([]Row) (int, error) +} + +// RowWriterTo writes parquet rows to a writer. +type RowWriterTo interface { + WriteRowsTo(RowWriter) (int64, error) +} + +// RowWriterWithSchema is an extension of the RowWriter interface which +// advertises the schema of rows expected to be passed to WriteRow calls. +type RowWriterWithSchema interface { + RowWriter + Schema() *Schema +} + +// RowReaderFunc is a function type implementing the RowReader interface. +type RowReaderFunc func([]Row) (int, error) + +func (f RowReaderFunc) ReadRows(rows []Row) (int, error) { return f(rows) } + +// RowWriterFunc is a function type implementing the RowWriter interface. +type RowWriterFunc func([]Row) (int, error) + +func (f RowWriterFunc) WriteRows(rows []Row) (int, error) { return f(rows) } + +// MultiRowWriter constructs a RowWriter which dispatches writes to all the +// writers passed as arguments. +// +// When writing rows, if any of the writers returns an error, the operation is +// aborted and the error returned. If one of the writers did not error, but did +// not write all the rows, the operation is aborted and io.ErrShortWrite is +// returned. +// +// Rows are written sequentially to each writer in the order they are given to +// this function. +func MultiRowWriter(writers ...RowWriter) RowWriter { + m := &multiRowWriter{writers: make([]RowWriter, len(writers))} + copy(m.writers, writers) + return m +} + +type multiRowWriter struct{ writers []RowWriter } + +func (m *multiRowWriter) WriteRows(rows []Row) (int, error) { + for _, w := range m.writers { + n, err := w.WriteRows(rows) + if err != nil { + return n, err + } + if n != len(rows) { + return n, io.ErrShortWrite + } + } + return len(rows), nil +} + +type forwardRowSeeker struct { + rows RowReader + seek int64 + index int64 +} + +func (r *forwardRowSeeker) ReadRows(rows []Row) (int, error) { + for { + n, err := r.rows.ReadRows(rows) + + if n > 0 && r.index < r.seek { + skip := r.seek - r.index + r.index += int64(n) + if skip >= int64(n) { + continue + } + + for i, j := 0, int(skip); j < n; i++ { + rows[i] = append(rows[i][:0], rows[j]...) + } + + n -= int(skip) + } + + return n, err + } +} + +func (r *forwardRowSeeker) SeekToRow(rowIndex int64) error { + if rowIndex >= r.index { + r.seek = rowIndex + return nil + } + return fmt.Errorf( + "SeekToRow: %T does not implement parquet.RowSeeker: cannot seek backward from row %d to %d", + r.rows, + r.index, + rowIndex, + ) +} + +// CopyRows copies rows from src to dst. +// +// The underlying types of src and dst are tested to determine if they expose +// information about the schema of rows that are read and expected to be +// written. If the schema information are available but do not match, the +// function will attempt to automatically convert the rows from the source +// schema to the destination. +// +// As an optimization, the src argument may implement RowWriterTo to bypass +// the default row copy logic and provide its own. The dst argument may also +// implement RowReaderFrom for the same purpose. +// +// The function returns the number of rows written, or any error encountered +// other than io.EOF. +func CopyRows(dst RowWriter, src RowReader) (int64, error) { + return copyRows(dst, src, nil) +} + +func copyRows(dst RowWriter, src RowReader, buf []Row) (written int64, err error) { + targetSchema := targetSchemaOf(dst) + sourceSchema := sourceSchemaOf(src) + + if targetSchema != nil && sourceSchema != nil { + if !EqualNodes(targetSchema, sourceSchema) { + conv, err := Convert(targetSchema, sourceSchema) + if err != nil { + return 0, err + } + // The conversion effectively disables a potential optimization + // if the source reader implemented RowWriterTo. It is a trade off + // we are making to optimize for safety rather than performance. + // + // Entering this code path should not be the common case tho, it is + // most often used when parquet schemas are evolving, but we expect + // that the majority of files of an application to be sharing a + // common schema. + src = ConvertRowReader(src, conv) + } + } + + if wt, ok := src.(RowWriterTo); ok { + return wt.WriteRowsTo(dst) + } + + if rf, ok := dst.(RowReaderFrom); ok { + return rf.ReadRowsFrom(src) + } + + if len(buf) == 0 { + buf = make([]Row, defaultRowBufferSize) + } + + defer clearRows(buf) + + for { + rn, err := src.ReadRows(buf) + + if rn > 0 { + wn, err := dst.WriteRows(buf[:rn]) + if err != nil { + return written, err + } + + written += int64(wn) + } + + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + return written, err + } + + if rn == 0 { + return written, io.ErrNoProgress + } + } +} + +func makeRows(n int) []Row { + buf := make([]Value, n) + row := make([]Row, n) + for i := range row { + row[i] = buf[i : i : i+1] + } + return row +} + +func clearRows(rows []Row) { + for i, values := range rows { + clearValues(values) + rows[i] = values[:0] + } +} + +func sourceSchemaOf(r RowReader) *Schema { + if rrs, ok := r.(RowReaderWithSchema); ok { + return rrs.Schema() + } + return nil +} + +func targetSchemaOf(w RowWriter) *Schema { + if rws, ok := w.(RowWriterWithSchema); ok { + return rws.Schema() + } + return nil +} + +// ============================================================================= +// Functions returning closures are marked with "go:noinline" below to prevent +// losing naming information of the closure in stack traces. +// +// Because some of the functions are very short (simply return a closure), the +// compiler inlines when at their call site, which result in the closure being +// named something like parquet.deconstructFuncOf.func2 instead of the original +// parquet.deconstructFuncOfLeaf.func1; the latter being much more meaningful +// when reading CPU or memory profiles. +// ============================================================================= + +type levels struct { + repetitionDepth byte + repetitionLevel byte + definitionLevel byte +} + +// deconstructFunc accepts a row, the current levels, the value to deserialize +// the current column onto, and returns the row minus the deserialied value(s) +// It recurses until it hits a leaf node, then deserializes that value +// individually as the base case. +type deconstructFunc func([][]Value, levels, reflect.Value) + +func deconstructFuncOf(columnIndex int16, node Node) (int16, deconstructFunc) { + switch { + case node.Optional(): + return deconstructFuncOfOptional(columnIndex, node) + case node.Repeated(): + return deconstructFuncOfRepeated(columnIndex, node) + case isList(node): + return deconstructFuncOfList(columnIndex, node) + case isMap(node): + return deconstructFuncOfMap(columnIndex, node) + default: + return deconstructFuncOfRequired(columnIndex, node) + } +} + +//go:noinline +func deconstructFuncOfOptional(columnIndex int16, node Node) (int16, deconstructFunc) { + columnIndex, deconstruct := deconstructFuncOf(columnIndex, Required(node)) + return columnIndex, func(columns [][]Value, levels levels, value reflect.Value) { + if value.IsValid() { + if value.IsZero() { + value = reflect.Value{} + } else { + if value.Kind() == reflect.Ptr { + value = value.Elem() + } + levels.definitionLevel++ + } + } + deconstruct(columns, levels, value) + } +} + +//go:noinline +func deconstructFuncOfRepeated(columnIndex int16, node Node) (int16, deconstructFunc) { + columnIndex, deconstruct := deconstructFuncOf(columnIndex, Required(node)) + return columnIndex, func(columns [][]Value, levels levels, value reflect.Value) { + if value.Kind() == reflect.Interface { + value = value.Elem() + } + + if !value.IsValid() || value.Len() == 0 { + deconstruct(columns, levels, reflect.Value{}) + return + } + + levels.repetitionDepth++ + levels.definitionLevel++ + + for i, n := 0, value.Len(); i < n; i++ { + deconstruct(columns, levels, value.Index(i)) + levels.repetitionLevel = levels.repetitionDepth + } + } +} + +func deconstructFuncOfRequired(columnIndex int16, node Node) (int16, deconstructFunc) { + switch { + case node.Leaf(): + return deconstructFuncOfLeaf(columnIndex, node) + default: + return deconstructFuncOfGroup(columnIndex, node) + } +} + +func deconstructFuncOfList(columnIndex int16, node Node) (int16, deconstructFunc) { + return deconstructFuncOf(columnIndex, Repeated(listElementOf(node))) +} + +//go:noinline +func deconstructFuncOfMap(columnIndex int16, node Node) (int16, deconstructFunc) { + keyValue := mapKeyValueOf(node) + keyValueType := keyValue.GoType() + keyValueElem := keyValueType.Elem() + keyType := keyValueElem.Field(0).Type + valueType := keyValueElem.Field(1).Type + nextColumnIndex, deconstruct := deconstructFuncOf(columnIndex, schemaOf(keyValueElem)) + return nextColumnIndex, func(columns [][]Value, levels levels, mapValue reflect.Value) { + if !mapValue.IsValid() || mapValue.Len() == 0 { + deconstruct(columns, levels, reflect.Value{}) + return + } + + levels.repetitionDepth++ + levels.definitionLevel++ + + elem := reflect.New(keyValueElem).Elem() + k := elem.Field(0) + v := elem.Field(1) + + for _, key := range mapValue.MapKeys() { + k.Set(key.Convert(keyType)) + v.Set(mapValue.MapIndex(key).Convert(valueType)) + deconstruct(columns, levels, elem) + levels.repetitionLevel = levels.repetitionDepth + } + } +} + +//go:noinline +func deconstructFuncOfGroup(columnIndex int16, node Node) (int16, deconstructFunc) { + fields := node.Fields() + funcs := make([]deconstructFunc, len(fields)) + for i, field := range fields { + columnIndex, funcs[i] = deconstructFuncOf(columnIndex, field) + } + return columnIndex, func(columns [][]Value, levels levels, value reflect.Value) { + if value.IsValid() { + for i, f := range funcs { + f(columns, levels, fields[i].Value(value)) + } + } else { + for _, f := range funcs { + f(columns, levels, value) + } + } + } +} + +//go:noinline +func deconstructFuncOfLeaf(columnIndex int16, node Node) (int16, deconstructFunc) { + if columnIndex > MaxColumnIndex { + panic("row cannot be deconstructed because it has more than 127 columns") + } + typ := node.Type() + kind := typ.Kind() + lt := typ.LogicalType() + valueColumnIndex := ^columnIndex + return columnIndex + 1, func(columns [][]Value, levels levels, value reflect.Value) { + v := Value{} + + if value.IsValid() { + v = makeValue(kind, lt, value) + } + + v.repetitionLevel = levels.repetitionLevel + v.definitionLevel = levels.definitionLevel + v.columnIndex = valueColumnIndex + + columns[columnIndex] = append(columns[columnIndex], v) + } +} + +// "reconstructX" turns a Go value into a Go representation of a Parquet series +// of values + +type reconstructFunc func(reflect.Value, levels, [][]Value) error + +func reconstructFuncOf(columnIndex int16, node Node) (int16, reconstructFunc) { + switch { + case node.Optional(): + return reconstructFuncOfOptional(columnIndex, node) + case node.Repeated(): + return reconstructFuncOfRepeated(columnIndex, node) + case isList(node): + return reconstructFuncOfList(columnIndex, node) + case isMap(node): + return reconstructFuncOfMap(columnIndex, node) + default: + return reconstructFuncOfRequired(columnIndex, node) + } +} + +//go:noinline +func reconstructFuncOfOptional(columnIndex int16, node Node) (int16, reconstructFunc) { + // We convert the optional func to required so that we eventually reach the + // leaf base-case. We're still using the heuristics of optional in the + // returned closure (see levels.definitionLevel++), but we don't actually do + // deserialization here, that happens in the leaf function, hence this line. + nextColumnIndex, reconstruct := reconstructFuncOf(columnIndex, Required(node)) + + return nextColumnIndex, func(value reflect.Value, levels levels, columns [][]Value) error { + levels.definitionLevel++ + + if columns[0][0].definitionLevel < levels.definitionLevel { + value.SetZero() + return nil + } + + if value.Kind() == reflect.Ptr { + if value.IsNil() { + value.Set(reflect.New(value.Type().Elem())) + } + value = value.Elem() + } + + return reconstruct(value, levels, columns) + } +} + +func setMakeSlice(v reflect.Value, n int) reflect.Value { + t := v.Type() + if t.Kind() == reflect.Interface { + t = reflect.TypeOf(([]any)(nil)) + } + s := reflect.MakeSlice(t, n, n) + v.Set(s) + return s +} + +func setNullSlice(v reflect.Value) reflect.Value { + t := v.Type() + if t.Kind() == reflect.Interface { + t = reflect.TypeOf(([]any)(nil)) + } + s := reflect.Zero(t) + v.Set(s) + return s +} + +//go:noinline +func reconstructFuncOfRepeated(columnIndex int16, node Node) (int16, reconstructFunc) { + nextColumnIndex, reconstruct := reconstructFuncOf(columnIndex, Required(node)) + return nextColumnIndex, func(value reflect.Value, levels levels, columns [][]Value) error { + levels.repetitionDepth++ + levels.definitionLevel++ + + if columns[0][0].definitionLevel < levels.definitionLevel { + setMakeSlice(value, 0) + return nil + } + + values := make([][]Value, len(columns)) + column := columns[0] + n := 0 + + for i, column := range columns { + values[i] = column[0:0:len(column)] + } + + for i := 0; i < len(column); { + i++ + n++ + + for i < len(column) && column[i].repetitionLevel > levels.repetitionDepth { + i++ + } + } + + value = setMakeSlice(value, n) + + for i := range n { + for j, column := range values { + column = column[:cap(column)] + if len(column) == 0 { + continue + } + + k := 1 + for k < len(column) && column[k].repetitionLevel > levels.repetitionDepth { + k++ + } + + values[j] = column[:k] + } + + if err := reconstruct(value.Index(i), levels, values); err != nil { + return err + } + + for j, column := range values { + values[j] = column[len(column):len(column):cap(column)] + } + + levels.repetitionLevel = levels.repetitionDepth + } + + return nil + } +} + +func reconstructFuncOfRequired(columnIndex int16, node Node) (int16, reconstructFunc) { + switch { + case node.Leaf(): + return reconstructFuncOfLeaf(columnIndex, node) + default: + return reconstructFuncOfGroup(columnIndex, node) + } +} + +func reconstructFuncOfList(columnIndex int16, node Node) (int16, reconstructFunc) { + return reconstructFuncOf(columnIndex, Repeated(listElementOf(node))) +} + +//go:noinline +func reconstructFuncOfMap(columnIndex int16, node Node) (int16, reconstructFunc) { + keyValue := mapKeyValueOf(node) + keyValueType := keyValue.GoType() + keyValueElem := keyValueType.Elem() + nextColumnIndex, reconstruct := reconstructFuncOf(columnIndex, schemaOf(keyValueElem)) + return nextColumnIndex, func(value reflect.Value, levels levels, columns [][]Value) error { + levels.repetitionDepth++ + levels.definitionLevel++ + + if columns[0][0].definitionLevel < levels.definitionLevel { + value.Set(reflect.MakeMap(value.Type())) + return nil + } + + values := make([][]Value, len(columns)) + column := columns[0] + t := value.Type() + if t.Kind() == reflect.Interface { + t = reflect.TypeOf((map[string]any)(nil)) + } + k := t.Key() + v := t.Elem() + n := 0 + + for i, column := range columns { + values[i] = column[0:0:len(column)] + } + + for i := 0; i < len(column); { + i++ + n++ + + for i < len(column) && column[i].repetitionLevel > levels.repetitionDepth { + i++ + } + } + + if value.IsNil() { + m := reflect.MakeMapWithSize(t, n) + value.Set(m) + value = m // track map instead of any for read[any]() + } + + elem := reflect.New(keyValueElem).Elem() + for range n { + for j, column := range values { + column = column[:cap(column)] + k := 1 + + for k < len(column) && column[k].repetitionLevel > levels.repetitionDepth { + k++ + } + + values[j] = column[:k] + } + + if err := reconstruct(elem, levels, values); err != nil { + return err + } + + for j, column := range values { + values[j] = column[len(column):len(column):cap(column)] + } + + value.SetMapIndex(elem.Field(0).Convert(k), elem.Field(1).Convert(v)) + elem.SetZero() + levels.repetitionLevel = levels.repetitionDepth + } + + return nil + } +} + +//go:noinline +func reconstructFuncOfGroup(columnIndex int16, node Node) (int16, reconstructFunc) { + fields := node.Fields() + funcs := make([]reconstructFunc, len(fields)) + columnOffsets := make([]int16, len(fields)) + firstColumnIndex := columnIndex + + for i, field := range fields { + columnIndex, funcs[i] = reconstructFuncOf(columnIndex, field) + columnOffsets[i] = columnIndex - firstColumnIndex + } + + return columnIndex, func(value reflect.Value, levels levels, columns [][]Value) error { + if value.Kind() == reflect.Interface { + value.Set(reflect.MakeMap(reflect.TypeOf((map[string]any)(nil)))) + value = value.Elem() + } + + if value.Kind() == reflect.Map { + elemType := value.Type().Elem() + name := reflect.New(reflect.TypeOf("")).Elem() + elem := reflect.New(elemType).Elem() + + if value.Len() > 0 { + value.Set(reflect.MakeMap(value.Type())) + } + + off := int16(0) + + for i, f := range funcs { + name.SetString(fields[i].Name()) + end := columnOffsets[i] + err := f(elem, levels, columns[off:end:end]) + if err != nil { + return fmt.Errorf("%s โ†’ %w", name, err) + } + off = end + value.SetMapIndex(name, elem) + elem.SetZero() + } + } else { + off := int16(0) + + for i, f := range funcs { + end := columnOffsets[i] + err := f(fields[i].Value(value), levels, columns[off:end:end]) + if err != nil { + return fmt.Errorf("%s โ†’ %w", fields[i].Name(), err) + } + off = end + } + } + + return nil + } +} + +//go:noinline +func reconstructFuncOfLeaf(columnIndex int16, node Node) (int16, reconstructFunc) { + typ := node.Type() + return columnIndex + 1, func(value reflect.Value, _ levels, columns [][]Value) error { + column := columns[0] + if len(column) == 0 { + return fmt.Errorf("no values found in parquet row for column %d", columnIndex) + } + return typ.AssignValue(value, column[0]) + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/row_buffer.go b/vendor/github.com/parquet-go/parquet-go/row_buffer.go new file mode 100644 index 0000000000..9f98e4d057 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/row_buffer.go @@ -0,0 +1,463 @@ +package parquet + +import ( + "io" + "sort" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" +) + +// RowBuffer is an implementation of the RowGroup interface which stores parquet +// rows in memory. +// +// Unlike GenericBuffer which uses a column layout to store values in memory +// buffers, RowBuffer uses a row layout. The use of row layout provides greater +// efficiency when sorting the buffer, which is the primary use case for the +// RowBuffer type. Applications which intend to sort rows prior to writing them +// to a parquet file will often see lower CPU utilization from using a RowBuffer +// than a GenericBuffer. +// +// RowBuffer values are not safe to use concurrently from multiple goroutines. +type RowBuffer[T any] struct { + alloc rowAllocator + schema *Schema + sorting []SortingColumn + rows []Row + values []Value + compare func(Row, Row) int +} + +// NewRowBuffer constructs a new row buffer. +func NewRowBuffer[T any](options ...RowGroupOption) *RowBuffer[T] { + config := DefaultRowGroupConfig() + config.Apply(options...) + if err := config.Validate(); err != nil { + panic(err) + } + + t := typeOf[T]() + if config.Schema == nil && t != nil { + config.Schema = schemaOf(dereference(t)) + } + + if config.Schema == nil { + panic("row buffer must be instantiated with schema or concrete type.") + } + + return &RowBuffer[T]{ + schema: config.Schema, + sorting: config.Sorting.SortingColumns, + compare: config.Schema.Comparator(config.Sorting.SortingColumns...), + } +} + +// Reset clears the content of the buffer without releasing its memory. +func (buf *RowBuffer[T]) Reset() { + for i := range buf.rows { + buf.rows[i] = nil + } + for i := range buf.values { + buf.values[i] = Value{} + } + buf.rows = buf.rows[:0] + buf.values = buf.values[:0] + buf.alloc.reset() +} + +// NumRows returns the number of rows currently written to the buffer. +func (buf *RowBuffer[T]) NumRows() int64 { return int64(len(buf.rows)) } + +// ColumnChunks returns a view of the buffer's columns. +// +// Note that reading columns of a RowBuffer will be less efficient than reading +// columns of a GenericBuffer since the latter uses a column layout. This method +// is mainly exposed to satisfy the RowGroup interface, applications which need +// compute-efficient column scans on in-memory buffers should likely use a +// GenericBuffer instead. +// +// The returned column chunks are snapshots at the time the method is called, +// they remain valid until the next call to Reset on the buffer. +func (buf *RowBuffer[T]) ColumnChunks() []ColumnChunk { + columns := buf.schema.Columns() + chunks := make([]rowBufferColumnChunk, len(columns)) + + for i, column := range columns { + leafColumn, _ := buf.schema.Lookup(column...) + chunks[i] = rowBufferColumnChunk{ + page: rowBufferPage{ + rows: buf.rows, + typ: leafColumn.Node.Type(), + column: leafColumn.ColumnIndex, + maxRepetitionLevel: byte(leafColumn.MaxRepetitionLevel), + maxDefinitionLevel: byte(leafColumn.MaxDefinitionLevel), + }, + } + } + + columnChunks := make([]ColumnChunk, len(chunks)) + for i := range chunks { + columnChunks[i] = &chunks[i] + } + return columnChunks +} + +// SortingColumns returns the list of columns that rows are expected to be +// sorted by. +// +// The list of sorting columns is configured when the buffer is created and used +// when it is sorted. +// +// Note that unless the buffer is explicitly sorted, there are no guarantees +// that the rows it contains will be in the order specified by the sorting +// columns. +func (buf *RowBuffer[T]) SortingColumns() []SortingColumn { return buf.sorting } + +// Schema returns the schema of rows in the buffer. +func (buf *RowBuffer[T]) Schema() *Schema { return buf.schema } + +// Len returns the number of rows in the buffer. +// +// The method contributes to satisfying sort.Interface. +func (buf *RowBuffer[T]) Len() int { return len(buf.rows) } + +// Less compares the rows at index i and j according to the sorting columns +// configured on the buffer. +// +// The method contributes to satisfying sort.Interface. +func (buf *RowBuffer[T]) Less(i, j int) bool { + return buf.compare(buf.rows[i], buf.rows[j]) < 0 +} + +// Swap exchanges the rows at index i and j in the buffer. +// +// The method contributes to satisfying sort.Interface. +func (buf *RowBuffer[T]) Swap(i, j int) { + buf.rows[i], buf.rows[j] = buf.rows[j], buf.rows[i] +} + +// Rows returns a Rows instance exposing rows stored in the buffer. +// +// The rows returned are a snapshot at the time the method is called. +// The returned rows and values read from it remain valid until the next call +// to Reset on the buffer. +func (buf *RowBuffer[T]) Rows() Rows { + return &rowBufferRows{rows: buf.rows, schema: buf.schema} +} + +// Write writes rows to the buffer, returning the number of rows written. +func (buf *RowBuffer[T]) Write(rows []T) (int, error) { + for i := range rows { + off := len(buf.values) + buf.values = buf.schema.Deconstruct(buf.values, &rows[i]) + end := len(buf.values) + row := buf.values[off:end:end] + buf.alloc.capture(row) + buf.rows = append(buf.rows, row) + } + return len(rows), nil +} + +// WriteRows writes parquet rows to the buffer, returing the number of rows +// written. +func (buf *RowBuffer[T]) WriteRows(rows []Row) (int, error) { + for i := range rows { + off := len(buf.values) + buf.values = append(buf.values, rows[i]...) + end := len(buf.values) + row := buf.values[off:end:end] + buf.alloc.capture(row) + buf.rows = append(buf.rows, row) + } + return len(rows), nil +} + +type rowBufferColumnChunk struct{ page rowBufferPage } + +func (c *rowBufferColumnChunk) Type() Type { return c.page.Type() } + +func (c *rowBufferColumnChunk) Column() int { return c.page.Column() } + +func (c *rowBufferColumnChunk) Pages() Pages { return onePage(&c.page) } + +func (c *rowBufferColumnChunk) ColumnIndex() (ColumnIndex, error) { return nil, nil } + +func (c *rowBufferColumnChunk) OffsetIndex() (OffsetIndex, error) { return nil, nil } + +func (c *rowBufferColumnChunk) BloomFilter() BloomFilter { return nil } + +func (c *rowBufferColumnChunk) NumValues() int64 { return c.page.NumValues() } + +type rowBufferPage struct { + rows []Row + typ Type + column int + maxRepetitionLevel byte + maxDefinitionLevel byte +} + +func (p *rowBufferPage) Type() Type { return p.typ } + +func (p *rowBufferPage) Column() int { return p.column } + +func (p *rowBufferPage) Dictionary() Dictionary { return nil } + +func (p *rowBufferPage) NumRows() int64 { return int64(len(p.rows)) } + +func (p *rowBufferPage) NumValues() int64 { + numValues := int64(0) + p.scan(func(value Value) { + if !value.isNull() { + numValues++ + } + }) + return numValues +} + +func (p *rowBufferPage) NumNulls() int64 { + numNulls := int64(0) + p.scan(func(value Value) { + if value.isNull() { + numNulls++ + } + }) + return numNulls +} + +func (p *rowBufferPage) Bounds() (min, max Value, ok bool) { + p.scan(func(value Value) { + if !value.IsNull() { + switch { + case !ok: + min, max, ok = value, value, true + case p.typ.Compare(value, min) < 0: + min = value + case p.typ.Compare(value, max) > 0: + max = value + } + } + }) + return min, max, ok +} + +func (p *rowBufferPage) Size() int64 { return 0 } + +func (p *rowBufferPage) Values() ValueReader { + return &rowBufferPageValueReader{ + page: p, + columnIndex: ^int16(p.column), + } +} + +func (p *rowBufferPage) Clone() Page { + rows := make([]Row, len(p.rows)) + for i := range rows { + rows[i] = p.rows[i].Clone() + } + return &rowBufferPage{ + rows: rows, + typ: p.typ, + column: p.column, + } +} + +func (p *rowBufferPage) Slice(i, j int64) Page { + return &rowBufferPage{ + rows: p.rows[i:j], + typ: p.typ, + column: p.column, + } +} + +func (p *rowBufferPage) RepetitionLevels() (repetitionLevels []byte) { + if p.maxRepetitionLevel != 0 { + repetitionLevels = make([]byte, 0, len(p.rows)) + p.scan(func(value Value) { + repetitionLevels = append(repetitionLevels, value.repetitionLevel) + }) + } + return repetitionLevels +} + +func (p *rowBufferPage) DefinitionLevels() (definitionLevels []byte) { + if p.maxDefinitionLevel != 0 { + definitionLevels = make([]byte, 0, len(p.rows)) + p.scan(func(value Value) { + definitionLevels = append(definitionLevels, value.definitionLevel) + }) + } + return definitionLevels +} + +func (p *rowBufferPage) Data() encoding.Values { + switch p.typ.Kind() { + case Boolean: + values := make([]byte, (len(p.rows)+7)/8) + numValues := 0 + p.scanNonNull(func(value Value) { + if value.boolean() { + i := uint(numValues) / 8 + j := uint(numValues) % 8 + values[i] |= 1 << j + } + numValues++ + }) + return encoding.BooleanValues(values[:(numValues+7)/8]) + + case Int32: + values := make([]int32, 0, len(p.rows)) + p.scanNonNull(func(value Value) { values = append(values, value.int32()) }) + return encoding.Int32Values(values) + + case Int64: + values := make([]int64, 0, len(p.rows)) + p.scanNonNull(func(value Value) { values = append(values, value.int64()) }) + return encoding.Int64Values(values) + + case Int96: + values := make([]deprecated.Int96, 0, len(p.rows)) + p.scanNonNull(func(value Value) { values = append(values, value.int96()) }) + return encoding.Int96Values(values) + + case Float: + values := make([]float32, 0, len(p.rows)) + p.scanNonNull(func(value Value) { values = append(values, value.float()) }) + return encoding.FloatValues(values) + + case Double: + values := make([]float64, 0, len(p.rows)) + p.scanNonNull(func(value Value) { values = append(values, value.double()) }) + return encoding.DoubleValues(values) + + case ByteArray: + values := make([]byte, 0, p.typ.EstimateSize(len(p.rows))) + offsets := make([]uint32, 0, len(p.rows)) + p.scanNonNull(func(value Value) { + offsets = append(offsets, uint32(len(values))) + values = append(values, value.byteArray()...) + }) + offsets = append(offsets, uint32(len(values))) + return encoding.ByteArrayValues(values, offsets) + + case FixedLenByteArray: + length := p.typ.Length() + values := make([]byte, 0, length*len(p.rows)) + p.scanNonNull(func(value Value) { values = append(values, value.byteArray()...) }) + return encoding.FixedLenByteArrayValues(values, length) + + default: + return encoding.Values{} + } +} + +func (p *rowBufferPage) scan(f func(Value)) { + columnIndex := ^int16(p.column) + + for _, row := range p.rows { + for _, value := range row { + if value.columnIndex == columnIndex { + f(value) + } + } + } +} + +func (p *rowBufferPage) scanNonNull(f func(Value)) { + p.scan(func(value Value) { + if !value.isNull() { + f(value) + } + }) +} + +type rowBufferPageValueReader struct { + page *rowBufferPage + rowIndex int + valueIndex int + columnIndex int16 +} + +func (r *rowBufferPageValueReader) ReadValues(values []Value) (n int, err error) { + for n < len(values) && r.rowIndex < len(r.page.rows) { + for n < len(values) && r.valueIndex < len(r.page.rows[r.rowIndex]) { + if v := r.page.rows[r.rowIndex][r.valueIndex]; v.columnIndex == r.columnIndex { + values[n] = v + n++ + } + r.valueIndex++ + } + r.rowIndex++ + r.valueIndex = 0 + } + if r.rowIndex == len(r.page.rows) { + err = io.EOF + } + return n, err +} + +type rowBufferRows struct { + rows []Row + index int + schema *Schema +} + +func (r *rowBufferRows) Close() error { + r.index = -1 + return nil +} + +func (r *rowBufferRows) Schema() *Schema { + return r.schema +} + +func (r *rowBufferRows) SeekToRow(rowIndex int64) error { + if rowIndex < 0 { + return ErrSeekOutOfRange + } + + if r.index < 0 { + return io.ErrClosedPipe + } + + maxRowIndex := int64(len(r.rows)) + if rowIndex > maxRowIndex { + rowIndex = maxRowIndex + } + + r.index = int(rowIndex) + return nil +} + +func (r *rowBufferRows) ReadRows(rows []Row) (n int, err error) { + if r.index < 0 { + return 0, io.EOF + } + + if n = len(r.rows) - r.index; n > len(rows) { + n = len(rows) + } + + for i, row := range r.rows[r.index : r.index+n] { + rows[i] = append(rows[i][:0], row...) + } + + if r.index += n; r.index == len(r.rows) { + err = io.EOF + } + + return n, err +} + +func (r *rowBufferRows) WriteRowsTo(w RowWriter) (int64, error) { + n, err := w.WriteRows(r.rows[r.index:]) + r.index += n + return int64(n), err +} + +var ( + _ RowGroup = (*RowBuffer[any])(nil) + _ RowWriter = (*RowBuffer[any])(nil) + _ sort.Interface = (*RowBuffer[any])(nil) + + _ RowWriterTo = (*rowBufferRows)(nil) +) diff --git a/vendor/github.com/parquet-go/parquet-go/row_builder.go b/vendor/github.com/parquet-go/parquet-go/row_builder.go new file mode 100644 index 0000000000..e7eb96a379 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/row_builder.go @@ -0,0 +1,202 @@ +package parquet + +// RowBuilder is a type which helps build parquet rows incrementally by adding +// values to columns. +type RowBuilder struct { + columns [][]Value + models []Value + levels []columnLevel + groups []*columnGroup +} + +type columnLevel struct { + repetitionDepth byte + repetitionLevel byte + definitionLevel byte +} + +type columnGroup struct { + baseColumn []Value + members []int16 + startIndex int16 + endIndex int16 + repetitionLevel byte + definitionLevel byte +} + +// NewRowBuilder constructs a RowBuilder which builds rows for the parquet +// schema passed as argument. +func NewRowBuilder(schema Node) *RowBuilder { + if schema.Leaf() { + panic("schema of row builder must be a group") + } + n := numLeafColumnsOf(schema) + b := &RowBuilder{ + columns: make([][]Value, n), + models: make([]Value, n), + levels: make([]columnLevel, n), + } + buffers := make([]Value, len(b.columns)) + for i := range b.columns { + b.columns[i] = buffers[i : i : i+1] + } + topGroup := &columnGroup{baseColumn: []Value{{}}} + endIndex := b.configure(schema, 0, columnLevel{}, topGroup) + topGroup.endIndex = endIndex + b.groups = append(b.groups, topGroup) + return b +} + +func (b *RowBuilder) configure(node Node, columnIndex int16, level columnLevel, group *columnGroup) (endIndex int16) { + switch { + case node.Optional(): + level.definitionLevel++ + endIndex = b.configure(Required(node), columnIndex, level, group) + + for i := columnIndex; i < endIndex; i++ { + b.models[i].kind = 0 // null if not set + b.models[i].ptr = nil + b.models[i].u64 = 0 + } + + case node.Repeated(): + level.definitionLevel++ + + group = &columnGroup{ + startIndex: columnIndex, + repetitionLevel: level.repetitionDepth, + definitionLevel: level.definitionLevel, + } + + level.repetitionDepth++ + endIndex = b.configure(Required(node), columnIndex, level, group) + + for i := columnIndex; i < endIndex; i++ { + b.models[i].kind = 0 // null if not set + b.models[i].ptr = nil + b.models[i].u64 = 0 + } + + group.endIndex = endIndex + b.groups = append(b.groups, group) + + case node.Leaf(): + typ := node.Type() + kind := typ.Kind() + model := makeValueKind(kind) + model.repetitionLevel = level.repetitionLevel + model.definitionLevel = level.definitionLevel + // FIXED_LEN_BYTE_ARRAY is the only type which needs to be given a + // non-nil zero-value if the field is required. + if kind == FixedLenByteArray { + zero := make([]byte, typ.Length()) + model.ptr = &zero[0] + model.u64 = uint64(len(zero)) + } + group.members = append(group.members, columnIndex) + b.models[columnIndex] = model + b.levels[columnIndex] = level + endIndex = columnIndex + 1 + + default: + endIndex = columnIndex + + for _, field := range node.Fields() { + endIndex = b.configure(field, endIndex, level, group) + } + } + return endIndex +} + +// Add adds columnValue to the column at columnIndex. +func (b *RowBuilder) Add(columnIndex int, columnValue Value) { + level := &b.levels[columnIndex] + columnValue.repetitionLevel = level.repetitionLevel + columnValue.definitionLevel = level.definitionLevel + columnValue.columnIndex = ^int16(columnIndex) + level.repetitionLevel = level.repetitionDepth + b.columns[columnIndex] = append(b.columns[columnIndex], columnValue) +} + +// Next must be called to indicate the start of a new repeated record for the +// column at the given index. +// +// If the column index is part of a repeated group, the builder automatically +// starts a new record for all adjacent columns, the application does not need +// to call this method for each column of the repeated group. +// +// Next must be called after adding a sequence of records. +func (b *RowBuilder) Next(columnIndex int) { + for _, group := range b.groups { + if group.startIndex <= int16(columnIndex) && int16(columnIndex) < group.endIndex { + for i := group.startIndex; i < group.endIndex; i++ { + if level := &b.levels[i]; level.repetitionLevel != 0 { + level.repetitionLevel = group.repetitionLevel + } + } + break + } + } +} + +// Reset clears the internal state of b, making it possible to reuse while +// retaining the internal buffers. +func (b *RowBuilder) Reset() { + for i, column := range b.columns { + clearValues(column) + b.columns[i] = column[:0] + } + for i := range b.levels { + b.levels[i].repetitionLevel = 0 + } +} + +// Row materializes the current state of b into a parquet row. +func (b *RowBuilder) Row() Row { + numValues := 0 + for _, column := range b.columns { + numValues += len(column) + } + return b.AppendRow(make(Row, 0, numValues)) +} + +// AppendRow appends the current state of b to row and returns it. +func (b *RowBuilder) AppendRow(row Row) Row { + for _, group := range b.groups { + maxColumn := group.baseColumn + + for _, columnIndex := range group.members { + if column := b.columns[columnIndex]; len(column) > len(maxColumn) { + maxColumn = column + } + } + + if len(maxColumn) != 0 { + columns := b.columns[group.startIndex:group.endIndex] + + for i, column := range columns { + if len(column) < len(maxColumn) { + n := len(column) + column = append(column, maxColumn[n:]...) + + columnIndex := group.startIndex + int16(i) + model := b.models[columnIndex] + + for n < len(column) { + v := &column[n] + v.kind = model.kind + v.ptr = model.ptr + v.u64 = model.u64 + v.definitionLevel = group.definitionLevel + v.columnIndex = ^columnIndex + n++ + } + + columns[i] = column + } + } + } + } + + return appendRow(row, b.columns) +} diff --git a/vendor/github.com/parquet-go/parquet-go/row_group.go b/vendor/github.com/parquet-go/parquet-go/row_group.go new file mode 100644 index 0000000000..43a6d7a9d7 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/row_group.go @@ -0,0 +1,498 @@ +package parquet + +import ( + "errors" + "fmt" + "io" + + "github.com/parquet-go/parquet-go/internal/debug" +) + +// RowGroup is an interface representing a parquet row group. From the Parquet +// docs, a RowGroup is "a logical horizontal partitioning of the data into rows. +// There is no physical structure that is guaranteed for a row group. A row +// group consists of a column chunk for each column in the dataset." +// +// https://github.com/apache/parquet-format#glossary +type RowGroup interface { + // Returns the number of rows in the group. + NumRows() int64 + + // Returns the list of column chunks in this row group. The chunks are + // ordered in the order of leaf columns from the row group's schema. + // + // If the underlying implementation is not read-only, the returned + // parquet.ColumnChunk may implement other interfaces: for example, + // parquet.ColumnBuffer if the chunk is backed by an in-memory buffer, + // or typed writer interfaces like parquet.Int32Writer depending on the + // underlying type of values that can be written to the chunk. + // + // As an optimization, the row group may return the same slice across + // multiple calls to this method. Applications should treat the returned + // slice as read-only. + ColumnChunks() []ColumnChunk + + // Returns the schema of rows in the group. + Schema() *Schema + + // Returns the list of sorting columns describing how rows are sorted in the + // group. + // + // The method will return an empty slice if the rows are not sorted. + SortingColumns() []SortingColumn + + // Returns a reader exposing the rows of the row group. + // + // As an optimization, the returned parquet.Rows object may implement + // parquet.RowWriterTo, and test the RowWriter it receives for an + // implementation of the parquet.RowGroupWriter interface. + // + // This optimization mechanism is leveraged by the parquet.CopyRows function + // to skip the generic row-by-row copy algorithm and delegate the copy logic + // to the parquet.Rows object. + Rows() Rows +} + +// Rows is an interface implemented by row readers returned by calling the Rows +// method of RowGroup instances. +// +// Applications should call Close when they are done using a Rows instance in +// order to release the underlying resources held by the row sequence. +// +// After calling Close, all attempts to read more rows will return io.EOF. +type Rows interface { + RowReadSeekCloser + Schema() *Schema +} + +// RowGroupReader is an interface implemented by types that expose sequences of +// row groups to the application. +type RowGroupReader interface { + ReadRowGroup() (RowGroup, error) +} + +// RowGroupWriter is an interface implemented by types that allow the program +// to write row groups. +type RowGroupWriter interface { + WriteRowGroup(RowGroup) (int64, error) +} + +// SortingColumn represents a column by which a row group is sorted. +type SortingColumn interface { + // Returns the path of the column in the row group schema, omitting the name + // of the root node. + Path() []string + + // Returns true if the column will sort values in descending order. + Descending() bool + + // Returns true if the column will put null values at the beginning. + NullsFirst() bool +} + +// Ascending constructs a SortingColumn value which dictates to sort the column +// at the path given as argument in ascending order. +func Ascending(path ...string) SortingColumn { return ascending(path) } + +// Descending constructs a SortingColumn value which dictates to sort the column +// at the path given as argument in descending order. +func Descending(path ...string) SortingColumn { return descending(path) } + +// NullsFirst wraps the SortingColumn passed as argument so that it instructs +// the row group to place null values first in the column. +func NullsFirst(sortingColumn SortingColumn) SortingColumn { return nullsFirst{sortingColumn} } + +type ascending []string + +func (asc ascending) String() string { return fmt.Sprintf("ascending(%s)", columnPath(asc)) } +func (asc ascending) Path() []string { return asc } +func (asc ascending) Descending() bool { return false } +func (asc ascending) NullsFirst() bool { return false } + +type descending []string + +func (desc descending) String() string { return fmt.Sprintf("descending(%s)", columnPath(desc)) } +func (desc descending) Path() []string { return desc } +func (desc descending) Descending() bool { return true } +func (desc descending) NullsFirst() bool { return false } + +type nullsFirst struct{ SortingColumn } + +func (nf nullsFirst) String() string { return fmt.Sprintf("nulls_first+%s", nf.SortingColumn) } +func (nf nullsFirst) NullsFirst() bool { return true } + +func searchSortingColumn(sortingColumns []SortingColumn, path columnPath) int { + // There are usually a few sorting columns in a row group, so the linear + // scan is the fastest option and works whether the sorting column list + // is sorted or not. Please revisit this decision if this code path ends + // up being more costly than necessary. + for i, sorting := range sortingColumns { + if path.equal(sorting.Path()) { + return i + } + } + return len(sortingColumns) +} + +func sortingColumnsHavePrefix(sortingColumns, prefix []SortingColumn) bool { + if len(sortingColumns) < len(prefix) { + return false + } + for i, sortingColumn := range prefix { + if !sortingColumnsAreEqual(sortingColumns[i], sortingColumn) { + return false + } + } + return true +} + +func sortingColumnsAreEqual(s1, s2 SortingColumn) bool { + path1 := columnPath(s1.Path()) + path2 := columnPath(s2.Path()) + return path1.equal(path2) && s1.Descending() == s2.Descending() && s1.NullsFirst() == s2.NullsFirst() +} + +type rowGroup struct { + schema *Schema + numRows int64 + columns []ColumnChunk + sorting []SortingColumn +} + +func (r *rowGroup) NumRows() int64 { return r.numRows } +func (r *rowGroup) ColumnChunks() []ColumnChunk { return r.columns } +func (r *rowGroup) SortingColumns() []SortingColumn { return r.sorting } +func (r *rowGroup) Schema() *Schema { return r.schema } +func (r *rowGroup) Rows() Rows { return NewRowGroupRowReader(r) } + +func AsyncRowGroup(base RowGroup) RowGroup { + columnChunks := base.ColumnChunks() + asyncRowGroup := &rowGroup{ + schema: base.Schema(), + numRows: base.NumRows(), + sorting: base.SortingColumns(), + columns: make([]ColumnChunk, len(columnChunks)), + } + asyncColumnChunks := make([]asyncColumnChunk, len(columnChunks)) + for i, columnChunk := range columnChunks { + asyncColumnChunks[i].ColumnChunk = columnChunk + asyncRowGroup.columns[i] = &asyncColumnChunks[i] + } + return asyncRowGroup +} + +type rowGroupRows struct { + schema *Schema + bufsize int + buffers []Value + columns []columnChunkRows + closed bool + rowIndex int64 +} + +type columnChunkRows struct { + offset int32 + length int32 + reader columnChunkValueReader +} + +func (r *rowGroupRows) buffer(i int) []Value { + j := (i + 0) * r.bufsize + k := (i + 1) * r.bufsize + return r.buffers[j:k:k] +} + +// / NewRowGroupRowReader constructs a new row reader for the given row group. +func NewRowGroupRowReader(rowGroup RowGroup) Rows { + return newRowGroupRows(rowGroup.Schema(), rowGroup.ColumnChunks(), defaultValueBufferSize) +} + +func newRowGroupRows(schema *Schema, columns []ColumnChunk, bufferSize int) *rowGroupRows { + r := &rowGroupRows{ + schema: schema, + bufsize: bufferSize, + buffers: make([]Value, len(columns)*bufferSize), + columns: make([]columnChunkRows, len(columns)), + rowIndex: -1, + } + + for i, column := range columns { + var release func(Page) + // Only release pages that are not byte array because the values + // that were read from the page might be retained by the program + // after calls to ReadRows. + switch column.Type().Kind() { + case ByteArray, FixedLenByteArray: + release = func(Page) {} + default: + release = Release + } + r.columns[i].reader.release = release + r.columns[i].reader.pages = column.Pages() + } + + // This finalizer is used to ensure that the goroutines started by calling + // init on the underlying page readers will be shutdown in the event that + // Close isn't called and the rowGroupRows object is garbage collected. + debug.SetFinalizer(r, func(r *rowGroupRows) { r.Close() }) + return r +} + +func (r *rowGroupRows) clear() { + for i, c := range r.columns { + r.columns[i] = columnChunkRows{reader: c.reader} + } + clear(r.buffers) +} + +func (r *rowGroupRows) Reset() { + for i := range r.columns { + r.columns[i].reader.Reset() + } + r.clear() +} + +func (r *rowGroupRows) Close() error { + var errs []error + for i := range r.columns { + c := &r.columns[i] + c.offset = 0 + c.length = 0 + if err := c.reader.Close(); err != nil { + errs = append(errs, err) + } + } + r.clear() + r.closed = true + return errors.Join(errs...) +} + +func (r *rowGroupRows) SeekToRow(rowIndex int64) error { + if r.closed { + return io.ErrClosedPipe + } + if rowIndex != r.rowIndex { + for i := range r.columns { + if err := r.columns[i].reader.SeekToRow(rowIndex); err != nil { + return err + } + } + r.clear() + r.rowIndex = rowIndex + } + return nil +} + +func (r *rowGroupRows) ReadRows(rows []Row) (int, error) { + if r.closed { + return 0, io.EOF + } + + for rowIndex := range rows { + rows[rowIndex] = rows[rowIndex][:0] + } + + // When this is the first call to ReadRows, we issue a seek to the first row + // because this starts prefetching pages asynchronously on columns. + // + // This condition does not apply if SeekToRow was called before ReadRows, + // only when ReadRows is the very first method called on the row reader. + if r.rowIndex < 0 { + if err := r.SeekToRow(0); err != nil { + return 0, err + } + } + + eofCount := 0 + rowCount := 0 + +readColumnValues: + for columnIndex := range r.columns { + c := &r.columns[columnIndex] + b := r.buffer(columnIndex) + eof := false + + for rowIndex := range rows { + numValuesInRow := 1 + + for { + if c.offset == c.length { + n, err := c.reader.ReadValues(b) + c.offset = 0 + c.length = int32(n) + + if n == 0 { + if err == io.EOF { + eof = true + eofCount++ + break + } + return 0, err + } + } + + values := b[c.offset:c.length:c.length] + for numValuesInRow < len(values) && values[numValuesInRow].repetitionLevel != 0 { + numValuesInRow++ + } + if numValuesInRow == 0 { + break + } + + rows[rowIndex] = append(rows[rowIndex], values[:numValuesInRow]...) + rowCount = max(rowCount, rowIndex+1) + c.offset += int32(numValuesInRow) + + if numValuesInRow != len(values) { + break + } + if eof { + continue readColumnValues + } + numValuesInRow = 0 + } + } + } + + var err error + if eofCount > 0 { + err = io.EOF + } + r.rowIndex += int64(rowCount) + return rowCount, err +} + +func (r *rowGroupRows) Schema() *Schema { + return r.schema +} + +type seekRowGroup struct { + base RowGroup + seek int64 + columns []ColumnChunk +} + +func (g *seekRowGroup) NumRows() int64 { + return g.base.NumRows() - g.seek +} + +func (g *seekRowGroup) ColumnChunks() []ColumnChunk { + return g.columns +} + +func (g *seekRowGroup) Schema() *Schema { + return g.base.Schema() +} + +func (g *seekRowGroup) SortingColumns() []SortingColumn { + return g.base.SortingColumns() +} + +func (g *seekRowGroup) Rows() Rows { + rows := g.base.Rows() + rows.SeekToRow(g.seek) + return rows +} + +type seekColumnChunk struct { + base ColumnChunk + seek int64 +} + +func (c *seekColumnChunk) Type() Type { + return c.base.Type() +} + +func (c *seekColumnChunk) Column() int { + return c.base.Column() +} + +func (c *seekColumnChunk) Pages() Pages { + pages := c.base.Pages() + pages.SeekToRow(c.seek) + return pages +} + +func (c *seekColumnChunk) ColumnIndex() (ColumnIndex, error) { + return c.base.ColumnIndex() +} + +func (c *seekColumnChunk) OffsetIndex() (OffsetIndex, error) { + return c.base.OffsetIndex() +} + +func (c *seekColumnChunk) BloomFilter() BloomFilter { + return c.base.BloomFilter() +} + +func (c *seekColumnChunk) NumValues() int64 { + return c.base.NumValues() +} + +type emptyRowGroup struct { + schema *Schema + columns []ColumnChunk +} + +func newEmptyRowGroup(schema *Schema) *emptyRowGroup { + columns := schema.Columns() + rowGroup := &emptyRowGroup{ + schema: schema, + columns: make([]ColumnChunk, len(columns)), + } + emptyColumnChunks := make([]emptyColumnChunk, len(columns)) + for i, column := range schema.Columns() { + leaf, _ := schema.Lookup(column...) + emptyColumnChunks[i].typ = leaf.Node.Type() + emptyColumnChunks[i].column = int16(leaf.ColumnIndex) + rowGroup.columns[i] = &emptyColumnChunks[i] + } + return rowGroup +} + +func (g *emptyRowGroup) NumRows() int64 { return 0 } +func (g *emptyRowGroup) ColumnChunks() []ColumnChunk { return g.columns } +func (g *emptyRowGroup) Schema() *Schema { return g.schema } +func (g *emptyRowGroup) SortingColumns() []SortingColumn { return nil } +func (g *emptyRowGroup) Rows() Rows { return emptyRows{g.schema} } + +type emptyColumnChunk struct { + typ Type + column int16 +} + +func (c *emptyColumnChunk) Type() Type { return c.typ } +func (c *emptyColumnChunk) Column() int { return int(c.column) } +func (c *emptyColumnChunk) Pages() Pages { return emptyPages{} } +func (c *emptyColumnChunk) ColumnIndex() (ColumnIndex, error) { return emptyColumnIndex{}, nil } +func (c *emptyColumnChunk) OffsetIndex() (OffsetIndex, error) { return emptyOffsetIndex{}, nil } +func (c *emptyColumnChunk) BloomFilter() BloomFilter { return emptyBloomFilter{} } +func (c *emptyColumnChunk) NumValues() int64 { return 0 } + +type emptyBloomFilter struct{} + +func (emptyBloomFilter) ReadAt([]byte, int64) (int, error) { return 0, io.EOF } +func (emptyBloomFilter) Size() int64 { return 0 } +func (emptyBloomFilter) Check(Value) (bool, error) { return false, nil } + +type emptyRows struct{ schema *Schema } + +func (r emptyRows) Close() error { return nil } +func (r emptyRows) Schema() *Schema { return r.schema } +func (r emptyRows) ReadRows([]Row) (int, error) { return 0, io.EOF } +func (r emptyRows) SeekToRow(int64) error { return nil } +func (r emptyRows) WriteRowsTo(RowWriter) (int64, error) { return 0, nil } + +type emptyPages struct{} + +func (emptyPages) ReadPage() (Page, error) { return nil, io.EOF } +func (emptyPages) SeekToRow(int64) error { return nil } +func (emptyPages) Close() error { return nil } + +var ( + _ RowReaderWithSchema = (*rowGroupRows)(nil) + //_ RowWriterTo = (*rowGroupRows)(nil) + + _ RowReaderWithSchema = emptyRows{} + _ RowWriterTo = emptyRows{} +) diff --git a/vendor/github.com/parquet-go/parquet-go/scan.go b/vendor/github.com/parquet-go/parquet-go/scan.go new file mode 100644 index 0000000000..abc287e704 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/scan.go @@ -0,0 +1,33 @@ +package parquet + +import "io" + +// ScanRowReader constructs a RowReader which exposes rows from reader until +// the predicate returns false for one of the rows, or EOF is reached. +func ScanRowReader(reader RowReader, predicate func(Row, int64) bool) RowReader { + return &scanRowReader{reader: reader, predicate: predicate} +} + +type scanRowReader struct { + reader RowReader + predicate func(Row, int64) bool + rowIndex int64 +} + +func (s *scanRowReader) ReadRows(rows []Row) (int, error) { + if s.rowIndex < 0 { + return 0, io.EOF + } + + n, err := s.reader.ReadRows(rows) + + for i, row := range rows[:n] { + if !s.predicate(row, s.rowIndex) { + s.rowIndex = -1 + return i, io.EOF + } + s.rowIndex++ + } + + return n, err +} diff --git a/vendor/github.com/parquet-go/parquet-go/schema.go b/vendor/github.com/parquet-go/parquet-go/schema.go new file mode 100644 index 0000000000..08beebae0f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/schema.go @@ -0,0 +1,1082 @@ +package parquet + +import ( + "fmt" + "math" + "reflect" + "strconv" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" +) + +// Schema represents a parquet schema created from a Go value. +// +// Schema implements the Node interface to represent the root node of a parquet +// schema. +// +// Schema values are safe to use concurrently from multiple goroutines but must +// be passed by referenced after being created because their internal state +// contains synchronization primitives that are not safe to copy. +type Schema struct { + name string + root Node + funcs onceValue[schemaFuncs] + state onceValue[schemaState] +} + +type schemaFuncs struct { + deconstruct deconstructFunc + reconstruct reconstructFunc +} + +type schemaState struct { + mapping columnMapping + columns [][]string +} + +type onceValue[T any] struct { + once sync.Once + value *T +} + +func (v *onceValue[T]) load(f func() *T) *T { + v.once.Do(func() { v.value = f() }) + return v.value +} + +// SchemaOf constructs a parquet schema from a Go value. +// +// The function can construct parquet schemas from struct or pointer-to-struct +// values only. A panic is raised if a Go value of a different type is passed +// to this function. +// +// When creating a parquet Schema from a Go value, the struct fields may contain +// a "parquet" tag to describe properties of the parquet node. The "parquet" tag +// follows the conventional format of Go struct tags: a comma-separated list of +// values describe the options, with the first one defining the name of the +// parquet column. +// +// The following options are also supported in the "parquet" struct tag: +// +// optional | make the parquet column optional +// snappy | sets the parquet column compression codec to snappy +// gzip | sets the parquet column compression codec to gzip +// brotli | sets the parquet column compression codec to brotli +// lz4 | sets the parquet column compression codec to lz4 +// zstd | sets the parquet column compression codec to zstd +// plain | enables the plain encoding (no-op default) +// dict | enables dictionary encoding on the parquet column +// delta | enables delta encoding on the parquet column +// list | for slice types, use the parquet LIST logical type +// enum | for string types, use the parquet ENUM logical type +// uuid | for string and [16]byte types, use the parquet UUID logical type +// decimal | for int32, int64 and [n]byte types, use the parquet DECIMAL logical type +// date | for int32 types use the DATE logical type +// time | for int32 and int64 types use the TIME logical type +// timestamp | for int64 types use the TIMESTAMP logical type with, by default, millisecond precision +// split | for float32/float64, use the BYTE_STREAM_SPLIT encoding +// id(n) | where n is int denoting a column field id. Example id(2) for a column with field id of 2 +// +// # The date logical type is an int32 value of the number of days since the unix epoch +// +// The timestamp precision can be changed by defining which precision to use as an argument. +// Supported precisions are: nanosecond, millisecond and microsecond. Example: +// +// type Message struct { +// TimestampMicros int64 `parquet:"timestamp_micros,timestamp(microsecond)" +// } +// +// Both the time and timestamp tags accept an optional second parameter +// to set the `isAdjustedToUTC` annotation of the parquet logical type. +// Valid values are "utc" or "local". If not specified, the default value +// for this annotation will be "utc", which will set the `isAdjustedToUTC` annotation +// value to true. Example: +// +// type Message struct { +// TimestampMicrosAdjusted int64 `parquet:"timestamp_micros_adjusted,timestamp(microsecond:utc)" +// TimestampMicrosNotAdjusted int64 `parquet:"timestamp_micros_not_adjusted,timestamp(microsecond:local)" +// } +// +// The decimal tag must be followed by two integer parameters, the first integer +// representing the scale and the second the precision; for example: +// +// type Item struct { +// Cost int64 `parquet:"cost,decimal(0:3)"` +// } +// +// Invalid combination of struct tags and Go types, or repeating options will +// cause the function to panic. +// +// As a special case, if the field tag is "-", the field is omitted from the schema +// and the data will not be written into the parquet file(s). +// Note that a field with name "-" can still be generated using the tag "-,". +// +// The configuration of Parquet maps are done via two tags: +// - The `parquet-key` tag allows to configure the key of a map. +// - The parquet-value tag allows users to configure a map's values, for example to declare their native Parquet types. +// +// When configuring a Parquet map, the `parquet` tag will configure the map itself. +// +// For example, the following will set the int64 key of the map to be a timestamp: +// +// type Actions struct { +// Action map[int64]string `parquet:"," parquet-key:",timestamp"` +// } +// +// The schema name is the Go type name of the value. +func SchemaOf(model any) *Schema { + return schemaOf(dereference(reflect.TypeOf(model))) +} + +var cachedSchemas sync.Map // map[reflect.Type]*Schema + +func schemaOf(model reflect.Type) *Schema { + cached, _ := cachedSchemas.Load(model) + schema, _ := cached.(*Schema) + if schema != nil { + return schema + } + if model.Kind() != reflect.Struct { + panic("cannot construct parquet schema from value of type " + model.String()) + } + schema = NewSchema(model.Name(), nodeOf(model, nil)) + if actual, loaded := cachedSchemas.LoadOrStore(model, schema); loaded { + schema = actual.(*Schema) + } + return schema +} + +// NewSchema constructs a new Schema object with the given name and root node. +// +// The function panics if Node contains more leaf columns than supported by the +// package (see parquet.MaxColumnIndex). +func NewSchema(name string, root Node) *Schema { + return &Schema{name: name, root: root} +} + +func dereference(t reflect.Type) reflect.Type { + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t +} + +func makeDeconstructFunc(node Node) (deconstruct deconstructFunc) { + if schema, _ := node.(*Schema); schema != nil { + return schema.lazyLoadFuncs().deconstruct + } + if !node.Leaf() { + _, deconstruct = deconstructFuncOf(0, node) + } + return deconstruct +} + +func makeReconstructFunc(node Node) (reconstruct reconstructFunc) { + if schema, _ := node.(*Schema); schema != nil { + return schema.lazyLoadFuncs().reconstruct + } + if !node.Leaf() { + _, reconstruct = reconstructFuncOf(0, node) + } + return reconstruct +} + +func (s *Schema) lazyLoadFuncs() *schemaFuncs { + return s.funcs.load(func() *schemaFuncs { + return &schemaFuncs{ + deconstruct: makeDeconstructFunc(s.root), + reconstruct: makeReconstructFunc(s.root), + } + }) +} + +func (s *Schema) lazyLoadState() *schemaState { + return s.state.load(func() *schemaState { + mapping, columns := columnMappingOf(s.root) + return &schemaState{ + mapping: mapping, + columns: columns, + } + }) +} + +// ConfigureRowGroup satisfies the RowGroupOption interface, allowing Schema +// instances to be passed to row group constructors to pre-declare the schema of +// the output parquet file. +func (s *Schema) ConfigureRowGroup(config *RowGroupConfig) { config.Schema = s } + +// ConfigureReader satisfies the ReaderOption interface, allowing Schema +// instances to be passed to NewReader to pre-declare the schema of rows +// read from the reader. +func (s *Schema) ConfigureReader(config *ReaderConfig) { config.Schema = s } + +// ConfigureWriter satisfies the WriterOption interface, allowing Schema +// instances to be passed to NewWriter to pre-declare the schema of the +// output parquet file. +func (s *Schema) ConfigureWriter(config *WriterConfig) { config.Schema = s } + +// ID returns field id of the root node. +func (s *Schema) ID() int { return s.root.ID() } + +// String returns a parquet schema representation of s. +func (s *Schema) String() string { return sprint(s.name, s.root) } + +// Name returns the name of s. +func (s *Schema) Name() string { return s.name } + +// Type returns the parquet type of s. +func (s *Schema) Type() Type { return s.root.Type() } + +// Optional returns false since the root node of a parquet schema is always required. +func (s *Schema) Optional() bool { return s.root.Optional() } + +// Repeated returns false since the root node of a parquet schema is always required. +func (s *Schema) Repeated() bool { return s.root.Repeated() } + +// Required returns true since the root node of a parquet schema is always required. +func (s *Schema) Required() bool { return s.root.Required() } + +// Leaf returns true if the root node of the parquet schema is a leaf column. +func (s *Schema) Leaf() bool { return s.root.Leaf() } + +// Fields returns the list of fields on the root node of the parquet schema. +func (s *Schema) Fields() []Field { return s.root.Fields() } + +// Encoding returns the encoding set on the root node of the parquet schema. +func (s *Schema) Encoding() encoding.Encoding { return s.root.Encoding() } + +// Compression returns the compression codec set on the root node of the parquet +// schema. +func (s *Schema) Compression() compress.Codec { return s.root.Compression() } + +// GoType returns the Go type that best represents the schema. +func (s *Schema) GoType() reflect.Type { return s.root.GoType() } + +// Deconstruct deconstructs a Go value and appends it to a row. +// +// The method panics is the structure of the go value does not match the +// parquet schema. +func (s *Schema) Deconstruct(row Row, value any) Row { + state := s.lazyLoadState() + funcs := s.lazyLoadFuncs() + columns := make([][]Value, len(state.columns)) + values := make([]Value, len(state.columns)) + + for i := range columns { + columns[i] = values[i : i : i+1] + } + + v := reflect.ValueOf(value) + for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface { + if v.IsNil() { + v = reflect.Value{} + break + } + v = v.Elem() + } + funcs.deconstruct(columns, levels{}, v) + return appendRow(row, columns) +} + +// Reconstruct reconstructs a Go value from a row. +// +// The go value passed as first argument must be a non-nil pointer for the +// row to be decoded into. +// +// The method panics if the structure of the go value and parquet row do not +// match. +func (s *Schema) Reconstruct(value any, row Row) error { + v := reflect.ValueOf(value) + if !v.IsValid() { + panic("cannot reconstruct row into go value of type ") + } + if v.Kind() != reflect.Ptr { + panic("cannot reconstruct row into go value of non-pointer type " + v.Type().String()) + } + if v.IsNil() { + panic("cannot reconstruct row into nil pointer of type " + v.Type().String()) + } + for v.Kind() == reflect.Ptr { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + + b := valuesSliceBufferPool.Get().(*valuesSliceBuffer) + + state := s.lazyLoadState() + funcs := s.lazyLoadFuncs() + columns := b.reserve(len(state.columns)) + row.Range(func(columnIndex int, columnValues []Value) bool { + if columnIndex < len(columns) { + columns[columnIndex] = columnValues + } + return true + }) + // we avoid the defer penalty by releasing b manually + err := funcs.reconstruct(v, levels{}, columns) + b.release() + return err +} + +type valuesSliceBuffer struct { + values [][]Value +} + +func (v *valuesSliceBuffer) reserve(n int) [][]Value { + if n <= cap(v.values) { + return v.values[:n] + } + // we can try to keep growing by the power of two, but we care more about the + // memory footprint so this should suffice. + // + // The nature of reads tends to be from similar number of columns.The less work + // we do here the better performance we can get. + v.values = make([][]Value, n) + return v.values +} + +func (v *valuesSliceBuffer) release() { + v.values = v.values[:0] + valuesSliceBufferPool.Put(v) +} + +var valuesSliceBufferPool = &sync.Pool{ + New: func() any { + return &valuesSliceBuffer{ + // use 64 as a cache friendly base estimate of max column numbers we will be + // reading. + values: make([][]Value, 0, 64), + } + }, +} + +// Lookup returns the leaf column at the given path. +// +// The path is the sequence of column names identifying a leaf column (not +// including the root). +// +// If the path was not found in the mapping, or if it did not represent a +// leaf column of the parquet schema, the boolean will be false. +func (s *Schema) Lookup(path ...string) (LeafColumn, bool) { + leaf := s.lazyLoadState().mapping.lookup(path) + return LeafColumn{ + Node: leaf.node, + Path: leaf.path, + ColumnIndex: int(leaf.columnIndex), + MaxRepetitionLevel: int(leaf.maxRepetitionLevel), + MaxDefinitionLevel: int(leaf.maxDefinitionLevel), + }, leaf.node != nil +} + +// Columns returns the list of column paths available in the schema. +// +// The method always returns the same slice value across calls to ColumnPaths, +// applications should treat it as immutable. +func (s *Schema) Columns() [][]string { return s.lazyLoadState().columns } + +// Comparator constructs a comparator function which orders rows according to +// the list of sorting columns passed as arguments. +func (s *Schema) Comparator(sortingColumns ...SortingColumn) func(Row, Row) int { + return compareRowsFuncOf(s, sortingColumns) +} + +func (s *Schema) forEachNode(do func(name string, node Node)) { + forEachNodeOf(s.Name(), s, do) +} + +type structNode struct { + gotype reflect.Type + fields []structField +} + +func structNodeOf(t reflect.Type) *structNode { + // Collect struct fields first so we can order them before generating the + // column indexes. + fields := structFieldsOf(t) + + s := &structNode{ + gotype: t, + fields: make([]structField, len(fields)), + } + + for i := range fields { + field := structField{name: fields[i].Name, index: fields[i].Index} + field.Node = makeNodeOf(fields[i].Type, fields[i].Name, []string{ + fields[i].Tag.Get("parquet"), + fields[i].Tag.Get("parquet-key"), + fields[i].Tag.Get("parquet-value"), + }) + s.fields[i] = field + } + + return s +} + +func structFieldsOf(t reflect.Type) []reflect.StructField { + fields := appendStructFields(t, nil, nil, 0) + + for i := range fields { + f := &fields[i] + + if tag := f.Tag.Get("parquet"); tag != "" { + name, _ := split(tag) + if name != "" { + f.Name = name + } + } + } + + return fields +} + +func appendStructFields(t reflect.Type, fields []reflect.StructField, index []int, offset uintptr) []reflect.StructField { + for i, n := 0, t.NumField(); i < n; i++ { + f := t.Field(i) + if tag := f.Tag.Get("parquet"); tag != "" { + name, _ := split(tag) + if tag != "-," && name == "-" { + continue + } + } + + fieldIndex := index[:len(index):len(index)] + fieldIndex = append(fieldIndex, i) + + f.Offset += offset + + if f.Anonymous { + fields = appendStructFields(f.Type, fields, fieldIndex, f.Offset) + } else if f.IsExported() { + f.Index = fieldIndex + fields = append(fields, f) + } + } + return fields +} + +func (s *structNode) Optional() bool { return false } + +func (s *structNode) Repeated() bool { return false } + +func (s *structNode) Required() bool { return true } + +func (s *structNode) Leaf() bool { return false } + +func (s *structNode) Encoding() encoding.Encoding { return nil } + +func (s *structNode) Compression() compress.Codec { return nil } + +func (s *structNode) GoType() reflect.Type { return s.gotype } + +func (s *structNode) ID() int { return 0 } + +func (s *structNode) String() string { return sprint("", s) } + +func (s *structNode) Type() Type { return groupType{} } + +func (s *structNode) Fields() []Field { + fields := make([]Field, len(s.fields)) + for i := range s.fields { + fields[i] = &s.fields[i] + } + return fields +} + +// fieldByIndex is like reflect.Value.FieldByIndex but returns the zero-value of +// reflect.Value if one of the fields was a nil pointer instead of panicking. +func fieldByIndex(v reflect.Value, index []int) reflect.Value { + for _, i := range index { + if v = v.Field(i); v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + v = v.Elem() + break + } else { + v = v.Elem() + } + } + } + return v +} + +type structField struct { + Node + name string + index []int +} + +func (f *structField) Name() string { return f.name } + +func (f *structField) Value(base reflect.Value) reflect.Value { + switch base.Kind() { + case reflect.Map: + return base.MapIndex(reflect.ValueOf(&f.name).Elem()) + case reflect.Ptr: + if base.IsNil() { + base.Set(reflect.New(base.Type().Elem())) + } + return fieldByIndex(base.Elem(), f.index) + default: + if len(f.index) == 1 { + return base.Field(f.index[0]) + } else { + return fieldByIndex(base, f.index) + } + } +} + +func nodeString(t reflect.Type, name string, tag ...string) string { + return fmt.Sprintf("%s %s %v", name, t.String(), tag) +} + +func throwInvalidTag(t reflect.Type, name string, tag string) { + panic(tag + " is an invalid parquet tag: " + nodeString(t, name, tag)) +} + +func throwUnknownTag(t reflect.Type, name string, tag string) { + panic(tag + " is an unrecognized parquet tag: " + nodeString(t, name, tag)) +} + +func throwInvalidNode(t reflect.Type, msg, name string, tag ...string) { + panic(msg + ": " + nodeString(t, name, tag...)) +} + +// FixedLenByteArray decimals are sized based on precision +// this function calculates the necessary byte array size. +func decimalFixedLenByteArraySize(precision int) int { + return int(math.Ceil((math.Log10(2) + float64(precision)) / math.Log10(256))) +} + +func forEachStructTagOption(sf reflect.StructField, do func(t reflect.Type, option, args string)) { + if tag := sf.Tag.Get("parquet"); tag != "" { + _, tag = split(tag) // skip the field name + for tag != "" { + option := "" + args := "" + option, tag = split(tag) + option, args = splitOptionArgs(option) + ft := sf.Type + if ft.Kind() == reflect.Ptr { + ft = ft.Elem() + } + do(ft, option, args) + } + } +} + +func nodeOf(t reflect.Type, tag []string) Node { + switch t { + case reflect.TypeOf(deprecated.Int96{}): + return Leaf(Int96Type) + case reflect.TypeOf(uuid.UUID{}): + return UUID() + case reflect.TypeOf(time.Time{}): + return Timestamp(Nanosecond) + } + + var n Node + switch t.Kind() { + case reflect.Bool: + n = Leaf(BooleanType) + + case reflect.Int, reflect.Int64: + n = Int(64) + + case reflect.Int8, reflect.Int16, reflect.Int32: + n = Int(t.Bits()) + + case reflect.Uint, reflect.Uintptr, reflect.Uint64: + n = Uint(64) + + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + n = Uint(t.Bits()) + + case reflect.Float32: + n = Leaf(FloatType) + + case reflect.Float64: + n = Leaf(DoubleType) + + case reflect.String: + n = String() + + case reflect.Ptr: + n = Optional(nodeOf(t.Elem(), nil)) + + case reflect.Slice: + if elem := t.Elem(); elem.Kind() == reflect.Uint8 { // []byte? + n = Leaf(ByteArrayType) + } else { + n = Repeated(nodeOf(elem, nil)) + } + + case reflect.Array: + if t.Elem().Kind() == reflect.Uint8 { + n = Leaf(FixedLenByteArrayType(t.Len())) + } + + case reflect.Map: + var mapTag, valueTag, keyTag string + if len(tag) > 0 { + mapTag = tag[0] + if len(tag) > 1 { + keyTag = tag[1] + } + if len(tag) >= 2 { + valueTag = tag[2] + } + } + + if strings.Contains(mapTag, "json") { + n = JSON() + } else { + n = Map( + makeNodeOf(t.Key(), t.Name(), []string{keyTag}), + makeNodeOf(t.Elem(), t.Name(), []string{valueTag}), + ) + } + + forEachTagOption([]string{mapTag}, func(option, args string) { + switch option { + case "", "json": + return + case "optional": + n = Optional(n) + case "id": + id, err := parseIDArgs(args) + if err != nil { + throwInvalidTag(t, "map", option) + } + n = FieldID(n, id) + default: + throwUnknownTag(t, "map", option) + } + }) + + case reflect.Struct: + return structNodeOf(t) + } + + if n == nil { + panic("cannot create parquet node from go value of type " + t.String()) + } + + return &goNode{Node: n, gotype: t} +} + +func split(s string) (head, tail string) { + if i := strings.IndexByte(s, ','); i < 0 { + head = s + } else { + head, tail = s[:i], s[i+1:] + } + return +} + +func splitOptionArgs(s string) (option, args string) { + if i := strings.IndexByte(s, '('); i >= 0 { + option = s[:i] + args = s[i:] + } else { + option = s + args = "()" + } + return +} + +func parseDecimalArgs(args string) (scale, precision int, err error) { + if !strings.HasPrefix(args, "(") || !strings.HasSuffix(args, ")") { + return 0, 0, fmt.Errorf("malformed decimal args: %s", args) + } + args = strings.TrimPrefix(args, "(") + args = strings.TrimSuffix(args, ")") + parts := strings.Split(args, ":") + if len(parts) != 2 { + return 0, 0, fmt.Errorf("malformed decimal args: (%s)", args) + } + s, err := strconv.ParseInt(parts[0], 10, 32) + if err != nil { + return 0, 0, err + } + p, err := strconv.ParseInt(parts[1], 10, 32) + if err != nil { + return 0, 0, err + } + return int(s), int(p), nil +} + +func parseIDArgs(args string) (int, error) { + if !strings.HasPrefix(args, "(") || !strings.HasSuffix(args, ")") { + return 0, fmt.Errorf("malformed id args: %s", args) + } + args = strings.TrimPrefix(args, "(") + args = strings.TrimSuffix(args, ")") + return strconv.Atoi(args) +} + +func parseTimestampArgs(args string) (unit TimeUnit, isUTCNormalized bool, err error) { + if !strings.HasPrefix(args, "(") || !strings.HasSuffix(args, ")") { + return nil, false, fmt.Errorf("malformed timestamp args: %s", args) + } + + args = strings.TrimPrefix(args, "(") + args = strings.TrimSuffix(args, ")") + + if len(args) == 0 { + return Millisecond, true, nil + } + + parts := strings.Split(args, ":") + if len(parts) > 2 { + return nil, false, fmt.Errorf("malformed timestamp args: (%s)", args) + } + + unit, err = parseTimeUnit(parts[0]) + if err != nil { + return nil, false, err + } + + adjusted := true + if len(parts) > 1 { + adjusted, err = parseUTCNormalization(parts[1]) + if err != nil { + return nil, false, err + } + } + + return unit, adjusted, nil +} + +func parseTimeUnit(arg string) (TimeUnit, error) { + switch arg { + case "millisecond": + return Millisecond, nil + case "microsecond": + return Microsecond, nil + case "nanosecond": + return Nanosecond, nil + default: + } + + return nil, fmt.Errorf("unknown time unit: %s", arg) +} + +func parseUTCNormalization(arg string) (isUTCNormalized bool, err error) { + switch arg { + case "utc": + return true, nil + case "local": + return false, nil + default: + return false, fmt.Errorf("unknown utc normalization: %s", arg) + } +} + +type goNode struct { + Node + gotype reflect.Type +} + +func (n *goNode) GoType() reflect.Type { return n.gotype } + +var ( + _ RowGroupOption = (*Schema)(nil) + _ ReaderOption = (*Schema)(nil) + _ WriterOption = (*Schema)(nil) +) + +func makeNodeOf(t reflect.Type, name string, tag []string) Node { + var ( + node Node + optional bool + list bool + encoded encoding.Encoding + compressed compress.Codec + fieldID int + ) + + setNode := func(n Node) { + if node != nil { + throwInvalidNode(t, "struct field has multiple logical parquet types declared", name, tag...) + } + node = n + } + + setOptional := func() { + if optional { + throwInvalidNode(t, "struct field has multiple declaration of the optional tag", name, tag...) + } + optional = true + } + + setList := func() { + if list { + throwInvalidNode(t, "struct field has multiple declaration of the list tag", name, tag...) + } + list = true + } + + setEncoding := func(e encoding.Encoding) { + if encoded != nil { + throwInvalidNode(t, "struct field has encoding declared multiple time", name, tag...) + } + encoded = e + } + + setCompression := func(c compress.Codec) { + if compressed != nil { + throwInvalidNode(t, "struct field has compression codecs declared multiple times", name, tag...) + } + compressed = c + } + + forEachTagOption(tag, func(option, args string) { + if t.Kind() == reflect.Map { + node = nodeOf(t, tag) + return + } + switch option { + case "": + return + case "optional": + setOptional() + + case "snappy": + setCompression(&Snappy) + + case "gzip": + setCompression(&Gzip) + + case "brotli": + setCompression(&Brotli) + + case "lz4": + setCompression(&Lz4Raw) + + case "zstd": + setCompression(&Zstd) + + case "uncompressed": + setCompression(&Uncompressed) + + case "plain": + setEncoding(&Plain) + + case "dict": + setEncoding(&RLEDictionary) + + case "json": + setNode(JSON()) + + case "delta": + switch t.Kind() { + case reflect.Int, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint32, reflect.Uint64: + setEncoding(&DeltaBinaryPacked) + case reflect.String: + setEncoding(&DeltaByteArray) + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { // []byte? + setEncoding(&DeltaByteArray) + } else { + throwInvalidTag(t, name, option) + } + case reflect.Array: + if t.Elem().Kind() == reflect.Uint8 { // [N]byte? + setEncoding(&DeltaByteArray) + } else { + throwInvalidTag(t, name, option) + } + default: + switch t { + case reflect.TypeOf(time.Time{}): + setEncoding(&DeltaBinaryPacked) + default: + throwInvalidTag(t, name, option) + } + } + + case "split": + switch t.Kind() { + case reflect.Float32, reflect.Float64: + setEncoding(&ByteStreamSplit) + default: + throwInvalidTag(t, name, option) + } + + case "list": + switch t.Kind() { + case reflect.Slice: + element := nodeOf(t.Elem(), nil) + setNode(element) + setList() + default: + throwInvalidTag(t, name, option) + } + + case "enum": + switch t.Kind() { + case reflect.String: + setNode(Enum()) + default: + throwInvalidTag(t, name, option) + } + + case "uuid": + switch t.Kind() { + case reflect.Array: + if t.Elem().Kind() != reflect.Uint8 || t.Len() != 16 { + throwInvalidTag(t, name, option) + } + default: + throwInvalidTag(t, name, option) + } + + case "decimal": + scale, precision, err := parseDecimalArgs(args) + if err != nil { + throwInvalidTag(t, name, option+args) + } + var baseType Type + switch t.Kind() { + case reflect.Int32: + baseType = Int32Type + case reflect.Int64: + baseType = Int64Type + case reflect.Array, reflect.Slice: + baseType = FixedLenByteArrayType(decimalFixedLenByteArraySize(precision)) + default: + throwInvalidTag(t, name, option) + } + + setNode(Decimal(scale, precision, baseType)) + case "date": + switch t.Kind() { + case reflect.Int32: + setNode(Date()) + default: + throwInvalidTag(t, name, option) + } + case "time": + switch t.Kind() { + case reflect.Int32: + timeUnit, adjusted, err := parseTimestampArgs(args) + if err != nil || timeUnit.Duration() < time.Millisecond { + throwInvalidTag(t, name, option+args) + } + setNode(TimeAdjusted(timeUnit, adjusted)) + case reflect.Int64: + timeUnit, adjusted, err := parseTimestampArgs(args) + if t == reflect.TypeOf(time.Duration(0)) { + if args == "()" { + timeUnit = Nanosecond + } else if timeUnit != Nanosecond { + throwInvalidTag(t, name, option+args) + } + } + if err != nil || timeUnit.Duration() == time.Millisecond { + throwInvalidTag(t, name, option+args) + } + setNode(TimeAdjusted(timeUnit, adjusted)) + default: + throwInvalidTag(t, name, option) + } + case "timestamp": + switch t.Kind() { + case reflect.Int64: + timeUnit, adjusted, err := parseTimestampArgs(args) + if err != nil { + throwInvalidTag(t, name, option+args) + } + setNode(TimestampAdjusted(timeUnit, adjusted)) + default: + switch t { + case reflect.TypeOf(time.Time{}): + timeUnit, adjusted, err := parseTimestampArgs(args) + if err != nil { + throwInvalidTag(t, name, option+args) + } + setNode(TimestampAdjusted(timeUnit, adjusted)) + default: + throwInvalidTag(t, name, option) + } + } + case "id": + id, err := parseIDArgs(args) + if err != nil { + throwInvalidNode(t, "struct field has field id that is not a valid int", name, tag...) + } + fieldID = id + } + }) + + // Special case: an "optional" struct tag on a slice applies to the + // individual items, not the overall list. The least messy way to + // deal with this is at this level, instead of passing down optional + // information into the nodeOf function, and then passing back whether an + // optional tag was applied. + if node == nil && t.Kind() == reflect.Slice { + isUint8 := t.Elem().Kind() == reflect.Uint8 + // Note for strings "optional" applies only to the entire BYTE_ARRAY and + // not each individual byte. + if optional && !isUint8 { + node = Repeated(Optional(nodeOf(t.Elem(), tag))) + // Don't also apply "optional" to the whole list. + optional = false + } + } + + if node == nil { + node = nodeOf(t, tag) + } + + if compressed != nil { + node = Compressed(node, compressed) + } + + if encoded != nil { + node = Encoded(node, encoded) + } + + if list { + node = List(node) + } + + if node.Repeated() && !list { + repeated := node.GoType().Elem() + if repeated.Kind() == reflect.Slice { + // Special case: allow [][]uint as seen in a logical map of strings + if repeated.Elem().Kind() != reflect.Uint8 { + panic("unhandled nested slice on parquet schema without list tag") + } + } + } + + if optional { + node = Optional(node) + } + if fieldID != 0 { + node = FieldID(node, fieldID) + } + return node +} + +func forEachTagOption(tags []string, do func(option, args string)) { + for _, tag := range tags { + _, tag = split(tag) // skip the field name + for tag != "" { + option := "" + option, tag = split(tag) + var args string + option, args = splitOptionArgs(option) + do(option, args) + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/search.go b/vendor/github.com/parquet-go/parquet-go/search.go new file mode 100644 index 0000000000..26d5b8c04e --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/search.go @@ -0,0 +1,116 @@ +package parquet + +// Search is like Find, but uses the default ordering of the given type. Search +// and Find are scoped to a given ColumnChunk and find the pages within a +// ColumnChunk which might contain the result. See Find for more details. +func Search(index ColumnIndex, value Value, typ Type) int { + return Find(index, value, CompareNullsLast(typ.Compare)) +} + +// Find uses the ColumnIndex passed as argument to find the page in a column +// chunk (determined by the given ColumnIndex) that the given value is expected +// to be found in. +// +// The function returns the index of the first page that might contain the +// value. If the function determines that the value does not exist in the +// index, NumPages is returned. +// +// If you want to search the entire parquet file, you must iterate over the +// RowGroups and search each one individually, if there are multiple in the +// file. If you call writer.Flush before closing the file, then you will have +// multiple RowGroups to iterate over, otherwise Flush is called once on Close. +// +// The comparison function passed as last argument is used to determine the +// relative order of values. This should generally be the Compare method of +// the column type, but can sometimes be customized to modify how null values +// are interpreted, for example: +// +// pageIndex := parquet.Find(columnIndex, value, +// parquet.CompareNullsFirst(typ.Compare), +// ) +func Find(index ColumnIndex, value Value, cmp func(Value, Value) int) int { + switch { + case index.IsAscending(): + return binarySearch(index, value, cmp) + default: + return linearSearch(index, value, cmp) + } +} + +func binarySearch(index ColumnIndex, value Value, cmp func(Value, Value) int) int { + n := index.NumPages() + curIdx := 0 + topIdx := n + + // while there's at least one more page to check + for (topIdx - curIdx) > 1 { + + // nextIdx is set to halfway between curIdx and topIdx + nextIdx := ((topIdx - curIdx) / 2) + curIdx + + smallerThanMin := cmp(value, index.MinValue(nextIdx)) + + switch { + // search below pages[nextIdx] + case smallerThanMin < 0: + topIdx = nextIdx + // search pages[nextIdx] and above + case smallerThanMin > 0: + curIdx = nextIdx + case smallerThanMin == 0: + // this case is hit when winValue == value of nextIdx + // we must check below this index to find if there's + // another page before this. + // e.g. searching for first page 3 is in: + // [1,2,3] + // [3,4,5] + // [6,7,8] + + // if the page proceeding this has a maxValue matching the value we're + // searching, continue the search. + // otherwise, we can return early + // + // cases covered by else block + // if cmp(value, index.MaxValue(nextIdx-1)) < 0: the value is only in this page + // if cmp(value, index.MaxValue(nextIdx-1)) > 0: we've got a sorting problem with overlapping pages + // + // bounds check not needed for nextIdx-1 because nextIdx is guaranteed to be at least curIdx + 1 + // line 82 & 85 above + if cmp(value, index.MaxValue(nextIdx-1)) == 0 { + topIdx = nextIdx + } else { + return nextIdx + } + } + } + + // last page check, if it wasn't explicitly found above + if curIdx < n { + + // check pages[curIdx] for value + min := index.MinValue(curIdx) + max := index.MaxValue(curIdx) + + // if value is not in pages[curIdx], then it's not in this columnChunk + if cmp(value, min) < 0 || cmp(value, max) > 0 { + curIdx = n + } + } + + return curIdx +} + +func linearSearch(index ColumnIndex, value Value, cmp func(Value, Value) int) int { + n := index.NumPages() + + for i := range n { + min := index.MinValue(i) + max := index.MaxValue(i) + + if cmp(min, value) <= 0 && cmp(value, max) <= 0 { + return i + } + } + + return n +} diff --git a/vendor/github.com/parquet-go/parquet-go/sorting.go b/vendor/github.com/parquet-go/parquet-go/sorting.go new file mode 100644 index 0000000000..8f3bea18b5 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sorting.go @@ -0,0 +1,224 @@ +package parquet + +import ( + "io" + "sort" +) + +// SortingWriter is a type similar to GenericWriter but it ensures that rows +// are sorted according to the sorting columns configured on the writer. +// +// The writer accumulates rows in an in-memory buffer which is sorted when it +// reaches the target number of rows, then written to a temporary row group. +// When the writer is flushed or closed, the temporary row groups are merged +// into a row group in the output file, ensuring that rows remain sorted in the +// final row group. +// +// Because row groups get encoded and compressed, they hold a lot less memory +// than if all rows were retained in memory. Sorting then merging rows chunks +// also tends to be a lot more efficient than sorting all rows in memory as it +// results in better CPU cache utilization since sorting multi-megabyte arrays +// causes a lot of cache misses since the data set cannot be held in CPU caches. +type SortingWriter[T any] struct { + rowbuf *RowBuffer[T] + writer *GenericWriter[T] + output *GenericWriter[T] + buffer io.ReadWriteSeeker + maxRows int64 + numRows int64 + sorting SortingConfig + dedupe dedupe +} + +// NewSortingWriter constructs a new sorting writer which writes a parquet file +// where rows of each row group are ordered according to the sorting columns +// configured on the writer. +// +// The sortRowCount argument defines the target number of rows that will be +// sorted in memory before being written to temporary row groups. The greater +// this value the more memory is needed to buffer rows in memory. Choosing a +// value that is too small limits the maximum number of rows that can exist in +// the output file since the writer cannot create more than 32K temporary row +// groups to hold the sorted row chunks. +func NewSortingWriter[T any](output io.Writer, sortRowCount int64, options ...WriterOption) *SortingWriter[T] { + config, err := NewWriterConfig(options...) + if err != nil { + panic(err) + } + return &SortingWriter[T]{ + rowbuf: NewRowBuffer[T](&RowGroupConfig{ + Schema: config.Schema, + Sorting: config.Sorting, + }), + writer: NewGenericWriter[T](io.Discard, &WriterConfig{ + CreatedBy: config.CreatedBy, + ColumnPageBuffers: config.ColumnPageBuffers, + ColumnIndexSizeLimit: config.ColumnIndexSizeLimit, + PageBufferSize: config.PageBufferSize, + WriteBufferSize: config.WriteBufferSize, + DataPageVersion: config.DataPageVersion, + Schema: config.Schema, + Compression: config.Compression, + Sorting: config.Sorting, + }), + output: NewGenericWriter[T](output, config), + maxRows: sortRowCount, + sorting: config.Sorting, + } +} + +func (w *SortingWriter[T]) Close() error { + if err := w.Flush(); err != nil { + return err + } + return w.output.Close() +} + +func (w *SortingWriter[T]) Flush() error { + defer w.resetSortingBuffer() + + if err := w.sortAndWriteBufferedRows(); err != nil { + return err + } + + if w.numRows == 0 { + return nil + } + + if err := w.writer.Close(); err != nil { + return err + } + + size, err := w.buffer.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + + f, err := OpenFile(newReaderAt(w.buffer), size, + &FileConfig{ + SkipPageIndex: true, + SkipBloomFilters: true, + ReadBufferSize: defaultReadBufferSize, + }, + ) + if err != nil { + return err + } + + m, err := MergeRowGroups(f.RowGroups(), + &RowGroupConfig{ + Schema: w.Schema(), + Sorting: w.sorting, + }, + ) + if err != nil { + return err + } + + rows := m.Rows() + defer rows.Close() + + reader := RowReader(rows) + if w.sorting.DropDuplicatedRows { + reader = DedupeRowReader(rows, w.rowbuf.compare) + } + + if _, err := CopyRows(w.output, reader); err != nil { + return err + } + + return w.output.Flush() +} + +func (w *SortingWriter[T]) Reset(output io.Writer) { + w.output.Reset(output) + w.rowbuf.Reset() + w.resetSortingBuffer() +} + +func (w *SortingWriter[T]) resetSortingBuffer() { + w.writer.Reset(io.Discard) + w.numRows = 0 + + if w.buffer != nil { + w.sorting.SortingBuffers.PutBuffer(w.buffer) + w.buffer = nil + } +} + +func (w *SortingWriter[T]) Write(rows []T) (int, error) { + return w.writeRows(len(rows), func(i, j int) (int, error) { return w.rowbuf.Write(rows[i:j]) }) +} + +func (w *SortingWriter[T]) WriteRows(rows []Row) (int, error) { + return w.writeRows(len(rows), func(i, j int) (int, error) { return w.rowbuf.WriteRows(rows[i:j]) }) +} + +func (w *SortingWriter[T]) writeRows(numRows int, writeRows func(i, j int) (int, error)) (int, error) { + wn := 0 + + for wn < numRows { + if w.rowbuf.NumRows() >= w.maxRows { + if err := w.sortAndWriteBufferedRows(); err != nil { + return wn, err + } + } + + n := int(w.maxRows - w.rowbuf.NumRows()) + n += wn + if n > numRows { + n = numRows + } + + n, err := writeRows(wn, n) + wn += n + + if err != nil { + return wn, err + } + } + + return wn, nil +} + +func (w *SortingWriter[T]) SetKeyValueMetadata(key, value string) { + w.output.SetKeyValueMetadata(key, value) +} + +func (w *SortingWriter[T]) Schema() *Schema { + return w.output.Schema() +} + +func (w *SortingWriter[T]) sortAndWriteBufferedRows() error { + if w.rowbuf.Len() == 0 { + return nil + } + + defer w.rowbuf.Reset() + sort.Sort(w.rowbuf) + + if w.sorting.DropDuplicatedRows { + w.rowbuf.rows = w.rowbuf.rows[:w.dedupe.deduplicate(w.rowbuf.rows, w.rowbuf.compare)] + defer w.dedupe.reset() + } + + rows := w.rowbuf.Rows() + defer rows.Close() + + if w.buffer == nil { + w.buffer = w.sorting.SortingBuffers.GetBuffer() + w.writer.Reset(w.buffer) + } + + n, err := CopyRows(w.writer, rows) + if err != nil { + return err + } + + if err := w.writer.Flush(); err != nil { + return err + } + + w.numRows += n + return nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/sparse/array.go b/vendor/github.com/parquet-go/parquet-go/sparse/array.go new file mode 100644 index 0000000000..fecfb4dc4d --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sparse/array.go @@ -0,0 +1,320 @@ +package sparse + +import ( + "time" + "unsafe" +) + +type Array struct{ array } + +func UnsafeArray(base unsafe.Pointer, length int, offset uintptr) Array { + return Array{unsafeArray(base, length, offset)} +} + +func (a Array) Len() int { return int(a.len) } +func (a Array) Index(i int) unsafe.Pointer { return a.index(i) } +func (a Array) Slice(i, j int) Array { return Array{a.slice(i, j)} } +func (a Array) Offset(off uintptr) Array { return Array{a.offset(off)} } +func (a Array) BoolArray() BoolArray { return BoolArray{a.array} } +func (a Array) Int8Array() Int8Array { return Int8Array{a.array} } +func (a Array) Int16Array() Int16Array { return Int16Array{a.array} } +func (a Array) Int32Array() Int32Array { return Int32Array{a.array} } +func (a Array) Int64Array() Int64Array { return Int64Array{a.array} } +func (a Array) Float32Array() Float32Array { return Float32Array{a.array} } +func (a Array) Float64Array() Float64Array { return Float64Array{a.array} } +func (a Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Array) Uint32Array() Uint32Array { return Uint32Array{a.array} } +func (a Array) Uint64Array() Uint64Array { return Uint64Array{a.array} } +func (a Array) Uint128Array() Uint128Array { return Uint128Array{a.array} } +func (a Array) StringArray() StringArray { return StringArray{a.array} } +func (a Array) TimeArray() TimeArray { return TimeArray{a.array} } + +type array struct { + ptr unsafe.Pointer + len uintptr + off uintptr +} + +func makeArray[T any](base []T) array { + var z T + return array{ + ptr: unsafe.Pointer(unsafe.SliceData(base)), + len: uintptr(len(base)), + off: unsafe.Sizeof(z), + } +} + +func unsafeArray(base unsafe.Pointer, length int, offset uintptr) array { + return array{ptr: base, len: uintptr(length), off: offset} +} + +func (a array) index(i int) unsafe.Pointer { + if uintptr(i) >= a.len { + panic("index out of bounds") + } + return unsafe.Add(a.ptr, a.off*uintptr(i)) +} + +func (a array) slice(i, j int) array { + if uintptr(i) > a.len || uintptr(j) > a.len || i > j { + panic("slice index out of bounds") + } + return array{ + ptr: unsafe.Add(a.ptr, a.off*uintptr(i)), + len: uintptr(j - i), + off: a.off, + } +} + +func (a array) offset(off uintptr) array { + if a.ptr == nil { + panic("offset of nil array") + } + return array{ + ptr: unsafe.Add(a.ptr, off), + len: a.len, + off: a.off, + } +} + +type BoolArray struct{ array } + +func MakeBoolArray(values []bool) BoolArray { + return BoolArray{makeArray(values)} +} + +func UnsafeBoolArray(base unsafe.Pointer, length int, offset uintptr) BoolArray { + return BoolArray{unsafeArray(base, length, offset)} +} + +func (a BoolArray) Len() int { return int(a.len) } +func (a BoolArray) Index(i int) bool { return *(*byte)(a.index(i)) != 0 } +func (a BoolArray) Slice(i, j int) BoolArray { return BoolArray{a.slice(i, j)} } +func (a BoolArray) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a BoolArray) UnsafeArray() Array { return Array{a.array} } + +type Int8Array struct{ array } + +func MakeInt8Array(values []int8) Int8Array { + return Int8Array{makeArray(values)} +} + +func UnsafeInt8Array(base unsafe.Pointer, length int, offset uintptr) Int8Array { + return Int8Array{unsafeArray(base, length, offset)} +} + +func (a Int8Array) Len() int { return int(a.len) } +func (a Int8Array) Index(i int) int8 { return *(*int8)(a.index(i)) } +func (a Int8Array) Slice(i, j int) Int8Array { return Int8Array{a.slice(i, j)} } +func (a Int8Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Int8Array) UnsafeArray() Array { return Array{a.array} } + +type Int16Array struct{ array } + +func MakeInt16Array(values []int16) Int16Array { + return Int16Array{makeArray(values)} +} + +func UnsafeInt16Array(base unsafe.Pointer, length int, offset uintptr) Int16Array { + return Int16Array{unsafeArray(base, length, offset)} +} + +func (a Int16Array) Len() int { return int(a.len) } +func (a Int16Array) Index(i int) int16 { return *(*int16)(a.index(i)) } +func (a Int16Array) Slice(i, j int) Int16Array { return Int16Array{a.slice(i, j)} } +func (a Int16Array) Int8Array() Int8Array { return Int8Array{a.array} } +func (a Int16Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Int16Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Int16Array) UnsafeArray() Array { return Array{a.array} } + +type Int32Array struct{ array } + +func MakeInt32Array(values []int32) Int32Array { + return Int32Array{makeArray(values)} +} + +func UnsafeInt32Array(base unsafe.Pointer, length int, offset uintptr) Int32Array { + return Int32Array{unsafeArray(base, length, offset)} +} + +func (a Int32Array) Len() int { return int(a.len) } +func (a Int32Array) Index(i int) int32 { return *(*int32)(a.index(i)) } +func (a Int32Array) Slice(i, j int) Int32Array { return Int32Array{a.slice(i, j)} } +func (a Int32Array) Int8Array() Int8Array { return Int8Array{a.array} } +func (a Int32Array) Int16Array() Int16Array { return Int16Array{a.array} } +func (a Int32Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Int32Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Int32Array) Uint32Array() Uint32Array { return Uint32Array{a.array} } +func (a Int32Array) UnsafeArray() Array { return Array{a.array} } + +type Int64Array struct{ array } + +func MakeInt64Array(values []int64) Int64Array { + return Int64Array{makeArray(values)} +} + +func UnsafeInt64Array(base unsafe.Pointer, length int, offset uintptr) Int64Array { + return Int64Array{unsafeArray(base, length, offset)} +} + +func (a Int64Array) Len() int { return int(a.len) } +func (a Int64Array) Index(i int) int64 { return *(*int64)(a.index(i)) } +func (a Int64Array) Slice(i, j int) Int64Array { return Int64Array{a.slice(i, j)} } +func (a Int64Array) Int8Array() Int8Array { return Int8Array{a.array} } +func (a Int64Array) Int16Array() Int16Array { return Int16Array{a.array} } +func (a Int64Array) Int32Array() Int32Array { return Int32Array{a.array} } +func (a Int64Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Int64Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Int64Array) Uint32Array() Uint32Array { return Uint32Array{a.array} } +func (a Int64Array) Uint64Array() Uint64Array { return Uint64Array{a.array} } +func (a Int64Array) UnsafeArray() Array { return Array{a.array} } + +type Float32Array struct{ array } + +func MakeFloat32Array(values []float32) Float32Array { + return Float32Array{makeArray(values)} +} + +func UnsafeFloat32Array(base unsafe.Pointer, length int, offset uintptr) Float32Array { + return Float32Array{unsafeArray(base, length, offset)} +} + +func (a Float32Array) Len() int { return int(a.len) } +func (a Float32Array) Index(i int) float32 { return *(*float32)(a.index(i)) } +func (a Float32Array) Slice(i, j int) Float32Array { return Float32Array{a.slice(i, j)} } +func (a Float32Array) Array() Array { return Array{a.array} } +func (a Float32Array) Uint32Array() Uint32Array { return Uint32Array{a.array} } +func (a Float32Array) UnsafeArray() Array { return Array{a.array} } + +type Float64Array struct{ array } + +func MakeFloat64Array(values []float64) Float64Array { + return Float64Array{makeArray(values)} +} + +func UnsafeFloat64Array(base unsafe.Pointer, length int, offset uintptr) Float64Array { + return Float64Array{unsafeArray(base, length, offset)} +} + +func (a Float64Array) Len() int { return int(a.len) } +func (a Float64Array) Index(i int) float64 { return *(*float64)(a.index(i)) } +func (a Float64Array) Slice(i, j int) Float64Array { return Float64Array{a.slice(i, j)} } +func (a Float64Array) Uint64Array() Uint64Array { return Uint64Array{a.array} } +func (a Float64Array) UnsafeArray() Array { return Array{a.array} } + +type Uint8Array struct{ array } + +func MakeUint8Array(values []uint8) Uint8Array { + return Uint8Array{makeArray(values)} +} + +func UnsafeUint8Array(base unsafe.Pointer, length int, offset uintptr) Uint8Array { + return Uint8Array{unsafeArray(base, length, offset)} +} + +func (a Uint8Array) Len() int { return int(a.len) } +func (a Uint8Array) Index(i int) uint8 { return *(*uint8)(a.index(i)) } +func (a Uint8Array) Slice(i, j int) Uint8Array { return Uint8Array{a.slice(i, j)} } +func (a Uint8Array) UnsafeArray() Array { return Array{a.array} } + +type Uint16Array struct{ array } + +func MakeUint16Array(values []uint16) Uint16Array { + return Uint16Array{makeArray(values)} +} + +func UnsafeUint16Array(base unsafe.Pointer, length int, offset uintptr) Uint16Array { + return Uint16Array{unsafeArray(base, length, offset)} +} + +func (a Uint16Array) Len() int { return int(a.len) } +func (a Uint16Array) Index(i int) uint16 { return *(*uint16)(a.index(i)) } +func (a Uint16Array) Slice(i, j int) Uint16Array { return Uint16Array{a.slice(i, j)} } +func (a Uint16Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Uint16Array) UnsafeArray() Array { return Array{a.array} } + +type Uint32Array struct{ array } + +func MakeUint32Array(values []uint32) Uint32Array { + return Uint32Array{makeArray(values)} +} + +func UnsafeUint32Array(base unsafe.Pointer, length int, offset uintptr) Uint32Array { + return Uint32Array{unsafeArray(base, length, offset)} +} + +func (a Uint32Array) Len() int { return int(a.len) } +func (a Uint32Array) Index(i int) uint32 { return *(*uint32)(a.index(i)) } +func (a Uint32Array) Slice(i, j int) Uint32Array { return Uint32Array{a.slice(i, j)} } +func (a Uint32Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Uint32Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Uint32Array) UnsafeArray() Array { return Array{a.array} } + +type Uint64Array struct{ array } + +func MakeUint64Array(values []uint64) Uint64Array { + return Uint64Array{makeArray(values)} +} + +func UnsafeUint64Array(base unsafe.Pointer, length int, offset uintptr) Uint64Array { + return Uint64Array{unsafeArray(base, length, offset)} +} + +func (a Uint64Array) Len() int { return int(a.len) } +func (a Uint64Array) Index(i int) uint64 { return *(*uint64)(a.index(i)) } +func (a Uint64Array) Slice(i, j int) Uint64Array { return Uint64Array{a.slice(i, j)} } +func (a Uint64Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Uint64Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Uint64Array) Uint32Array() Uint32Array { return Uint32Array{a.array} } +func (a Uint64Array) UnsafeArray() Array { return Array{a.array} } + +type Uint128Array struct{ array } + +func MakeUint128Array(values [][16]byte) Uint128Array { + return Uint128Array{makeArray(values)} +} + +func UnsafeUint128Array(base unsafe.Pointer, length int, offset uintptr) Uint128Array { + return Uint128Array{unsafeArray(base, length, offset)} +} + +func (a Uint128Array) Len() int { return int(a.len) } +func (a Uint128Array) Index(i int) [16]byte { return *(*[16]byte)(a.index(i)) } +func (a Uint128Array) Slice(i, j int) Uint128Array { return Uint128Array{a.slice(i, j)} } +func (a Uint128Array) Uint8Array() Uint8Array { return Uint8Array{a.array} } +func (a Uint128Array) Uint16Array() Uint16Array { return Uint16Array{a.array} } +func (a Uint128Array) Uint32Array() Uint32Array { return Uint32Array{a.array} } +func (a Uint128Array) Uint64Array() Uint64Array { return Uint64Array{a.array} } +func (a Uint128Array) UnsafeArray() Array { return Array{a.array} } + +type StringArray struct{ array } + +func MakeStringArray(values []string) StringArray { + const sizeOfString = unsafe.Sizeof("") + return StringArray{makeArray(values)} +} + +func UnsafeStringArray(base unsafe.Pointer, length int, offset uintptr) StringArray { + return StringArray{unsafeArray(base, length, offset)} +} + +func (a StringArray) Len() int { return int(a.len) } +func (a StringArray) Index(i int) string { return *(*string)(a.index(i)) } +func (a StringArray) Slice(i, j int) StringArray { return StringArray{a.slice(i, j)} } +func (a StringArray) UnsafeArray() Array { return Array{a.array} } + +type TimeArray struct{ array } + +func MakeTimeArray(values []time.Time) TimeArray { + return TimeArray{makeArray(values)} +} + +func UnsafeTimeArray(base unsafe.Pointer, length int, offset uintptr) TimeArray { + return TimeArray{unsafeArray(base, length, offset)} +} + +func (a TimeArray) Len() int { return int(a.len) } +func (a TimeArray) Index(i int) time.Time { return *(*time.Time)(a.index(i)) } +func (a TimeArray) Slice(i, j int) TimeArray { return TimeArray{a.slice(i, j)} } +func (a TimeArray) UnsafeArray() Array { return Array{a.array} } diff --git a/vendor/github.com/parquet-go/parquet-go/sparse/gather.go b/vendor/github.com/parquet-go/parquet-go/sparse/gather.go new file mode 100644 index 0000000000..d7d72d091b --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sparse/gather.go @@ -0,0 +1,37 @@ +package sparse + +import "github.com/parquet-go/parquet-go/internal/unsafecast" + +func GatherInt32(dst []int32, src Int32Array) int { + return GatherUint32(unsafecast.Slice[uint32](dst), src.Uint32Array()) +} + +func GatherInt64(dst []int64, src Int64Array) int { + return GatherUint64(unsafecast.Slice[uint64](dst), src.Uint64Array()) +} + +func GatherFloat32(dst []float32, src Float32Array) int { + return GatherUint32(unsafecast.Slice[uint32](dst), src.Uint32Array()) +} + +func GatherFloat64(dst []float64, src Float64Array) int { + return GatherUint64(unsafecast.Slice[uint64](dst), src.Uint64Array()) +} + +func GatherBits(dst []byte, src Uint8Array) int { return gatherBits(dst, src) } + +func GatherUint32(dst []uint32, src Uint32Array) int { return gather32(dst, src) } + +func GatherUint64(dst []uint64, src Uint64Array) int { return gather64(dst, src) } + +func GatherUint128(dst [][16]byte, src Uint128Array) int { return gather128(dst, src) } + +func GatherString(dst []string, src StringArray) int { + n := min(len(dst), src.Len()) + + for i := range dst[:n] { + dst[i] = src.Index(i) + } + + return n +} diff --git a/vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.go b/vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.go new file mode 100644 index 0000000000..427abbccbf --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.go @@ -0,0 +1,85 @@ +//go:build !purego + +package sparse + +import ( + "golang.org/x/sys/cpu" +) + +func gatherBits(dst []byte, src Uint8Array) int { + n := min(len(dst)*8, src.Len()) + i := 0 + + if n >= 8 { + i = (n / 8) * 8 + // Make sure `offset` is at least 4 bytes, otherwise VPGATHERDD may read + // data beyond the end of the program memory and trigger a fault. + // + // If the boolean values do not have enough padding we must fallback to + // the scalar algorithm to be able to load single bytes from memory. + if src.off >= 4 && cpu.X86.HasAVX2 { + gatherBitsAVX2(dst, src.Slice(0, i)) + } else { + gatherBitsDefault(dst, src.Slice(0, i)) + } + } + + for i < n { + x := i / 8 + y := i % 8 + b := src.Index(i) + dst[x] = ((b & 1) << y) | (dst[x] & ^(1 << y)) + i++ + } + + return n +} + +func gather32(dst []uint32, src Uint32Array) int { + n := min(len(dst), src.Len()) + i := 0 + + if n >= 16 && cpu.X86.HasAVX2 { + i = (n / 8) * 8 + gather32AVX2(dst[:i:i], src) + } + + for i < n { + dst[i] = src.Index(i) + i++ + } + + return n +} + +func gather64(dst []uint64, src Uint64Array) int { + n := min(len(dst), src.Len()) + i := 0 + + if n >= 8 && cpu.X86.HasAVX2 { + i = (n / 4) * 4 + gather64AVX2(dst[:i:i], src) + } + + for i < n { + dst[i] = src.Index(i) + i++ + } + + return n +} + +//go:noescape +func gatherBitsAVX2(dst []byte, src Uint8Array) + +//go:noescape +func gatherBitsDefault(dst []byte, src Uint8Array) + +//go:noescape +func gather32AVX2(dst []uint32, src Uint32Array) + +//go:noescape +func gather64AVX2(dst []uint64, src Uint64Array) + +//go:noescape +func gather128(dst [][16]byte, src Uint128Array) int diff --git a/vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.s b/vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.s new file mode 100644 index 0000000000..e87dee82af --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sparse/gather_amd64.s @@ -0,0 +1,193 @@ +//go:build !purego + +#include "textflag.h" + +// func gatherBitsAVX2(dst []byte, src Uint8Array) +TEXT ยทgatherBitsAVX2(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ src_array_ptr+24(FP), BX + MOVQ src_array_len+32(FP), CX + MOVQ src_array_off+40(FP), DX + XORQ SI, SI + SHRQ $3, CX + + VPBROADCASTD src_array_off+40(FP), Y0 + VPMULLD range0n7<>(SB), Y0, Y0 + VPCMPEQD Y1, Y1, Y1 + VPCMPEQD Y2, Y2, Y2 +loop: + VPGATHERDD Y1, (BX)(Y0*1), Y3 + VMOVDQU Y2, Y1 + VPSLLD $31, Y3, Y3 + VMOVMSKPS Y3, DI + + MOVB DI, (AX)(SI*1) + + LEAQ (BX)(DX*8), BX + INCQ SI + CMPQ SI, CX + JNE loop + VZEROUPPER + RET + +// func gatherBitsDefault(dst []byte, src Uint8Array) +TEXT ยทgatherBitsDefault(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ src_array_ptr+24(FP), BX + MOVQ src_array_len+32(FP), CX + MOVQ src_array_off+40(FP), DX + XORQ SI, SI + SHRQ $3, CX +loop: + LEAQ (BX)(DX*2), DI + MOVBQZX (BX), R8 + MOVBQZX (BX)(DX*1), R9 + MOVBQZX (DI), R10 + MOVBQZX (DI)(DX*1), R11 + LEAQ (BX)(DX*4), BX + LEAQ (DI)(DX*4), DI + MOVBQZX (BX), R12 + MOVBQZX (BX)(DX*1), R13 + MOVBQZX (DI), R14 + MOVBQZX (DI)(DX*1), R15 + LEAQ (BX)(DX*4), BX + + ANDQ $1, R8 + ANDQ $1, R9 + ANDQ $1, R10 + ANDQ $1, R11 + ANDQ $1, R12 + ANDQ $1, R13 + ANDQ $1, R14 + ANDQ $1, R15 + + SHLQ $1, R9 + SHLQ $2, R10 + SHLQ $3, R11 + SHLQ $4, R12 + SHLQ $5, R13 + SHLQ $6, R14 + SHLQ $7, R15 + + ORQ R9, R8 + ORQ R11, R10 + ORQ R13, R12 + ORQ R15, R14 + ORQ R10, R8 + ORQ R12, R8 + ORQ R14, R8 + + MOVB R8, (AX)(SI*1) + + INCQ SI + CMPQ SI, CX + JNE loop + RET + +// func gather32AVX2(dst []uint32, src Uint32Array) +TEXT ยทgather32AVX2(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), CX + MOVQ src_array_ptr+24(FP), BX + MOVQ src_array_off+40(FP), DX + XORQ SI, SI + + VPBROADCASTD src_array_off+40(FP), Y0 + VPMULLD range0n7<>(SB), Y0, Y0 + VPCMPEQD Y1, Y1, Y1 + VPCMPEQD Y2, Y2, Y2 +loop: + VPGATHERDD Y1, (BX)(Y0*1), Y3 + VMOVDQU Y3, (AX)(SI*4) + VMOVDQU Y2, Y1 + + LEAQ (BX)(DX*8), BX + ADDQ $8, SI + CMPQ SI, CX + JNE loop + VZEROUPPER + RET + +// func gather64AVX2(dst []uint64, src Uint64Array) +TEXT ยทgather64AVX2(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), CX + MOVQ src_array_ptr+24(FP), BX + MOVQ src_array_off+40(FP), DX + XORQ SI, SI + + VPBROADCASTQ src_array_off+40(FP), Y0 + VPMULLD range0n3<>(SB), Y0, Y0 + VPCMPEQQ Y1, Y1, Y1 + VPCMPEQQ Y2, Y2, Y2 +loop: + VPGATHERQQ Y1, (BX)(Y0*1), Y3 + VMOVDQU Y3, (AX)(SI*8) + VMOVDQU Y2, Y1 + + LEAQ (BX)(DX*4), BX + ADDQ $4, SI + CMPQ SI, CX + JNE loop + VZEROUPPER + RET + +// func gather128(dst [][16]byte, src Uint128Array) int +TEXT ยทgather128(SB), NOSPLIT, $0-56 + MOVQ dst_base+0(FP), AX + MOVQ dst_len+8(FP), CX + MOVQ src_array_ptr+24(FP), BX + MOVQ src_array_len+32(FP), DI + MOVQ src_array_off+40(FP), DX + XORQ SI, SI + + CMPQ DI, CX + CMOVQLT DI, CX + + CMPQ CX, $0 + JE done + + CMPQ CX, $1 + JE tail + + XORQ SI, SI + MOVQ CX, DI + SHRQ $1, DI + SHLQ $1, DI +loop: + MOVOU (BX), X0 + MOVOU (BX)(DX*1), X1 + + MOVOU X0, (AX) + MOVOU X1, 16(AX) + + LEAQ (BX)(DX*2), BX + ADDQ $32, AX + ADDQ $2, SI + CMPQ SI, DI + JNE loop + + CMPQ SI, CX + JE done +tail: + MOVOU (BX), X0 + MOVOU X0, (AX) +done: + MOVQ CX, ret+48(FP) + RET + +GLOBL range0n3<>(SB), RODATA|NOPTR, $32 +DATA range0n3<>+0(SB)/8, $0 +DATA range0n3<>+8(SB)/8, $1 +DATA range0n3<>+16(SB)/8, $2 +DATA range0n3<>+24(SB)/8, $3 + +GLOBL range0n7<>(SB), RODATA|NOPTR, $32 +DATA range0n7<>+0(SB)/4, $0 +DATA range0n7<>+4(SB)/4, $1 +DATA range0n7<>+8(SB)/4, $2 +DATA range0n7<>+12(SB)/4, $3 +DATA range0n7<>+16(SB)/4, $4 +DATA range0n7<>+20(SB)/4, $5 +DATA range0n7<>+24(SB)/4, $6 +DATA range0n7<>+28(SB)/4, $7 diff --git a/vendor/github.com/parquet-go/parquet-go/sparse/gather_purego.go b/vendor/github.com/parquet-go/parquet-go/sparse/gather_purego.go new file mode 100644 index 0000000000..2f25c4486f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sparse/gather_purego.go @@ -0,0 +1,72 @@ +//go:build purego || !amd64 + +package sparse + +func gatherBits(dst []byte, src Uint8Array) int { + n := min(len(dst)*8, src.Len()) + i := 0 + + if k := (n / 8) * 8; k > 0 { + for j := 0; i < k; j++ { + b0 := src.Index(i + 0) + b1 := src.Index(i + 1) + b2 := src.Index(i + 2) + b3 := src.Index(i + 3) + b4 := src.Index(i + 4) + b5 := src.Index(i + 5) + b6 := src.Index(i + 6) + b7 := src.Index(i + 7) + + dst[j] = (b0 & 1) | + ((b1 & 1) << 1) | + ((b2 & 1) << 2) | + ((b3 & 1) << 3) | + ((b4 & 1) << 4) | + ((b5 & 1) << 5) | + ((b6 & 1) << 6) | + ((b7 & 1) << 7) + + i += 8 + } + } + + for i < n { + x := i / 8 + y := i % 8 + b := src.Index(i) + dst[x] = ((b & 1) << y) | (dst[x] & ^(1 << y)) + i++ + } + + return n +} + +func gather32(dst []uint32, src Uint32Array) int { + n := min(len(dst), src.Len()) + + for i := range dst[:n] { + dst[i] = src.Index(i) + } + + return n +} + +func gather64(dst []uint64, src Uint64Array) int { + n := min(len(dst), src.Len()) + + for i := range dst[:n] { + dst[i] = src.Index(i) + } + + return n +} + +func gather128(dst [][16]byte, src Uint128Array) int { + n := min(len(dst), src.Len()) + + for i := range dst[:n] { + dst[i] = src.Index(i) + } + + return n +} diff --git a/vendor/github.com/parquet-go/parquet-go/sparse/sparse.go b/vendor/github.com/parquet-go/parquet-go/sparse/sparse.go new file mode 100644 index 0000000000..b2680bf554 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/sparse/sparse.go @@ -0,0 +1,20 @@ +// Package sparse contains abstractions to help work on arrays of values in +// sparse memory locations. +// +// Conversion between array types is supported when converting integers to a +// lower size (e.g. int32 to int16, or uint64 to uint8), or converting from +// signed integers to unsigned. Float types can also be converted to unsigned +// integers of the same size, in which case the conversion is similar to using +// the standard library's math.Float32bits and math.Float64bits functions. +// +// All array types can be converted to a generic Array type that can be used to erase +// type information and bypass type conversion rules. This conversion is similar +// to using Go's unsafe package to bypass Go's type system and should usually be +// avoided and a sign that the application is attempting to break type safety +// boundaries. +// +// The package provides Gather* functions which retrieve values from sparse +// arrays into contiguous memory buffers. On platforms that support it, these +// operations are implemented using SIMD gather instructions (e.g. VPGATHER on +// Intel CPUs). +package sparse diff --git a/vendor/github.com/parquet-go/parquet-go/transform.go b/vendor/github.com/parquet-go/parquet-go/transform.go new file mode 100644 index 0000000000..318e27c90a --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/transform.go @@ -0,0 +1,140 @@ +package parquet + +// TransformRowReader constructs a RowReader which applies the given transform +// to each row rad from reader. +// +// The transformation function appends the transformed src row to dst, returning +// dst and any error that occurred during the transformation. If dst is returned +// unchanged, the row is skipped. +func TransformRowReader(reader RowReader, transform func(dst, src Row) (Row, error)) RowReader { + return &transformRowReader{reader: reader, transform: transform} +} + +type transformRowReader struct { + reader RowReader + transform func(Row, Row) (Row, error) + rows []Row + offset int + length int +} + +func (t *transformRowReader) ReadRows(rows []Row) (n int, err error) { + if len(t.rows) == 0 { + t.rows = makeRows(len(rows)) + } + + for { + for n < len(rows) && t.offset < t.length { + dst := rows[n][:0] + src := t.rows[t.offset] + rows[n], err = t.transform(dst, src) + if err != nil { + return n, err + } + clearValues(src) + t.rows[t.offset] = src[:0] + t.offset++ + n++ + } + + if n == len(rows) { + return n, nil + } + + r, err := t.reader.ReadRows(t.rows) + if r == 0 && err != nil { + return n, err + } + t.offset = 0 + t.length = r + } +} + +type transformRowBuffer struct { + buffer []Row + offset int32 + length int32 +} + +func (b *transformRowBuffer) init(n int) { + b.buffer = makeRows(n) + b.offset = 0 + b.length = 0 +} + +func (b *transformRowBuffer) discard() { + row := b.buffer[b.offset] + clearValues(row) + b.buffer[b.offset] = row[:0] + + if b.offset++; b.offset == b.length { + b.reset(0) + } +} + +func (b *transformRowBuffer) reset(n int) { + b.offset = 0 + b.length = int32(n) +} + +func (b *transformRowBuffer) rows() []Row { + return b.buffer[b.offset:b.length] +} + +func (b *transformRowBuffer) cap() int { + return len(b.buffer) +} + +func (b *transformRowBuffer) len() int { + return int(b.length - b.offset) +} + +// TransformRowWriter constructs a RowWriter which applies the given transform +// to each row writter to writer. +// +// The transformation function appends the transformed src row to dst, returning +// dst and any error that occurred during the transformation. If dst is returned +// unchanged, the row is skipped. +func TransformRowWriter(writer RowWriter, transform func(dst, src Row) (Row, error)) RowWriter { + return &transformRowWriter{writer: writer, transform: transform} +} + +type transformRowWriter struct { + writer RowWriter + transform func(Row, Row) (Row, error) + rows []Row +} + +func (t *transformRowWriter) WriteRows(rows []Row) (n int, err error) { + if len(t.rows) == 0 { + t.rows = makeRows(len(rows)) + } + + for n < len(rows) { + numRows := min(len(rows)-n, len(t.rows)) + if err := t.writeRows(rows[n : n+numRows]); err != nil { + return n, err + } + n += numRows + } + + return n, nil +} + +func (t *transformRowWriter) writeRows(rows []Row) (err error) { + numRows := 0 + defer func() { clearRows(t.rows[:numRows]) }() + + for _, row := range rows { + t.rows[numRows], err = t.transform(t.rows[numRows][:0], row) + if err != nil { + return err + } + if len(t.rows[numRows]) != 0 { + numRows++ + } + } + + _, err = t.writer.WriteRows(t.rows[:numRows]) + return err +} diff --git a/vendor/github.com/parquet-go/parquet-go/type.go b/vendor/github.com/parquet-go/parquet-go/type.go new file mode 100644 index 0000000000..c425c0d187 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/type.go @@ -0,0 +1,2436 @@ +package parquet + +import ( + "bytes" + "encoding/json" + "fmt" + "math/bits" + "reflect" + "time" + "unsafe" + + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/format" +) + +// Kind is an enumeration type representing the physical types supported by the +// parquet type system. +type Kind int8 + +const ( + Boolean Kind = Kind(format.Boolean) + Int32 Kind = Kind(format.Int32) + Int64 Kind = Kind(format.Int64) + Int96 Kind = Kind(format.Int96) + Float Kind = Kind(format.Float) + Double Kind = Kind(format.Double) + ByteArray Kind = Kind(format.ByteArray) + FixedLenByteArray Kind = Kind(format.FixedLenByteArray) +) + +// String returns a human-readable representation of the physical type. +func (k Kind) String() string { return format.Type(k).String() } + +// Value constructs a value from k and v. +// +// The method panics if the data is not a valid representation of the value +// kind; for example, if the kind is Int32 but the data is not 4 bytes long. +func (k Kind) Value(v []byte) Value { + x, err := parseValue(k, v) + if err != nil { + panic(err) + } + return x +} + +// The Type interface represents logical types of the parquet type system. +// +// Types are immutable and therefore safe to access from multiple goroutines. +type Type interface { + // Returns a human-readable representation of the parquet type. + String() string + + // Returns the Kind value representing the underlying physical type. + // + // The method panics if it is called on a group type. + Kind() Kind + + // For integer and floating point physical types, the method returns the + // size of values in bits. + // + // For fixed-length byte arrays, the method returns the size of elements + // in bytes. + // + // For other types, the value is zero. + Length() int + + // Returns an estimation of the number of bytes required to hold the given + // number of values of this type in memory. + // + // The method returns zero for group types. + EstimateSize(numValues int) int + + // Returns an estimation of the number of values of this type that can be + // held in the given byte size. + // + // The method returns zero for group types. + EstimateNumValues(size int) int + + // Compares two values and returns a negative integer if a < b, positive if + // a > b, or zero if a == b. + // + // The values' Kind must match the type, otherwise the result is undefined. + // + // The method panics if it is called on a group type. + Compare(a, b Value) int + + // ColumnOrder returns the type's column order. For group types, this method + // returns nil. + // + // The order describes the comparison logic implemented by the Less method. + // + // As an optimization, the method may return the same pointer across + // multiple calls. Applications must treat the returned value as immutable, + // mutating the value will result in undefined behavior. + ColumnOrder() *format.ColumnOrder + + // Returns the physical type as a *format.Type value. For group types, this + // method returns nil. + // + // As an optimization, the method may return the same pointer across + // multiple calls. Applications must treat the returned value as immutable, + // mutating the value will result in undefined behavior. + PhysicalType() *format.Type + + // Returns the logical type as a *format.LogicalType value. When the logical + // type is unknown, the method returns nil. + // + // As an optimization, the method may return the same pointer across + // multiple calls. Applications must treat the returned value as immutable, + // mutating the value will result in undefined behavior. + LogicalType() *format.LogicalType + + // Returns the logical type's equivalent converted type. When there are + // no equivalent converted type, the method returns nil. + // + // As an optimization, the method may return the same pointer across + // multiple calls. Applications must treat the returned value as immutable, + // mutating the value will result in undefined behavior. + ConvertedType() *deprecated.ConvertedType + + // Creates a column indexer for values of this type. + // + // The size limit is a hint to the column indexer that it is allowed to + // truncate the page boundaries to the given size. Only BYTE_ARRAY and + // FIXED_LEN_BYTE_ARRAY types currently take this value into account. + // + // A value of zero or less means no limits. + // + // The method panics if it is called on a group type. + NewColumnIndexer(sizeLimit int) ColumnIndexer + + // Creates a row group buffer column for values of this type. + // + // Column buffers are created using the index of the column they are + // accumulating values in memory for (relative to the parent schema), + // and the size of their memory buffer. + // + // The application may give an estimate of the number of values it expects + // to write to the buffer as second argument. This estimate helps set the + // initialize buffer capacity but is not a hard limit, the underlying memory + // buffer will grown as needed to allow more values to be written. Programs + // may use the Size method of the column buffer (or the parent row group, + // when relevant) to determine how many bytes are being used, and perform a + // flush of the buffers to a storage layer. + // + // The method panics if it is called on a group type. + NewColumnBuffer(columnIndex, numValues int) ColumnBuffer + + // Creates a dictionary holding values of this type. + // + // The dictionary retains the data buffer, it does not make a copy of it. + // If the application needs to share ownership of the memory buffer, it must + // ensure that it will not be modified while the page is in use, or it must + // make a copy of it prior to creating the dictionary. + // + // The method panics if the data type does not correspond to the parquet + // type it is called on. + NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary + + // Creates a page belonging to a column at the given index, backed by the + // data buffer. + // + // The page retains the data buffer, it does not make a copy of it. If the + // application needs to share ownership of the memory buffer, it must ensure + // that it will not be modified while the page is in use, or it must make a + // copy of it prior to creating the page. + // + // The method panics if the data type does not correspond to the parquet + // type it is called on. + NewPage(columnIndex, numValues int, data encoding.Values) Page + + // Creates an encoding.Values instance backed by the given buffers. + // + // The offsets is only used by BYTE_ARRAY types, where it represents the + // positions of each variable length value in the values buffer. + // + // The following expression creates an empty instance for any type: + // + // values := typ.NewValues(nil, nil) + // + // The method panics if it is called on group types. + NewValues(values []byte, offsets []uint32) encoding.Values + + // Assuming the src buffer contains PLAIN encoded values of the type it is + // called on, applies the given encoding and produces the output to the dst + // buffer passed as first argument by dispatching the call to one of the + // encoding methods. + Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) + + // Assuming the src buffer contains values encoding in the given encoding, + // decodes the input and produces the encoded values into the dst output + // buffer passed as first argument by dispatching the call to one of the + // encoding methods. + Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) + + // Returns an estimation of the output size after decoding the values passed + // as first argument with the given encoding. + // + // For most types, this is similar to calling EstimateSize with the known + // number of encoded values. For variable size types, using this method may + // provide a more precise result since it can inspect the input buffer. + EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int + + // Assigns a Parquet value to a Go value. Returns an error if assignment is + // not possible. The source Value must be an expected logical type for the + // receiver. This can be accomplished using ConvertValue. + AssignValue(dst reflect.Value, src Value) error + + // Convert a Parquet Value of the given Type into a Parquet Value that is + // compatible with the receiver. The returned Value is suitable to be passed + // to AssignValue. + ConvertValue(val Value, typ Type) (Value, error) +} + +var ( + BooleanType Type = booleanType{} + Int32Type Type = int32Type{} + Int64Type Type = int64Type{} + Int96Type Type = int96Type{} + FloatType Type = floatType{} + DoubleType Type = doubleType{} + ByteArrayType Type = byteArrayType{} +) + +// In the current parquet version supported by this library, only type-defined +// orders are supported. +var typeDefinedColumnOrder = format.ColumnOrder{ + TypeOrder: new(format.TypeDefinedOrder), +} + +var physicalTypes = [...]format.Type{ + 0: format.Boolean, + 1: format.Int32, + 2: format.Int64, + 3: format.Int96, + 4: format.Float, + 5: format.Double, + 6: format.ByteArray, + 7: format.FixedLenByteArray, +} + +var convertedTypes = [...]deprecated.ConvertedType{ + 0: deprecated.UTF8, + 1: deprecated.Map, + 2: deprecated.MapKeyValue, + 3: deprecated.List, + 4: deprecated.Enum, + 5: deprecated.Decimal, + 6: deprecated.Date, + 7: deprecated.TimeMillis, + 8: deprecated.TimeMicros, + 9: deprecated.TimestampMillis, + 10: deprecated.TimestampMicros, + 11: deprecated.Uint8, + 12: deprecated.Uint16, + 13: deprecated.Uint32, + 14: deprecated.Uint64, + 15: deprecated.Int8, + 16: deprecated.Int16, + 17: deprecated.Int32, + 18: deprecated.Int64, + 19: deprecated.Json, + 20: deprecated.Bson, + 21: deprecated.Interval, +} + +type booleanType struct{} + +func (t booleanType) String() string { return "BOOLEAN" } +func (t booleanType) Kind() Kind { return Boolean } +func (t booleanType) Length() int { return 1 } +func (t booleanType) EstimateSize(n int) int { return (n + 7) / 8 } +func (t booleanType) EstimateNumValues(n int) int { return 8 * n } +func (t booleanType) Compare(a, b Value) int { return compareBool(a.boolean(), b.boolean()) } +func (t booleanType) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t booleanType) LogicalType() *format.LogicalType { return nil } +func (t booleanType) ConvertedType() *deprecated.ConvertedType { return nil } +func (t booleanType) PhysicalType() *format.Type { return &physicalTypes[Boolean] } + +func (t booleanType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newBooleanColumnIndexer() +} + +func (t booleanType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newBooleanColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t booleanType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newBooleanDictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t booleanType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newBooleanPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t booleanType) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.BooleanValues(values) +} + +func (t booleanType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeBoolean(dst, src, enc) +} + +func (t booleanType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeBoolean(dst, src, enc) +} + +func (t booleanType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t booleanType) AssignValue(dst reflect.Value, src Value) error { + v := src.boolean() + switch dst.Kind() { + case reflect.Bool: + dst.SetBool(v) + default: + dst.Set(reflect.ValueOf(v)) + } + return nil +} + +func (t booleanType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToBoolean(val) + } + switch typ.Kind() { + case Boolean: + return val, nil + case Int32: + return convertInt32ToBoolean(val) + case Int64: + return convertInt64ToBoolean(val) + case Int96: + return convertInt96ToBoolean(val) + case Float: + return convertFloatToBoolean(val) + case Double: + return convertDoubleToBoolean(val) + case ByteArray, FixedLenByteArray: + return convertByteArrayToBoolean(val) + default: + return makeValueKind(Boolean), nil + } +} + +type int32Type struct{} + +func (t int32Type) String() string { return "INT32" } +func (t int32Type) Kind() Kind { return Int32 } +func (t int32Type) Length() int { return 32 } +func (t int32Type) EstimateSize(n int) int { return 4 * n } +func (t int32Type) EstimateNumValues(n int) int { return n / 4 } +func (t int32Type) Compare(a, b Value) int { return compareInt32(a.int32(), b.int32()) } +func (t int32Type) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t int32Type) LogicalType() *format.LogicalType { return nil } +func (t int32Type) ConvertedType() *deprecated.ConvertedType { return nil } +func (t int32Type) PhysicalType() *format.Type { return &physicalTypes[Int32] } + +func (t int32Type) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newInt32ColumnIndexer() +} + +func (t int32Type) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newInt32ColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t int32Type) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newInt32Dictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t int32Type) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newInt32Page(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t int32Type) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.Int32ValuesFromBytes(values) +} + +func (t int32Type) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeInt32(dst, src, enc) +} + +func (t int32Type) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeInt32(dst, src, enc) +} + +func (t int32Type) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t int32Type) AssignValue(dst reflect.Value, src Value) error { + v := src.int32() + switch dst.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32: + dst.SetInt(int64(v)) + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + dst.SetUint(uint64(v)) + default: + dst.Set(reflect.ValueOf(v)) + } + return nil +} + +func (t int32Type) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToInt32(val) + } + switch typ.Kind() { + case Boolean: + return convertBooleanToInt32(val) + case Int32: + return val, nil + case Int64: + return convertInt64ToInt32(val) + case Int96: + return convertInt96ToInt32(val) + case Float: + return convertFloatToInt32(val) + case Double: + return convertDoubleToInt32(val) + case ByteArray, FixedLenByteArray: + return convertByteArrayToInt32(val) + default: + return makeValueKind(Int32), nil + } +} + +type int64Type struct{} + +func (t int64Type) String() string { return "INT64" } +func (t int64Type) Kind() Kind { return Int64 } +func (t int64Type) Length() int { return 64 } +func (t int64Type) EstimateSize(n int) int { return 8 * n } +func (t int64Type) EstimateNumValues(n int) int { return n / 8 } +func (t int64Type) Compare(a, b Value) int { return compareInt64(a.int64(), b.int64()) } +func (t int64Type) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t int64Type) LogicalType() *format.LogicalType { return nil } +func (t int64Type) ConvertedType() *deprecated.ConvertedType { return nil } +func (t int64Type) PhysicalType() *format.Type { return &physicalTypes[Int64] } + +func (t int64Type) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newInt64ColumnIndexer() +} + +func (t int64Type) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newInt64ColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t int64Type) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newInt64Dictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t int64Type) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newInt64Page(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t int64Type) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.Int64ValuesFromBytes(values) +} + +func (t int64Type) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeInt64(dst, src, enc) +} + +func (t int64Type) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeInt64(dst, src, enc) +} + +func (t int64Type) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t int64Type) AssignValue(dst reflect.Value, src Value) error { + v := src.int64() + switch dst.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + dst.SetInt(v) + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr: + dst.SetUint(uint64(v)) + default: + dst.Set(reflect.ValueOf(v)) + } + return nil +} + +func (t int64Type) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToInt64(val) + } + switch typ.Kind() { + case Boolean: + return convertBooleanToInt64(val) + case Int32: + return convertInt32ToInt64(val) + case Int64: + return val, nil + case Int96: + return convertInt96ToInt64(val) + case Float: + return convertFloatToInt64(val) + case Double: + return convertDoubleToInt64(val) + case ByteArray, FixedLenByteArray: + return convertByteArrayToInt64(val) + default: + return makeValueKind(Int64), nil + } +} + +type int96Type struct{} + +func (t int96Type) String() string { return "INT96" } + +func (t int96Type) Kind() Kind { return Int96 } +func (t int96Type) Length() int { return 96 } +func (t int96Type) EstimateSize(n int) int { return 12 * n } +func (t int96Type) EstimateNumValues(n int) int { return n / 12 } +func (t int96Type) Compare(a, b Value) int { return compareInt96(a.int96(), b.int96()) } +func (t int96Type) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t int96Type) LogicalType() *format.LogicalType { return nil } +func (t int96Type) ConvertedType() *deprecated.ConvertedType { return nil } +func (t int96Type) PhysicalType() *format.Type { return &physicalTypes[Int96] } + +func (t int96Type) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newInt96ColumnIndexer() +} + +func (t int96Type) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newInt96ColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t int96Type) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newInt96Dictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t int96Type) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newInt96Page(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t int96Type) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.Int96ValuesFromBytes(values) +} + +func (t int96Type) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeInt96(dst, src, enc) +} + +func (t int96Type) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeInt96(dst, src, enc) +} + +func (t int96Type) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t int96Type) AssignValue(dst reflect.Value, src Value) error { + v := src.Int96() + dst.Set(reflect.ValueOf(v)) + return nil +} + +func (t int96Type) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToInt96(val) + } + switch typ.Kind() { + case Boolean: + return convertBooleanToInt96(val) + case Int32: + return convertInt32ToInt96(val) + case Int64: + return convertInt64ToInt96(val) + case Int96: + return val, nil + case Float: + return convertFloatToInt96(val) + case Double: + return convertDoubleToInt96(val) + case ByteArray, FixedLenByteArray: + return convertByteArrayToInt96(val) + default: + return makeValueKind(Int96), nil + } +} + +type floatType struct{} + +func (t floatType) String() string { return "FLOAT" } +func (t floatType) Kind() Kind { return Float } +func (t floatType) Length() int { return 32 } +func (t floatType) EstimateSize(n int) int { return 4 * n } +func (t floatType) EstimateNumValues(n int) int { return n / 4 } +func (t floatType) Compare(a, b Value) int { return compareFloat32(a.float(), b.float()) } +func (t floatType) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t floatType) LogicalType() *format.LogicalType { return nil } +func (t floatType) ConvertedType() *deprecated.ConvertedType { return nil } +func (t floatType) PhysicalType() *format.Type { return &physicalTypes[Float] } + +func (t floatType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newFloatColumnIndexer() +} + +func (t floatType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newFloatColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t floatType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newFloatDictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t floatType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newFloatPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t floatType) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.FloatValuesFromBytes(values) +} + +func (t floatType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeFloat(dst, src, enc) +} + +func (t floatType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeFloat(dst, src, enc) +} + +func (t floatType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t floatType) AssignValue(dst reflect.Value, src Value) error { + v := src.float() + switch dst.Kind() { + case reflect.Float32, reflect.Float64: + dst.SetFloat(float64(v)) + default: + dst.Set(reflect.ValueOf(v)) + } + return nil +} + +func (t floatType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToFloat(val) + } + switch typ.Kind() { + case Boolean: + return convertBooleanToFloat(val) + case Int32: + return convertInt32ToFloat(val) + case Int64: + return convertInt64ToFloat(val) + case Int96: + return convertInt96ToFloat(val) + case Float: + return val, nil + case Double: + return convertDoubleToFloat(val) + case ByteArray, FixedLenByteArray: + return convertByteArrayToFloat(val) + default: + return makeValueKind(Float), nil + } +} + +type doubleType struct{} + +func (t doubleType) String() string { return "DOUBLE" } +func (t doubleType) Kind() Kind { return Double } +func (t doubleType) Length() int { return 64 } +func (t doubleType) EstimateSize(n int) int { return 8 * n } +func (t doubleType) EstimateNumValues(n int) int { return n / 8 } +func (t doubleType) Compare(a, b Value) int { return compareFloat64(a.double(), b.double()) } +func (t doubleType) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t doubleType) LogicalType() *format.LogicalType { return nil } +func (t doubleType) ConvertedType() *deprecated.ConvertedType { return nil } +func (t doubleType) PhysicalType() *format.Type { return &physicalTypes[Double] } + +func (t doubleType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newDoubleColumnIndexer() +} + +func (t doubleType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newDoubleColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t doubleType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newDoubleDictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t doubleType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newDoublePage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t doubleType) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.DoubleValuesFromBytes(values) +} + +func (t doubleType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeDouble(dst, src, enc) +} + +func (t doubleType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeDouble(dst, src, enc) +} + +func (t doubleType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t doubleType) AssignValue(dst reflect.Value, src Value) error { + v := src.double() + switch dst.Kind() { + case reflect.Float32, reflect.Float64: + dst.SetFloat(v) + default: + dst.Set(reflect.ValueOf(v)) + } + return nil +} + +func (t doubleType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToDouble(val) + } + switch typ.Kind() { + case Boolean: + return convertBooleanToDouble(val) + case Int32: + return convertInt32ToDouble(val) + case Int64: + return convertInt64ToDouble(val) + case Int96: + return convertInt96ToDouble(val) + case Float: + return convertFloatToDouble(val) + case Double: + return val, nil + case ByteArray, FixedLenByteArray: + return convertByteArrayToDouble(val) + default: + return makeValueKind(Double), nil + } +} + +type byteArrayType struct{} + +func (t byteArrayType) String() string { return "BYTE_ARRAY" } +func (t byteArrayType) Kind() Kind { return ByteArray } +func (t byteArrayType) Length() int { return 0 } +func (t byteArrayType) EstimateSize(n int) int { return estimatedSizeOfByteArrayValues * n } +func (t byteArrayType) EstimateNumValues(n int) int { return n / estimatedSizeOfByteArrayValues } +func (t byteArrayType) Compare(a, b Value) int { return bytes.Compare(a.byteArray(), b.byteArray()) } +func (t byteArrayType) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } +func (t byteArrayType) LogicalType() *format.LogicalType { return nil } +func (t byteArrayType) ConvertedType() *deprecated.ConvertedType { return nil } +func (t byteArrayType) PhysicalType() *format.Type { return &physicalTypes[ByteArray] } + +func (t byteArrayType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newByteArrayColumnIndexer(sizeLimit) +} + +func (t byteArrayType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newByteArrayColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t byteArrayType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newByteArrayDictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t byteArrayType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newByteArrayPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t byteArrayType) NewValues(values []byte, offsets []uint32) encoding.Values { + return encoding.ByteArrayValues(values, offsets) +} + +func (t byteArrayType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeByteArray(dst, src, enc) +} + +func (t byteArrayType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeByteArray(dst, src, enc) +} + +func (t byteArrayType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return enc.EstimateDecodeByteArraySize(src) +} + +func (t byteArrayType) AssignValue(dst reflect.Value, src Value) error { + v := src.byteArray() + switch dst.Kind() { + case reflect.String: + dst.SetString(string(v)) + case reflect.Slice: + dst.SetBytes(copyBytes(v)) + default: + val := reflect.ValueOf(string(v)) + dst.Set(val) + } + return nil +} + +func (t byteArrayType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.Kind() { + case Boolean: + return convertBooleanToByteArray(val) + case Int32: + return convertInt32ToByteArray(val) + case Int64: + return convertInt64ToByteArray(val) + case Int96: + return convertInt96ToByteArray(val) + case Float: + return convertFloatToByteArray(val) + case Double: + return convertDoubleToByteArray(val) + case ByteArray, FixedLenByteArray: + return val, nil + default: + return makeValueKind(ByteArray), nil + } +} + +type fixedLenByteArrayType struct{ length int } + +func (t fixedLenByteArrayType) String() string { + return fmt.Sprintf("FIXED_LEN_BYTE_ARRAY(%d)", t.length) +} + +func (t fixedLenByteArrayType) Kind() Kind { return FixedLenByteArray } + +func (t fixedLenByteArrayType) Length() int { return t.length } + +func (t fixedLenByteArrayType) EstimateSize(n int) int { return t.length * n } + +func (t fixedLenByteArrayType) EstimateNumValues(n int) int { return n / t.length } + +func (t fixedLenByteArrayType) Compare(a, b Value) int { + return bytes.Compare(a.byteArray(), b.byteArray()) +} + +func (t fixedLenByteArrayType) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } + +func (t fixedLenByteArrayType) LogicalType() *format.LogicalType { return nil } + +func (t fixedLenByteArrayType) ConvertedType() *deprecated.ConvertedType { return nil } + +func (t fixedLenByteArrayType) PhysicalType() *format.Type { return &physicalTypes[FixedLenByteArray] } + +func (t fixedLenByteArrayType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newFixedLenByteArrayColumnIndexer(t.length, sizeLimit) +} + +func (t fixedLenByteArrayType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newFixedLenByteArrayColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t fixedLenByteArrayType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newFixedLenByteArrayDictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t fixedLenByteArrayType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newFixedLenByteArrayPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t fixedLenByteArrayType) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.FixedLenByteArrayValues(values, t.length) +} + +func (t fixedLenByteArrayType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeFixedLenByteArray(dst, src, enc) +} + +func (t fixedLenByteArrayType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeFixedLenByteArray(dst, src, enc) +} + +func (t fixedLenByteArrayType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t fixedLenByteArrayType) AssignValue(dst reflect.Value, src Value) error { + v := src.byteArray() + switch dst.Kind() { + case reflect.Array: + if dst.Type().Elem().Kind() == reflect.Uint8 && dst.Len() == len(v) { + // This code could be implemented as a call to reflect.Copy but + // it would require creating a reflect.Value from v which causes + // the heap allocation to pack the []byte value. To avoid this + // overhead we instead convert the reflect.Value holding the + // destination array into a byte slice which allows us to use + // a more efficient call to copy. + d := unsafe.Slice((*byte)(reflectValueData(dst)), len(v)) + copy(d, v) + return nil + } + case reflect.Slice: + dst.SetBytes(copyBytes(v)) + return nil + } + + val := reflect.ValueOf(copyBytes(v)) + dst.Set(val) + return nil +} + +func reflectValueData(v reflect.Value) unsafe.Pointer { + return (*[2]unsafe.Pointer)(unsafe.Pointer(&v))[1] +} + +func (t fixedLenByteArrayType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *stringType: + return convertStringToFixedLenByteArray(val, t.length) + } + switch typ.Kind() { + case Boolean: + return convertBooleanToFixedLenByteArray(val, t.length) + case Int32: + return convertInt32ToFixedLenByteArray(val, t.length) + case Int64: + return convertInt64ToFixedLenByteArray(val, t.length) + case Int96: + return convertInt96ToFixedLenByteArray(val, t.length) + case Float: + return convertFloatToFixedLenByteArray(val, t.length) + case Double: + return convertDoubleToFixedLenByteArray(val, t.length) + case ByteArray, FixedLenByteArray: + return convertByteArrayToFixedLenByteArray(val, t.length) + default: + return makeValueBytes(FixedLenByteArray, make([]byte, t.length)), nil + } +} + +type uint32Type struct{ int32Type } + +func (t uint32Type) Compare(a, b Value) int { + return compareUint32(a.uint32(), b.uint32()) +} + +func (t uint32Type) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newUint32ColumnIndexer() +} + +func (t uint32Type) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newUint32ColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t uint32Type) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newUint32Dictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t uint32Type) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newUint32Page(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +type uint64Type struct{ int64Type } + +func (t uint64Type) Compare(a, b Value) int { + return compareUint64(a.uint64(), b.uint64()) +} + +func (t uint64Type) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newUint64ColumnIndexer() +} + +func (t uint64Type) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newUint64ColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t uint64Type) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newUint64Dictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t uint64Type) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newUint64Page(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +// BE128 stands for "big-endian 128 bits". This type is used as a special case +// for fixed-length byte arrays of 16 bytes, which are commonly used to +// represent columns of random unique identifiers such as UUIDs. +// +// Comparisons of BE128 values use the natural byte order, the zeroth byte is +// the most significant byte. +// +// The special case is intended to provide optimizations based on the knowledge +// that the values are 16 bytes long. Stronger type checking can also be applied +// by the compiler when using [16]byte values rather than []byte, reducing the +// risk of errors on these common code paths. +type be128Type struct{} + +func (t be128Type) String() string { return "FIXED_LEN_BYTE_ARRAY(16)" } + +func (t be128Type) Kind() Kind { return FixedLenByteArray } + +func (t be128Type) Length() int { return 16 } + +func (t be128Type) EstimateSize(n int) int { return 16 * n } + +func (t be128Type) EstimateNumValues(n int) int { return n / 16 } + +func (t be128Type) Compare(a, b Value) int { return compareBE128(a.be128(), b.be128()) } + +func (t be128Type) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } + +func (t be128Type) LogicalType() *format.LogicalType { return nil } + +func (t be128Type) ConvertedType() *deprecated.ConvertedType { return nil } + +func (t be128Type) PhysicalType() *format.Type { return &physicalTypes[FixedLenByteArray] } + +func (t be128Type) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newBE128ColumnIndexer() +} + +func (t be128Type) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newBE128ColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t be128Type) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newBE128Dictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t be128Type) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newBE128Page(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t be128Type) NewValues(values []byte, _ []uint32) encoding.Values { + return encoding.FixedLenByteArrayValues(values, 16) +} + +func (t be128Type) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeFixedLenByteArray(dst, src, enc) +} + +func (t be128Type) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeFixedLenByteArray(dst, src, enc) +} + +func (t be128Type) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.EstimateSize(numValues) +} + +func (t be128Type) AssignValue(dst reflect.Value, src Value) error { + return fixedLenByteArrayType{length: 16}.AssignValue(dst, src) +} + +func (t be128Type) ConvertValue(val Value, typ Type) (Value, error) { + return fixedLenByteArrayType{length: 16}.ConvertValue(val, typ) +} + +// FixedLenByteArrayType constructs a type for fixed-length values of the given +// size (in bytes). +func FixedLenByteArrayType(length int) Type { + switch length { + case 16: + return be128Type{} + default: + return fixedLenByteArrayType{length: length} + } +} + +// Int constructs a leaf node of signed integer logical type of the given bit +// width. +// +// The bit width must be one of 8, 16, 32, 64, or the function will panic. +func Int(bitWidth int) Node { + return Leaf(integerType(bitWidth, &signedIntTypes)) +} + +// Uint constructs a leaf node of unsigned integer logical type of the given +// bit width. +// +// The bit width must be one of 8, 16, 32, 64, or the function will panic. +func Uint(bitWidth int) Node { + return Leaf(integerType(bitWidth, &unsignedIntTypes)) +} + +func integerType(bitWidth int, types *[4]intType) *intType { + switch bitWidth { + case 8: + return &types[0] + case 16: + return &types[1] + case 32: + return &types[2] + case 64: + return &types[3] + default: + panic(fmt.Sprintf("cannot create a %d bits parquet integer node", bitWidth)) + } +} + +var signedIntTypes = [...]intType{ + {BitWidth: 8, IsSigned: true}, + {BitWidth: 16, IsSigned: true}, + {BitWidth: 32, IsSigned: true}, + {BitWidth: 64, IsSigned: true}, +} + +var unsignedIntTypes = [...]intType{ + {BitWidth: 8, IsSigned: false}, + {BitWidth: 16, IsSigned: false}, + {BitWidth: 32, IsSigned: false}, + {BitWidth: 64, IsSigned: false}, +} + +type intType format.IntType + +func (t *intType) baseType() Type { + if t.IsSigned { + if t.BitWidth == 64 { + return int64Type{} + } else { + return int32Type{} + } + } else { + if t.BitWidth == 64 { + return uint64Type{} + } else { + return uint32Type{} + } + } +} + +func (t *intType) String() string { return (*format.IntType)(t).String() } + +func (t *intType) Kind() Kind { return t.baseType().Kind() } + +func (t *intType) Length() int { return int(t.BitWidth) } + +func (t *intType) EstimateSize(n int) int { return (int(t.BitWidth) / 8) * n } + +func (t *intType) EstimateNumValues(n int) int { return n / (int(t.BitWidth) / 8) } + +func (t *intType) Compare(a, b Value) int { + // This code is similar to t.baseType().Compare(a,b) but comparison methods + // tend to be invoked a lot (e.g. when sorting) so avoiding the interface + // indirection in this case yields much better throughput in some cases. + if t.BitWidth == 64 { + i1 := a.int64() + i2 := b.int64() + if t.IsSigned { + return compareInt64(i1, i2) + } else { + return compareUint64(uint64(i1), uint64(i2)) + } + } else { + i1 := a.int32() + i2 := b.int32() + if t.IsSigned { + return compareInt32(i1, i2) + } else { + return compareUint32(uint32(i1), uint32(i2)) + } + } +} + +func (t *intType) ColumnOrder() *format.ColumnOrder { return t.baseType().ColumnOrder() } + +func (t *intType) PhysicalType() *format.Type { return t.baseType().PhysicalType() } + +func (t *intType) LogicalType() *format.LogicalType { + return &format.LogicalType{Integer: (*format.IntType)(t)} +} + +func (t *intType) ConvertedType() *deprecated.ConvertedType { + convertedType := bits.Len8(uint8(t.BitWidth)/8) - 1 // 8=>0, 16=>1, 32=>2, 64=>4 + if t.IsSigned { + convertedType += int(deprecated.Int8) + } else { + convertedType += int(deprecated.Uint8) + } + return &convertedTypes[convertedType] +} + +func (t *intType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return t.baseType().NewColumnIndexer(sizeLimit) +} + +func (t *intType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return t.baseType().NewColumnBuffer(columnIndex, numValues) +} + +func (t *intType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return t.baseType().NewDictionary(columnIndex, numValues, data) +} + +func (t *intType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return t.baseType().NewPage(columnIndex, numValues, data) +} + +func (t *intType) NewValues(values []byte, offsets []uint32) encoding.Values { + return t.baseType().NewValues(values, offsets) +} + +func (t *intType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return t.baseType().Encode(dst, src, enc) +} + +func (t *intType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return t.baseType().Decode(dst, src, enc) +} + +func (t *intType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.baseType().EstimateDecodeSize(numValues, src, enc) +} + +func (t *intType) AssignValue(dst reflect.Value, src Value) error { + if t.BitWidth == 64 { + return int64Type{}.AssignValue(dst, src) + } else { + return int32Type{}.AssignValue(dst, src) + } +} + +func (t *intType) ConvertValue(val Value, typ Type) (Value, error) { + if t.BitWidth == 64 { + return int64Type{}.ConvertValue(val, typ) + } else { + return int32Type{}.ConvertValue(val, typ) + } +} + +// Decimal constructs a leaf node of decimal logical type with the given +// scale, precision, and underlying type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal +func Decimal(scale, precision int, typ Type) Node { + switch typ.Kind() { + case Int32, Int64, FixedLenByteArray: + default: + panic("DECIMAL node must annotate Int32, Int64 or FixedLenByteArray but got " + typ.String()) + } + return Leaf(&decimalType{ + decimal: format.DecimalType{ + Scale: int32(scale), + Precision: int32(precision), + }, + Type: typ, + }) +} + +type decimalType struct { + decimal format.DecimalType + Type +} + +func (t *decimalType) String() string { return t.decimal.String() } + +func (t *decimalType) LogicalType() *format.LogicalType { + return &format.LogicalType{Decimal: &t.decimal} +} + +func (t *decimalType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.Decimal] +} + +// String constructs a leaf node of UTF8 logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string +func String() Node { return Leaf(&stringType{}) } + +type stringType format.StringType + +func (t *stringType) String() string { return (*format.StringType)(t).String() } + +func (t *stringType) Kind() Kind { return ByteArray } + +func (t *stringType) Length() int { return 0 } + +func (t *stringType) EstimateSize(n int) int { return byteArrayType{}.EstimateSize(n) } + +func (t *stringType) EstimateNumValues(n int) int { return byteArrayType{}.EstimateNumValues(n) } + +func (t *stringType) Compare(a, b Value) int { + return bytes.Compare(a.byteArray(), b.byteArray()) +} + +func (t *stringType) ColumnOrder() *format.ColumnOrder { + return &typeDefinedColumnOrder +} + +func (t *stringType) PhysicalType() *format.Type { + return &physicalTypes[ByteArray] +} + +func (t *stringType) LogicalType() *format.LogicalType { + return &format.LogicalType{UTF8: (*format.StringType)(t)} +} + +func (t *stringType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.UTF8] +} + +func (t *stringType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return newByteArrayColumnIndexer(sizeLimit) +} + +func (t *stringType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return newByteArrayDictionary(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t *stringType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return newByteArrayColumnBuffer(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t *stringType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return newByteArrayPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues), data) +} + +func (t *stringType) NewValues(values []byte, offsets []uint32) encoding.Values { + return encoding.ByteArrayValues(values, offsets) +} + +func (t *stringType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return encoding.EncodeByteArray(dst, src, enc) +} + +func (t *stringType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return encoding.DecodeByteArray(dst, src, enc) +} + +func (t *stringType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return byteArrayType{}.EstimateDecodeSize(numValues, src, enc) +} + +func (t *stringType) AssignValue(dst reflect.Value, src Value) error { + return byteArrayType{}.AssignValue(dst, src) +} + +func (t *stringType) ConvertValue(val Value, typ Type) (Value, error) { + switch t2 := typ.(type) { + case *dateType: + return convertDateToString(val) + case *timeType: + tz := t2.tz() + if t2.Unit.Micros != nil { + return convertTimeMicrosToString(val, tz) + } else { + return convertTimeMillisToString(val, tz) + } + } + switch typ.Kind() { + case Boolean: + return convertBooleanToString(val) + case Int32: + return convertInt32ToString(val) + case Int64: + return convertInt64ToString(val) + case Int96: + return convertInt96ToString(val) + case Float: + return convertFloatToString(val) + case Double: + return convertDoubleToString(val) + case ByteArray: + return val, nil + case FixedLenByteArray: + return convertFixedLenByteArrayToString(val) + default: + return makeValueKind(ByteArray), nil + } +} + +// UUID constructs a leaf node of UUID logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid +func UUID() Node { return Leaf(&uuidType{}) } + +type uuidType format.UUIDType + +func (t *uuidType) String() string { return (*format.UUIDType)(t).String() } + +func (t *uuidType) Kind() Kind { return be128Type{}.Kind() } + +func (t *uuidType) Length() int { return be128Type{}.Length() } + +func (t *uuidType) EstimateSize(n int) int { return be128Type{}.EstimateSize(n) } + +func (t *uuidType) EstimateNumValues(n int) int { return be128Type{}.EstimateNumValues(n) } + +func (t *uuidType) Compare(a, b Value) int { return be128Type{}.Compare(a, b) } + +func (t *uuidType) ColumnOrder() *format.ColumnOrder { return &typeDefinedColumnOrder } + +func (t *uuidType) PhysicalType() *format.Type { return &physicalTypes[FixedLenByteArray] } + +func (t *uuidType) LogicalType() *format.LogicalType { + return &format.LogicalType{UUID: (*format.UUIDType)(t)} +} + +func (t *uuidType) ConvertedType() *deprecated.ConvertedType { return nil } + +func (t *uuidType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return be128Type{}.NewColumnIndexer(sizeLimit) +} + +func (t *uuidType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return be128Type{}.NewDictionary(columnIndex, numValues, data) +} + +func (t *uuidType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return be128Type{}.NewColumnBuffer(columnIndex, numValues) +} + +func (t *uuidType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return be128Type{}.NewPage(columnIndex, numValues, data) +} + +func (t *uuidType) NewValues(values []byte, offsets []uint32) encoding.Values { + return be128Type{}.NewValues(values, offsets) +} + +func (t *uuidType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return be128Type{}.Encode(dst, src, enc) +} + +func (t *uuidType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return be128Type{}.Decode(dst, src, enc) +} + +func (t *uuidType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return be128Type{}.EstimateDecodeSize(numValues, src, enc) +} + +func (t *uuidType) AssignValue(dst reflect.Value, src Value) error { + return be128Type{}.AssignValue(dst, src) +} + +func (t *uuidType) ConvertValue(val Value, typ Type) (Value, error) { + return be128Type{}.ConvertValue(val, typ) +} + +// Enum constructs a leaf node with a logical type representing enumerations. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum +func Enum() Node { return Leaf(&enumType{}) } + +type enumType format.EnumType + +func (t *enumType) String() string { return (*format.EnumType)(t).String() } + +func (t *enumType) Kind() Kind { return new(stringType).Kind() } + +func (t *enumType) Length() int { return new(stringType).Length() } + +func (t *enumType) EstimateSize(n int) int { return new(stringType).EstimateSize(n) } + +func (t *enumType) EstimateNumValues(n int) int { return new(stringType).EstimateNumValues(n) } + +func (t *enumType) Compare(a, b Value) int { return new(stringType).Compare(a, b) } + +func (t *enumType) ColumnOrder() *format.ColumnOrder { return new(stringType).ColumnOrder() } + +func (t *enumType) PhysicalType() *format.Type { return new(stringType).PhysicalType() } + +func (t *enumType) LogicalType() *format.LogicalType { + return &format.LogicalType{Enum: (*format.EnumType)(t)} +} + +func (t *enumType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.Enum] +} + +func (t *enumType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return new(stringType).NewColumnIndexer(sizeLimit) +} + +func (t *enumType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return new(stringType).NewDictionary(columnIndex, numValues, data) +} + +func (t *enumType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return new(stringType).NewColumnBuffer(columnIndex, numValues) +} + +func (t *enumType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return new(stringType).NewPage(columnIndex, numValues, data) +} + +func (t *enumType) NewValues(values []byte, offsets []uint32) encoding.Values { + return new(stringType).NewValues(values, offsets) +} + +func (t *enumType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return new(stringType).Encode(dst, src, enc) +} + +func (t *enumType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return new(stringType).Decode(dst, src, enc) +} + +func (t *enumType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return new(stringType).EstimateDecodeSize(numValues, src, enc) +} + +func (t *enumType) AssignValue(dst reflect.Value, src Value) error { + return new(stringType).AssignValue(dst, src) +} + +func (t *enumType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *byteArrayType, *stringType, *enumType: + return val, nil + default: + return val, invalidConversion(val, "ENUM", typ.String()) + } +} + +// JSON constructs a leaf node of JSON logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json +func JSON() Node { return Leaf(&jsonType{}) } + +type jsonType format.JsonType + +func (t *jsonType) String() string { return (*format.JsonType)(t).String() } + +func (t *jsonType) Kind() Kind { return byteArrayType{}.Kind() } + +func (t *jsonType) Length() int { return byteArrayType{}.Length() } + +func (t *jsonType) EstimateSize(n int) int { return byteArrayType{}.EstimateSize(n) } + +func (t *jsonType) EstimateNumValues(n int) int { return byteArrayType{}.EstimateNumValues(n) } + +func (t *jsonType) Compare(a, b Value) int { return byteArrayType{}.Compare(a, b) } + +func (t *jsonType) ColumnOrder() *format.ColumnOrder { return byteArrayType{}.ColumnOrder() } + +func (t *jsonType) PhysicalType() *format.Type { return byteArrayType{}.PhysicalType() } + +func (t *jsonType) LogicalType() *format.LogicalType { + return &format.LogicalType{Json: (*format.JsonType)(t)} +} + +func (t *jsonType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.Json] +} + +func (t *jsonType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return byteArrayType{}.NewColumnIndexer(sizeLimit) +} + +func (t *jsonType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return byteArrayType{}.NewDictionary(columnIndex, numValues, data) +} + +func (t *jsonType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return byteArrayType{}.NewColumnBuffer(columnIndex, numValues) +} + +func (t *jsonType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return byteArrayType{}.NewPage(columnIndex, numValues, data) +} + +func (t *jsonType) NewValues(values []byte, offsets []uint32) encoding.Values { + return byteArrayType{}.NewValues(values, offsets) +} + +func (t *jsonType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return byteArrayType{}.Encode(dst, src, enc) +} + +func (t *jsonType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return byteArrayType{}.Decode(dst, src, enc) +} + +func (t *jsonType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return byteArrayType{}.EstimateDecodeSize(numValues, src, enc) +} + +func (t *jsonType) AssignValue(dst reflect.Value, src Value) error { + // Assign value using ByteArrayType for BC... + switch dst.Kind() { + case reflect.String: + return byteArrayType{}.AssignValue(dst, src) + case reflect.Slice: + if dst.Type().Elem().Kind() == reflect.Uint8 { + return byteArrayType{}.AssignValue(dst, src) + } + } + + // Otherwise handle with json.Unmarshal + b := src.byteArray() + val := reflect.New(dst.Type()).Elem() + err := json.Unmarshal(b, val.Addr().Interface()) + if err != nil { + return err + } + dst.Set(val) + return nil +} + +func (t *jsonType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *byteArrayType, *stringType, *jsonType: + return val, nil + default: + return val, invalidConversion(val, "JSON", typ.String()) + } +} + +// BSON constructs a leaf node of BSON logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson +func BSON() Node { return Leaf(&bsonType{}) } + +type bsonType format.BsonType + +func (t *bsonType) String() string { return (*format.BsonType)(t).String() } + +func (t *bsonType) Kind() Kind { return byteArrayType{}.Kind() } + +func (t *bsonType) Length() int { return byteArrayType{}.Length() } + +func (t *bsonType) EstimateSize(n int) int { return byteArrayType{}.EstimateSize(n) } + +func (t *bsonType) EstimateNumValues(n int) int { return byteArrayType{}.EstimateNumValues(n) } + +func (t *bsonType) Compare(a, b Value) int { return byteArrayType{}.Compare(a, b) } + +func (t *bsonType) ColumnOrder() *format.ColumnOrder { return byteArrayType{}.ColumnOrder() } + +func (t *bsonType) PhysicalType() *format.Type { return byteArrayType{}.PhysicalType() } + +func (t *bsonType) LogicalType() *format.LogicalType { + return &format.LogicalType{Bson: (*format.BsonType)(t)} +} + +func (t *bsonType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.Bson] +} + +func (t *bsonType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return byteArrayType{}.NewColumnIndexer(sizeLimit) +} + +func (t *bsonType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return byteArrayType{}.NewDictionary(columnIndex, numValues, data) +} + +func (t *bsonType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return byteArrayType{}.NewColumnBuffer(columnIndex, numValues) +} + +func (t *bsonType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return byteArrayType{}.NewPage(columnIndex, numValues, data) +} + +func (t *bsonType) NewValues(values []byte, offsets []uint32) encoding.Values { + return byteArrayType{}.NewValues(values, offsets) +} + +func (t *bsonType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return byteArrayType{}.Encode(dst, src, enc) +} + +func (t *bsonType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return byteArrayType{}.Decode(dst, src, enc) +} + +func (t *bsonType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return byteArrayType{}.EstimateDecodeSize(numValues, src, enc) +} + +func (t *bsonType) AssignValue(dst reflect.Value, src Value) error { + return byteArrayType{}.AssignValue(dst, src) +} + +func (t *bsonType) ConvertValue(val Value, typ Type) (Value, error) { + switch typ.(type) { + case *byteArrayType, *bsonType: + return val, nil + default: + return val, invalidConversion(val, "BSON", typ.String()) + } +} + +// Date constructs a leaf node of DATE logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date +func Date() Node { return Leaf(&dateType{}) } + +type dateType format.DateType + +func (t *dateType) String() string { return (*format.DateType)(t).String() } + +func (t *dateType) Kind() Kind { return int32Type{}.Kind() } + +func (t *dateType) Length() int { return int32Type{}.Length() } + +func (t *dateType) EstimateSize(n int) int { return int32Type{}.EstimateSize(n) } + +func (t *dateType) EstimateNumValues(n int) int { return int32Type{}.EstimateNumValues(n) } + +func (t *dateType) Compare(a, b Value) int { return int32Type{}.Compare(a, b) } + +func (t *dateType) ColumnOrder() *format.ColumnOrder { return int32Type{}.ColumnOrder() } + +func (t *dateType) PhysicalType() *format.Type { return int32Type{}.PhysicalType() } + +func (t *dateType) LogicalType() *format.LogicalType { + return &format.LogicalType{Date: (*format.DateType)(t)} +} + +func (t *dateType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.Date] +} + +func (t *dateType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return int32Type{}.NewColumnIndexer(sizeLimit) +} + +func (t *dateType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return int32Type{}.NewDictionary(columnIndex, numValues, data) +} + +func (t *dateType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return int32Type{}.NewColumnBuffer(columnIndex, numValues) +} + +func (t *dateType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return int32Type{}.NewPage(columnIndex, numValues, data) +} + +func (t *dateType) NewValues(values []byte, offsets []uint32) encoding.Values { + return int32Type{}.NewValues(values, offsets) +} + +func (t *dateType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return int32Type{}.Encode(dst, src, enc) +} + +func (t *dateType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return int32Type{}.Decode(dst, src, enc) +} + +func (t *dateType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return int32Type{}.EstimateDecodeSize(numValues, src, enc) +} + +func (t *dateType) AssignValue(dst reflect.Value, src Value) error { + return int32Type{}.AssignValue(dst, src) +} + +func (t *dateType) ConvertValue(val Value, typ Type) (Value, error) { + switch src := typ.(type) { + case *stringType: + return convertStringToDate(val, time.UTC) + case *timestampType: + return convertTimestampToDate(val, src.Unit, src.tz()) + } + return int32Type{}.ConvertValue(val, typ) +} + +// TimeUnit represents units of time in the parquet type system. +type TimeUnit interface { + // Returns the precision of the time unit as a time.Duration value. + Duration() time.Duration + // Converts the TimeUnit value to its representation in the parquet thrift + // format. + TimeUnit() format.TimeUnit +} + +var ( + Millisecond TimeUnit = &millisecond{} + Microsecond TimeUnit = µsecond{} + Nanosecond TimeUnit = &nanosecond{} +) + +type millisecond format.MilliSeconds + +func (u *millisecond) Duration() time.Duration { return time.Millisecond } +func (u *millisecond) TimeUnit() format.TimeUnit { + return format.TimeUnit{Millis: (*format.MilliSeconds)(u)} +} + +type microsecond format.MicroSeconds + +func (u *microsecond) Duration() time.Duration { return time.Microsecond } +func (u *microsecond) TimeUnit() format.TimeUnit { + return format.TimeUnit{Micros: (*format.MicroSeconds)(u)} +} + +type nanosecond format.NanoSeconds + +func (u *nanosecond) Duration() time.Duration { return time.Nanosecond } +func (u *nanosecond) TimeUnit() format.TimeUnit { + return format.TimeUnit{Nanos: (*format.NanoSeconds)(u)} +} + +// Time constructs a leaf node of TIME logical type. +// IsAdjustedToUTC is true by default. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time +func Time(unit TimeUnit) Node { + return TimeAdjusted(unit, true) +} + +// TimeAdjusted constructs a leaf node of TIME logical type +// with the IsAdjustedToUTC property explicitly set. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time +func TimeAdjusted(unit TimeUnit, isAdjustedToUTC bool) Node { + return Leaf(&timeType{IsAdjustedToUTC: isAdjustedToUTC, Unit: unit.TimeUnit()}) +} + +type timeType format.TimeType + +func (t *timeType) tz() *time.Location { + if t.IsAdjustedToUTC { + return time.UTC + } else { + return time.Local + } +} + +func (t *timeType) baseType() Type { + if t.useInt32() { + return int32Type{} + } else { + return int64Type{} + } +} + +func (t *timeType) useInt32() bool { return t.Unit.Millis != nil } + +func (t *timeType) useInt64() bool { return t.Unit.Micros != nil } + +func (t *timeType) String() string { return (*format.TimeType)(t).String() } + +func (t *timeType) Kind() Kind { return t.baseType().Kind() } + +func (t *timeType) Length() int { return t.baseType().Length() } + +func (t *timeType) EstimateSize(n int) int { return t.baseType().EstimateSize(n) } + +func (t *timeType) EstimateNumValues(n int) int { return t.baseType().EstimateNumValues(n) } + +func (t *timeType) Compare(a, b Value) int { return t.baseType().Compare(a, b) } + +func (t *timeType) ColumnOrder() *format.ColumnOrder { return t.baseType().ColumnOrder() } + +func (t *timeType) PhysicalType() *format.Type { return t.baseType().PhysicalType() } + +func (t *timeType) LogicalType() *format.LogicalType { + return &format.LogicalType{Time: (*format.TimeType)(t)} +} + +func (t *timeType) ConvertedType() *deprecated.ConvertedType { + switch { + case t.useInt32(): + return &convertedTypes[deprecated.TimeMillis] + case t.useInt64(): + return &convertedTypes[deprecated.TimeMicros] + default: + return nil + } +} + +func (t *timeType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return t.baseType().NewColumnIndexer(sizeLimit) +} + +func (t *timeType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return t.baseType().NewColumnBuffer(columnIndex, numValues) +} + +func (t *timeType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return t.baseType().NewDictionary(columnIndex, numValues, data) +} + +func (t *timeType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return t.baseType().NewPage(columnIndex, numValues, data) +} + +func (t *timeType) NewValues(values []byte, offset []uint32) encoding.Values { + return t.baseType().NewValues(values, offset) +} + +func (t *timeType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return t.baseType().Encode(dst, src, enc) +} + +func (t *timeType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return t.baseType().Decode(dst, src, enc) +} + +func (t *timeType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return t.baseType().EstimateDecodeSize(numValues, src, enc) +} + +func (t *timeType) AssignValue(dst reflect.Value, src Value) error { + return t.baseType().AssignValue(dst, src) +} + +func (t *timeType) ConvertValue(val Value, typ Type) (Value, error) { + switch src := typ.(type) { + case *stringType: + tz := t.tz() + if t.Unit.Micros != nil { + return convertStringToTimeMicros(val, tz) + } else { + return convertStringToTimeMillis(val, tz) + } + case *timestampType: + tz := t.tz() + if t.Unit.Micros != nil { + return convertTimestampToTimeMicros(val, src.Unit, src.tz(), tz) + } else { + return convertTimestampToTimeMillis(val, src.Unit, src.tz(), tz) + } + } + return t.baseType().ConvertValue(val, typ) +} + +// Timestamp constructs of leaf node of TIMESTAMP logical type. +// IsAdjustedToUTC is true by default. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp +func Timestamp(unit TimeUnit) Node { + return TimestampAdjusted(unit, true) +} + +// TimestampAdjusted constructs a leaf node of TIMESTAMP logical type +// with the IsAdjustedToUTC property explicitly set. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time +func TimestampAdjusted(unit TimeUnit, isAdjustedToUTC bool) Node { + return Leaf(×tampType{IsAdjustedToUTC: isAdjustedToUTC, Unit: unit.TimeUnit()}) +} + +type timestampType format.TimestampType + +func (t *timestampType) tz() *time.Location { + if t.IsAdjustedToUTC { + return time.UTC + } else { + return time.Local + } +} + +func (t *timestampType) String() string { return (*format.TimestampType)(t).String() } + +func (t *timestampType) Kind() Kind { return int64Type{}.Kind() } + +func (t *timestampType) Length() int { return int64Type{}.Length() } + +func (t *timestampType) EstimateSize(n int) int { return int64Type{}.EstimateSize(n) } + +func (t *timestampType) EstimateNumValues(n int) int { return int64Type{}.EstimateNumValues(n) } + +func (t *timestampType) Compare(a, b Value) int { return int64Type{}.Compare(a, b) } + +func (t *timestampType) ColumnOrder() *format.ColumnOrder { return int64Type{}.ColumnOrder() } + +func (t *timestampType) PhysicalType() *format.Type { return int64Type{}.PhysicalType() } + +func (t *timestampType) LogicalType() *format.LogicalType { + return &format.LogicalType{Timestamp: (*format.TimestampType)(t)} +} + +func (t *timestampType) ConvertedType() *deprecated.ConvertedType { + switch { + case t.Unit.Millis != nil: + return &convertedTypes[deprecated.TimestampMillis] + case t.Unit.Micros != nil: + return &convertedTypes[deprecated.TimestampMicros] + default: + return nil + } +} + +func (t *timestampType) NewColumnIndexer(sizeLimit int) ColumnIndexer { + return int64Type{}.NewColumnIndexer(sizeLimit) +} + +func (t *timestampType) NewDictionary(columnIndex, numValues int, data encoding.Values) Dictionary { + return int64Type{}.NewDictionary(columnIndex, numValues, data) +} + +func (t *timestampType) NewColumnBuffer(columnIndex, numValues int) ColumnBuffer { + return int64Type{}.NewColumnBuffer(columnIndex, numValues) +} + +func (t *timestampType) NewPage(columnIndex, numValues int, data encoding.Values) Page { + return int64Type{}.NewPage(columnIndex, numValues, data) +} + +func (t *timestampType) NewValues(values []byte, offsets []uint32) encoding.Values { + return int64Type{}.NewValues(values, offsets) +} + +func (t *timestampType) Encode(dst []byte, src encoding.Values, enc encoding.Encoding) ([]byte, error) { + return int64Type{}.Encode(dst, src, enc) +} + +func (t *timestampType) Decode(dst encoding.Values, src []byte, enc encoding.Encoding) (encoding.Values, error) { + return int64Type{}.Decode(dst, src, enc) +} + +func (t *timestampType) EstimateDecodeSize(numValues int, src []byte, enc encoding.Encoding) int { + return int64Type{}.EstimateDecodeSize(numValues, src, enc) +} + +func (t *timestampType) AssignValue(dst reflect.Value, src Value) error { + switch dst.Type() { + case reflect.TypeOf(time.Time{}): + unit := Nanosecond.TimeUnit() + lt := t.LogicalType() + if lt != nil && lt.Timestamp != nil { + unit = lt.Timestamp.Unit + } + + nanos := src.int64() + switch { + case unit.Millis != nil: + nanos = nanos * 1e6 + case unit.Micros != nil: + nanos = nanos * 1e3 + } + + val := time.Unix(0, nanos).UTC() + dst.Set(reflect.ValueOf(val)) + return nil + default: + return int64Type{}.AssignValue(dst, src) + } +} + +func (t *timestampType) ConvertValue(val Value, typ Type) (Value, error) { + switch src := typ.(type) { + case *timestampType: + return convertTimestampToTimestamp(val, src.Unit, t.Unit) + case *dateType: + return convertDateToTimestamp(val, t.Unit, t.tz()) + } + return int64Type{}.ConvertValue(val, typ) +} + +// List constructs a node of LIST logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists +func List(of Node) Node { + return listNode{Group{"list": Repeated(Group{"element": of})}} +} + +type listNode struct{ Group } + +func (listNode) Type() Type { return &listType{} } + +type listType format.ListType + +func (t *listType) String() string { return (*format.ListType)(t).String() } + +func (t *listType) Kind() Kind { panic("cannot call Kind on parquet LIST type") } + +func (t *listType) Length() int { return 0 } + +func (t *listType) EstimateSize(int) int { return 0 } + +func (t *listType) EstimateNumValues(int) int { return 0 } + +func (t *listType) Compare(Value, Value) int { panic("cannot compare values on parquet LIST type") } + +func (t *listType) ColumnOrder() *format.ColumnOrder { return nil } + +func (t *listType) PhysicalType() *format.Type { return nil } + +func (t *listType) LogicalType() *format.LogicalType { + return &format.LogicalType{List: (*format.ListType)(t)} +} + +func (t *listType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.List] +} + +func (t *listType) NewColumnIndexer(int) ColumnIndexer { + panic("create create column indexer from parquet LIST type") +} + +func (t *listType) NewDictionary(int, int, encoding.Values) Dictionary { + panic("cannot create dictionary from parquet LIST type") +} + +func (t *listType) NewColumnBuffer(int, int) ColumnBuffer { + panic("cannot create column buffer from parquet LIST type") +} + +func (t *listType) NewPage(int, int, encoding.Values) Page { + panic("cannot create page from parquet LIST type") +} + +func (t *listType) NewValues(values []byte, _ []uint32) encoding.Values { + panic("cannot create values from parquet LIST type") +} + +func (t *listType) Encode(_ []byte, _ encoding.Values, _ encoding.Encoding) ([]byte, error) { + panic("cannot encode parquet LIST type") +} + +func (t *listType) Decode(_ encoding.Values, _ []byte, _ encoding.Encoding) (encoding.Values, error) { + panic("cannot decode parquet LIST type") +} + +func (t *listType) EstimateDecodeSize(_ int, _ []byte, _ encoding.Encoding) int { + panic("cannot estimate decode size of parquet LIST type") +} + +func (t *listType) AssignValue(reflect.Value, Value) error { + panic("cannot assign value to a parquet LIST type") +} + +func (t *listType) ConvertValue(Value, Type) (Value, error) { + panic("cannot convert value to a parquet LIST type") +} + +// Map constructs a node of MAP logical type. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps +func Map(key, value Node) Node { + return mapNode{Group{ + "key_value": Repeated(Group{ + "key": Required(key), + "value": value, + }), + }} +} + +type mapNode struct{ Group } + +func (mapNode) Type() Type { return &mapType{} } + +type mapType format.MapType + +func (t *mapType) String() string { return (*format.MapType)(t).String() } + +func (t *mapType) Kind() Kind { panic("cannot call Kind on parquet MAP type") } + +func (t *mapType) Length() int { return 0 } + +func (t *mapType) EstimateSize(int) int { return 0 } + +func (t *mapType) EstimateNumValues(int) int { return 0 } + +func (t *mapType) Compare(Value, Value) int { panic("cannot compare values on parquet MAP type") } + +func (t *mapType) ColumnOrder() *format.ColumnOrder { return nil } + +func (t *mapType) PhysicalType() *format.Type { return nil } + +func (t *mapType) LogicalType() *format.LogicalType { + return &format.LogicalType{Map: (*format.MapType)(t)} +} + +func (t *mapType) ConvertedType() *deprecated.ConvertedType { + return &convertedTypes[deprecated.Map] +} + +func (t *mapType) NewColumnIndexer(int) ColumnIndexer { + panic("create create column indexer from parquet MAP type") +} + +func (t *mapType) NewDictionary(int, int, encoding.Values) Dictionary { + panic("cannot create dictionary from parquet MAP type") +} + +func (t *mapType) NewColumnBuffer(int, int) ColumnBuffer { + panic("cannot create column buffer from parquet MAP type") +} + +func (t *mapType) NewPage(int, int, encoding.Values) Page { + panic("cannot create page from parquet MAP type") +} + +func (t *mapType) NewValues(values []byte, _ []uint32) encoding.Values { + panic("cannot create values from parquet MAP type") +} + +func (t *mapType) Encode(_ []byte, _ encoding.Values, _ encoding.Encoding) ([]byte, error) { + panic("cannot encode parquet MAP type") +} + +func (t *mapType) Decode(_ encoding.Values, _ []byte, _ encoding.Encoding) (encoding.Values, error) { + panic("cannot decode parquet MAP type") +} + +func (t *mapType) EstimateDecodeSize(_ int, _ []byte, _ encoding.Encoding) int { + panic("cannot estimate decode size of parquet MAP type") +} + +func (t *mapType) AssignValue(reflect.Value, Value) error { + panic("cannot assign value to a parquet MAP type") +} + +func (t *mapType) ConvertValue(Value, Type) (Value, error) { + panic("cannot convert value to a parquet MAP type") +} + +type nullType format.NullType + +func (t *nullType) String() string { return (*format.NullType)(t).String() } + +func (t *nullType) Kind() Kind { return -1 } + +func (t *nullType) Length() int { return 0 } + +func (t *nullType) EstimateSize(int) int { return 0 } + +func (t *nullType) EstimateNumValues(int) int { return 0 } + +func (t *nullType) Compare(Value, Value) int { panic("cannot compare values on parquet NULL type") } + +func (t *nullType) ColumnOrder() *format.ColumnOrder { return nil } + +func (t *nullType) PhysicalType() *format.Type { return nil } + +func (t *nullType) LogicalType() *format.LogicalType { + return &format.LogicalType{Unknown: (*format.NullType)(t)} +} + +func (t *nullType) ConvertedType() *deprecated.ConvertedType { return nil } + +func (t *nullType) NewColumnIndexer(int) ColumnIndexer { + panic("create create column indexer from parquet NULL type") +} + +func (t *nullType) NewDictionary(int, int, encoding.Values) Dictionary { + panic("cannot create dictionary from parquet NULL type") +} + +func (t *nullType) NewColumnBuffer(int, int) ColumnBuffer { + panic("cannot create column buffer from parquet NULL type") +} + +func (t *nullType) NewPage(columnIndex, numValues int, _ encoding.Values) Page { + return newNullPage(t, makeColumnIndex(columnIndex), makeNumValues(numValues)) +} + +func (t *nullType) NewValues(_ []byte, _ []uint32) encoding.Values { + return encoding.Values{} +} + +func (t *nullType) Encode(dst []byte, _ encoding.Values, _ encoding.Encoding) ([]byte, error) { + return dst[:0], nil +} + +func (t *nullType) Decode(dst encoding.Values, _ []byte, _ encoding.Encoding) (encoding.Values, error) { + return dst, nil +} + +func (t *nullType) EstimateDecodeSize(_ int, _ []byte, _ encoding.Encoding) int { + return 0 +} + +func (t *nullType) AssignValue(reflect.Value, Value) error { + return nil +} + +func (t *nullType) ConvertValue(val Value, _ Type) (Value, error) { + return val, nil +} + +// Variant constructs a node of unshredded VARIANT logical type. It is a group with +// two required fields, "metadata" and "value", both byte arrays. +// +// Experimental: The specification for variants is still being developed and the type +// is not fully adopted. Support for this type is subject to change. +// +// Initial support does not attempt to process the variant data. So reading and writing +// data of this type behaves as if it were just a group with two byte array fields, as +// if the logical type annotation were absent. This may change in the future. +// +// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#variant +func Variant() Node { + return variantNode{Group{"metadata": Required(Leaf(ByteArrayType)), "value": Required(Leaf(ByteArrayType))}} +} + +// TODO: add ShreddedVariant(Node) function, to create a shredded variant +// where the argument defines the type/structure of the shredded value(s). + +type variantNode struct{ Group } + +func (variantNode) Type() Type { return &variantType{} } + +type variantType format.VariantType + +func (t *variantType) String() string { return (*format.VariantType)(t).String() } + +func (t *variantType) Kind() Kind { panic("cannot call Kind on parquet VARIANT type") } + +func (t *variantType) Length() int { return 0 } + +func (t *variantType) EstimateSize(int) int { return 0 } + +func (t *variantType) EstimateNumValues(int) int { return 0 } + +func (t *variantType) Compare(Value, Value) int { + panic("cannot compare values on parquet VARIANT type") +} + +func (t *variantType) ColumnOrder() *format.ColumnOrder { return nil } + +func (t *variantType) PhysicalType() *format.Type { return nil } + +func (t *variantType) LogicalType() *format.LogicalType { + return &format.LogicalType{Variant: (*format.VariantType)(t)} +} + +func (t *variantType) ConvertedType() *deprecated.ConvertedType { return nil } + +func (t *variantType) NewColumnIndexer(int) ColumnIndexer { + panic("create create column indexer from parquet VARIANT type") +} + +func (t *variantType) NewDictionary(int, int, encoding.Values) Dictionary { + panic("cannot create dictionary from parquet VARIANT type") +} + +func (t *variantType) NewColumnBuffer(int, int) ColumnBuffer { + panic("cannot create column buffer from parquet VARIANT type") +} + +func (t *variantType) NewPage(int, int, encoding.Values) Page { + panic("cannot create page from parquet VARIANT type") +} + +func (t *variantType) NewValues(values []byte, _ []uint32) encoding.Values { + panic("cannot create values from parquet VARIANT type") +} + +func (t *variantType) Encode(_ []byte, _ encoding.Values, _ encoding.Encoding) ([]byte, error) { + panic("cannot encode parquet VARIANT type") +} + +func (t *variantType) Decode(_ encoding.Values, _ []byte, _ encoding.Encoding) (encoding.Values, error) { + panic("cannot decode parquet VARIANT type") +} + +func (t *variantType) EstimateDecodeSize(_ int, _ []byte, _ encoding.Encoding) int { + panic("cannot estimate decode size of parquet VARIANT type") +} + +func (t *variantType) AssignValue(reflect.Value, Value) error { + panic("cannot assign value to a parquet VARIANT type") +} + +func (t *variantType) ConvertValue(Value, Type) (Value, error) { + panic("cannot convert value to a parquet VARIANT type") +} + +type groupType struct{} + +func (groupType) String() string { return "group" } + +func (groupType) Kind() Kind { + panic("cannot call Kind on parquet group") +} + +func (groupType) Compare(Value, Value) int { + panic("cannot compare values on parquet group") +} + +func (groupType) NewColumnIndexer(int) ColumnIndexer { + panic("cannot create column indexer from parquet group") +} + +func (groupType) NewDictionary(int, int, encoding.Values) Dictionary { + panic("cannot create dictionary from parquet group") +} + +func (t groupType) NewColumnBuffer(int, int) ColumnBuffer { + panic("cannot create column buffer from parquet group") +} + +func (t groupType) NewPage(int, int, encoding.Values) Page { + panic("cannot create page from parquet group") +} + +func (t groupType) NewValues(_ []byte, _ []uint32) encoding.Values { + panic("cannot create values from parquet group") +} + +func (groupType) Encode(_ []byte, _ encoding.Values, _ encoding.Encoding) ([]byte, error) { + panic("cannot encode parquet group") +} + +func (groupType) Decode(_ encoding.Values, _ []byte, _ encoding.Encoding) (encoding.Values, error) { + panic("cannot decode parquet group") +} + +func (groupType) EstimateDecodeSize(_ int, _ []byte, _ encoding.Encoding) int { + panic("cannot estimate decode size of parquet group") +} + +func (groupType) AssignValue(reflect.Value, Value) error { + panic("cannot assign value to a parquet group") +} + +func (t groupType) ConvertValue(Value, Type) (Value, error) { + panic("cannot convert value to a parquet group") +} + +func (groupType) Length() int { return 0 } + +func (groupType) EstimateSize(int) int { return 0 } + +func (groupType) EstimateNumValues(int) int { return 0 } + +func (groupType) ColumnOrder() *format.ColumnOrder { return nil } + +func (groupType) PhysicalType() *format.Type { return nil } + +func (groupType) LogicalType() *format.LogicalType { return nil } + +func (groupType) ConvertedType() *deprecated.ConvertedType { return nil } + +func checkTypeKindEqual(to, from Type) error { + if to.Kind() != from.Kind() { + return fmt.Errorf("cannot convert from parquet value of type %s to %s", from, to) + } + return nil +} diff --git a/vendor/github.com/parquet-go/parquet-go/value.go b/vendor/github.com/parquet-go/parquet-go/value.go new file mode 100644 index 0000000000..9281493dd1 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/value.go @@ -0,0 +1,1072 @@ +package parquet + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" + "unsafe" + + "github.com/google/uuid" + "github.com/parquet-go/parquet-go/deprecated" + "github.com/parquet-go/parquet-go/format" +) + +const ( + // 170 x sizeof(Value) = 4KB + defaultValueBufferSize = 170 + + offsetOfPtr = unsafe.Offsetof(Value{}.ptr) + offsetOfU64 = unsafe.Offsetof(Value{}.u64) + offsetOfU32 = offsetOfU64 + firstByteOffsetOf32BitsValue + offsetOfBool = offsetOfU64 + firstByteOffsetOfBooleanValue +) + +// The Value type is similar to the reflect.Value abstraction of Go values, but +// for parquet values. Value instances wrap underlying Go values mapped to one +// of the parquet physical types. +// +// Value instances are small, immutable objects, and usually passed by value +// between function calls. +// +// The zero-value of Value represents the null parquet value. +type Value struct { + // data + ptr *byte + u64 uint64 + // type + kind int8 // XOR(Kind) so the zero-value is + // levels + definitionLevel byte + repetitionLevel byte + columnIndex int16 // XOR so the zero-value is -1 +} + +// ValueReader is an interface implemented by types that support reading +// batches of values. +type ValueReader interface { + // Read values into the buffer passed as argument and return the number of + // values read. When all values have been read, the error will be io.EOF. + ReadValues([]Value) (int, error) +} + +// ValueReaderAt is an interface implemented by types that support reading +// values at offsets specified by the application. +type ValueReaderAt interface { + ReadValuesAt([]Value, int64) (int, error) +} + +// ValueReaderFrom is an interface implemented by value writers to read values +// from a reader. +type ValueReaderFrom interface { + ReadValuesFrom(ValueReader) (int64, error) +} + +// ValueWriter is an interface implemented by types that support reading +// batches of values. +type ValueWriter interface { + // Write values from the buffer passed as argument and returns the number + // of values written. + WriteValues([]Value) (int, error) +} + +// ValueWriterTo is an interface implemented by value readers to write values to +// a writer. +type ValueWriterTo interface { + WriteValuesTo(ValueWriter) (int64, error) +} + +// ValueReaderFunc is a function type implementing the ValueReader interface. +type ValueReaderFunc func([]Value) (int, error) + +func (f ValueReaderFunc) ReadValues(values []Value) (int, error) { return f(values) } + +// ValueWriterFunc is a function type implementing the ValueWriter interface. +type ValueWriterFunc func([]Value) (int, error) + +func (f ValueWriterFunc) WriteValues(values []Value) (int, error) { return f(values) } + +// CopyValues copies values from src to dst, returning the number of values +// that were written. +// +// As an optimization, the reader and writer may choose to implement +// ValueReaderFrom and ValueWriterTo to provide their own copy logic. +// +// The function returns any error it encounters reading or writing pages, except +// for io.EOF from the reader which indicates that there were no more values to +// read. +func CopyValues(dst ValueWriter, src ValueReader) (int64, error) { + return copyValues(dst, src, nil) +} + +func copyValues(dst ValueWriter, src ValueReader, buf []Value) (written int64, err error) { + if wt, ok := src.(ValueWriterTo); ok { + return wt.WriteValuesTo(dst) + } + + if rf, ok := dst.(ValueReaderFrom); ok { + return rf.ReadValuesFrom(src) + } + + if len(buf) == 0 { + buf = make([]Value, defaultValueBufferSize) + } + + defer clearValues(buf) + + for { + n, err := src.ReadValues(buf) + + if n > 0 { + wn, werr := dst.WriteValues(buf[:n]) + written += int64(wn) + if werr != nil { + return written, werr + } + } + + if err != nil { + if err == io.EOF { + err = nil + } + return written, err + } + + if n == 0 { + return written, io.ErrNoProgress + } + } +} + +// ValueOf constructs a parquet value from a Go value v. +// +// The physical type of the value is assumed from the Go type of v using the +// following conversion table: +// +// Go type | Parquet physical type +// ------- | --------------------- +// nil | NULL +// bool | BOOLEAN +// int8 | INT32 +// int16 | INT32 +// int32 | INT32 +// int64 | INT64 +// int | INT64 +// uint8 | INT32 +// uint16 | INT32 +// uint32 | INT32 +// uint64 | INT64 +// uintptr | INT64 +// float32 | FLOAT +// float64 | DOUBLE +// string | BYTE_ARRAY +// []byte | BYTE_ARRAY +// [*]byte | FIXED_LEN_BYTE_ARRAY +// +// When converting a []byte or [*]byte value, the underlying byte array is not +// copied; instead, the returned parquet value holds a reference to it. +// +// The repetition and definition levels of the returned value are both zero. +// +// The function panics if the Go value cannot be represented in parquet. +func ValueOf(v any) Value { + k := Kind(-1) + t := reflect.TypeOf(v) + + switch value := v.(type) { + case nil: + return Value{} + case uuid.UUID: + return makeValueBytes(FixedLenByteArray, value[:]) + case deprecated.Int96: + return makeValueInt96(value) + case time.Time: + k = Int64 + } + + switch t.Kind() { + case reflect.Bool: + k = Boolean + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint8, reflect.Uint16, reflect.Uint32: + k = Int32 + case reflect.Int64, reflect.Int, reflect.Uint64, reflect.Uint, reflect.Uintptr: + k = Int64 + case reflect.Float32: + k = Float + case reflect.Float64: + k = Double + case reflect.String: + k = ByteArray + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + k = ByteArray + } + case reflect.Array: + if t.Elem().Kind() == reflect.Uint8 { + k = FixedLenByteArray + } + } + + if k < 0 { + panic("cannot create parquet value from go value of type " + t.String()) + } + + return makeValue(k, nil, reflect.ValueOf(v)) +} + +// NulLValue constructs a null value, which is the zero-value of the Value type. +func NullValue() Value { return Value{} } + +// ZeroValue constructs a zero value of the given kind. +func ZeroValue(kind Kind) Value { return makeValueKind(kind) } + +// BooleanValue constructs a BOOLEAN parquet value from the bool passed as +// argument. +func BooleanValue(value bool) Value { return makeValueBoolean(value) } + +// Int32Value constructs a INT32 parquet value from the int32 passed as +// argument. +func Int32Value(value int32) Value { return makeValueInt32(value) } + +// Int64Value constructs a INT64 parquet value from the int64 passed as +// argument. +func Int64Value(value int64) Value { return makeValueInt64(value) } + +// Int96Value constructs a INT96 parquet value from the deprecated.Int96 passed +// as argument. +func Int96Value(value deprecated.Int96) Value { return makeValueInt96(value) } + +// FloatValue constructs a FLOAT parquet value from the float32 passed as +// argument. +func FloatValue(value float32) Value { return makeValueFloat(value) } + +// DoubleValue constructs a DOUBLE parquet value from the float64 passed as +// argument. +func DoubleValue(value float64) Value { return makeValueDouble(value) } + +// ByteArrayValue constructs a BYTE_ARRAY parquet value from the byte slice +// passed as argument. +func ByteArrayValue(value []byte) Value { return makeValueBytes(ByteArray, value) } + +// FixedLenByteArrayValue constructs a BYTE_ARRAY parquet value from the byte +// slice passed as argument. +func FixedLenByteArrayValue(value []byte) Value { return makeValueBytes(FixedLenByteArray, value) } + +func makeValue(k Kind, lt *format.LogicalType, v reflect.Value) Value { + if v.Kind() == reflect.Interface { + if v.IsNil() { + return Value{} + } + if v = v.Elem(); v.Kind() == reflect.Pointer && v.IsNil() { + return Value{} + } + } + + switch v.Type() { + case reflect.TypeOf(time.Time{}): + unit := Nanosecond.TimeUnit() + if lt != nil && lt.Timestamp != nil { + unit = lt.Timestamp.Unit + } + + t := v.Interface().(time.Time) + var val int64 + switch { + case unit.Millis != nil: + val = t.UnixMilli() + case unit.Micros != nil: + val = t.UnixMicro() + default: + val = t.UnixNano() + } + return makeValueInt64(val) + } + + switch k { + case Boolean: + return makeValueBoolean(v.Bool()) + + case Int32: + switch v.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32: + return makeValueInt32(int32(v.Int())) + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + return makeValueInt32(int32(v.Uint())) + } + + case Int64: + switch v.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return makeValueInt64(v.Int()) + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr: + return makeValueUint64(v.Uint()) + } + + case Int96: + switch v.Type() { + case reflect.TypeOf(deprecated.Int96{}): + return makeValueInt96(v.Interface().(deprecated.Int96)) + } + + case Float: + switch v.Kind() { + case reflect.Float32: + return makeValueFloat(float32(v.Float())) + } + + case Double: + switch v.Kind() { + case reflect.Float32, reflect.Float64: + return makeValueDouble(v.Float()) + } + + case ByteArray: + switch v.Kind() { + case reflect.String: + return makeValueString(k, v.String()) + case reflect.Slice: + if v.Type().Elem().Kind() == reflect.Uint8 { + return makeValueBytes(k, v.Bytes()) + } + } + + case FixedLenByteArray: + switch v.Kind() { + case reflect.String: // uuid + return makeValueString(k, v.String()) + case reflect.Array: + if v.Type().Elem().Kind() == reflect.Uint8 { + return makeValueFixedLenByteArray(v) + } + case reflect.Slice: + if v.Type().Elem().Kind() == reflect.Uint8 { + return makeValueBytes(k, v.Bytes()) + } + } + } + + panic("cannot create parquet value of type " + k.String() + " from go value of type " + v.Type().String()) +} + +func makeValueKind(kind Kind) Value { + return Value{kind: ^int8(kind)} +} + +func makeValueBoolean(value bool) Value { + v := Value{kind: ^int8(Boolean)} + if value { + v.u64 = 1 + } + return v +} + +func makeValueInt32(value int32) Value { + return Value{ + kind: ^int8(Int32), + u64: uint64(value), + } +} + +func makeValueInt64(value int64) Value { + return Value{ + kind: ^int8(Int64), + u64: uint64(value), + } +} + +func makeValueInt96(value deprecated.Int96) Value { + // TODO: this is highly inefficient because we need a heap allocation to + // store the value; we don't expect INT96 to be used frequently since it + // is a deprecated feature of parquet, and it helps keep the Value type + // compact for all the other more common cases. + bits := [12]byte{} + binary.LittleEndian.PutUint32(bits[0:4], value[0]) + binary.LittleEndian.PutUint32(bits[4:8], value[1]) + binary.LittleEndian.PutUint32(bits[8:12], value[2]) + return Value{ + kind: ^int8(Int96), + ptr: &bits[0], + u64: 12, // set the length so we can use the ByteArray method + } +} + +func makeValueUint32(value uint32) Value { + return Value{ + kind: ^int8(Int32), + u64: uint64(value), + } +} + +func makeValueUint64(value uint64) Value { + return Value{ + kind: ^int8(Int64), + u64: value, + } +} + +func makeValueFloat(value float32) Value { + return Value{ + kind: ^int8(Float), + u64: uint64(math.Float32bits(value)), + } +} + +func makeValueDouble(value float64) Value { + return Value{ + kind: ^int8(Double), + u64: math.Float64bits(value), + } +} + +func makeValueBytes(kind Kind, value []byte) Value { + return makeValueByteArray(kind, unsafe.SliceData(value), len(value)) +} + +func makeValueString(kind Kind, value string) Value { + return makeValueByteArray(kind, unsafe.StringData(value), len(value)) +} + +func makeValueFixedLenByteArray(v reflect.Value) Value { + t := v.Type() + // When the array is addressable, we take advantage of this + // condition to avoid the heap allocation otherwise needed + // to pack the reference into an interface{} value. + if v.CanAddr() { + v = v.Addr() + } else { + u := reflect.New(t) + u.Elem().Set(v) + v = u + } + return makeValueByteArray(FixedLenByteArray, (*byte)(v.UnsafePointer()), t.Len()) +} + +func makeValueByteArray(kind Kind, data *byte, size int) Value { + return Value{ + kind: ^int8(kind), + ptr: data, + u64: uint64(size), + } +} + +// These methods are internal versions of methods exported by the Value type, +// they are usually inlined by the compiler and intended to be used inside the +// parquet-go package because they tend to generate better code than their +// exported counter part, which requires making a copy of the receiver. +func (v *Value) isNull() bool { return v.kind == 0 } +func (v *Value) byte() byte { return byte(v.u64) } +func (v *Value) boolean() bool { return v.u64 != 0 } +func (v *Value) int32() int32 { return int32(v.u64) } +func (v *Value) int64() int64 { return int64(v.u64) } +func (v *Value) int96() deprecated.Int96 { return makeInt96(v.byteArray()) } +func (v *Value) float() float32 { return math.Float32frombits(uint32(v.u64)) } +func (v *Value) double() float64 { return math.Float64frombits(uint64(v.u64)) } +func (v *Value) uint32() uint32 { return uint32(v.u64) } +func (v *Value) uint64() uint64 { return v.u64 } +func (v *Value) byteArray() []byte { return unsafe.Slice(v.ptr, v.u64) } +func (v *Value) string() string { return unsafe.String(v.ptr, v.u64) } +func (v *Value) be128() *[16]byte { return (*[16]byte)(unsafe.Pointer(v.ptr)) } +func (v *Value) column() int { return int(^v.columnIndex) } + +func (v Value) convertToBoolean(x bool) Value { + v.kind = ^int8(Boolean) + v.ptr = nil + v.u64 = 0 + if x { + v.u64 = 1 + } + return v +} + +func (v Value) convertToInt32(x int32) Value { + v.kind = ^int8(Int32) + v.ptr = nil + v.u64 = uint64(x) + return v +} + +func (v Value) convertToInt64(x int64) Value { + v.kind = ^int8(Int64) + v.ptr = nil + v.u64 = uint64(x) + return v +} + +func (v Value) convertToInt96(x deprecated.Int96) Value { + i96 := makeValueInt96(x) + v.kind = i96.kind + v.ptr = i96.ptr + v.u64 = i96.u64 + return v +} + +func (v Value) convertToFloat(x float32) Value { + v.kind = ^int8(Float) + v.ptr = nil + v.u64 = uint64(math.Float32bits(x)) + return v +} + +func (v Value) convertToDouble(x float64) Value { + v.kind = ^int8(Double) + v.ptr = nil + v.u64 = math.Float64bits(x) + return v +} + +func (v Value) convertToByteArray(x []byte) Value { + v.kind = ^int8(ByteArray) + v.ptr = unsafe.SliceData(x) + v.u64 = uint64(len(x)) + return v +} + +func (v Value) convertToFixedLenByteArray(x []byte) Value { + v.kind = ^int8(FixedLenByteArray) + v.ptr = unsafe.SliceData(x) + v.u64 = uint64(len(x)) + return v +} + +// Kind returns the kind of v, which represents its parquet physical type. +func (v Value) Kind() Kind { return ^Kind(v.kind) } + +// IsNull returns true if v is the null value. +func (v Value) IsNull() bool { return v.isNull() } + +// Byte returns v as a byte, which may truncate the underlying byte. +func (v Value) Byte() byte { return v.byte() } + +// Boolean returns v as a bool, assuming the underlying type is BOOLEAN. +func (v Value) Boolean() bool { return v.boolean() } + +// Int32 returns v as a int32, assuming the underlying type is INT32. +func (v Value) Int32() int32 { return v.int32() } + +// Int64 returns v as a int64, assuming the underlying type is INT64. +func (v Value) Int64() int64 { return v.int64() } + +// Int96 returns v as a int96, assuming the underlying type is INT96. +func (v Value) Int96() deprecated.Int96 { + var val deprecated.Int96 + if !v.isNull() { + val = v.int96() + } + return val +} + +// Float returns v as a float32, assuming the underlying type is FLOAT. +func (v Value) Float() float32 { return v.float() } + +// Double returns v as a float64, assuming the underlying type is DOUBLE. +func (v Value) Double() float64 { return v.double() } + +// Uint32 returns v as a uint32, assuming the underlying type is INT32. +func (v Value) Uint32() uint32 { return v.uint32() } + +// Uint64 returns v as a uint64, assuming the underlying type is INT64. +func (v Value) Uint64() uint64 { return v.uint64() } + +// ByteArray returns v as a []byte, assuming the underlying type is either +// BYTE_ARRAY or FIXED_LEN_BYTE_ARRAY. +// +// The application must treat the returned byte slice as a read-only value, +// mutating the content will result in undefined behaviors. +func (v Value) ByteArray() []byte { return v.byteArray() } + +// RepetitionLevel returns the repetition level of v. +func (v Value) RepetitionLevel() int { return int(v.repetitionLevel) } + +// DefinitionLevel returns the definition level of v. +func (v Value) DefinitionLevel() int { return int(v.definitionLevel) } + +// Column returns the column index within the row that v was created from. +// +// Returns -1 if the value does not carry a column index. +func (v Value) Column() int { return v.column() } + +// Bytes returns the binary representation of v. +// +// If v is the null value, an nil byte slice is returned. +func (v Value) Bytes() []byte { + switch v.Kind() { + case Boolean: + buf := [8]byte{} + binary.LittleEndian.PutUint32(buf[:4], v.uint32()) + return buf[0:1] + case Int32, Float: + buf := [8]byte{} + binary.LittleEndian.PutUint32(buf[:4], v.uint32()) + return buf[:4] + case Int64, Double: + buf := [8]byte{} + binary.LittleEndian.PutUint64(buf[:8], v.uint64()) + return buf[:8] + case ByteArray, FixedLenByteArray, Int96: + return v.byteArray() + default: + return nil + } +} + +// AppendBytes appends the binary representation of v to b. +// +// If v is the null value, b is returned unchanged. +func (v Value) AppendBytes(b []byte) []byte { + buf := [8]byte{} + switch v.Kind() { + case Boolean: + binary.LittleEndian.PutUint32(buf[:4], v.uint32()) + return append(b, buf[0]) + case Int32, Float: + binary.LittleEndian.PutUint32(buf[:4], v.uint32()) + return append(b, buf[:4]...) + case Int64, Double: + binary.LittleEndian.PutUint64(buf[:8], v.uint64()) + return append(b, buf[:8]...) + case ByteArray, FixedLenByteArray, Int96: + return append(b, v.byteArray()...) + default: + return b + } +} + +// Format outputs a human-readable representation of v to w, using r as the +// formatting verb to describe how the value should be printed. +// +// The following formatting options are supported: +// +// %c prints the column index +// %+c prints the column index, prefixed with "C:" +// %d prints the definition level +// %+d prints the definition level, prefixed with "D:" +// %r prints the repetition level +// %+r prints the repetition level, prefixed with "R:" +// %q prints the quoted representation of v +// %+q prints the quoted representation of v, prefixed with "V:" +// %s prints the string representation of v +// %+s prints the string representation of v, prefixed with "V:" +// %v same as %s +// %+v prints a verbose representation of v +// %#v prints a Go value representation of v +// +// Format satisfies the fmt.Formatter interface. +func (v Value) Format(w fmt.State, r rune) { + switch r { + case 'c': + if w.Flag('+') { + io.WriteString(w, "C:") + } + fmt.Fprint(w, v.column()) + + case 'd': + if w.Flag('+') { + io.WriteString(w, "D:") + } + fmt.Fprint(w, v.definitionLevel) + + case 'r': + if w.Flag('+') { + io.WriteString(w, "R:") + } + fmt.Fprint(w, v.repetitionLevel) + + case 'q': + if w.Flag('+') { + io.WriteString(w, "V:") + } + switch v.Kind() { + case ByteArray, FixedLenByteArray: + fmt.Fprintf(w, "%q", v.byteArray()) + default: + fmt.Fprintf(w, `"%s"`, v) + } + + case 's': + if w.Flag('+') { + io.WriteString(w, "V:") + } + switch v.Kind() { + case Boolean: + fmt.Fprint(w, v.boolean()) + case Int32: + fmt.Fprint(w, v.int32()) + case Int64: + fmt.Fprint(w, v.int64()) + case Int96: + fmt.Fprint(w, v.int96()) + case Float: + fmt.Fprint(w, v.float()) + case Double: + fmt.Fprint(w, v.double()) + case ByteArray, FixedLenByteArray: + w.Write(v.byteArray()) + default: + io.WriteString(w, "") + } + + case 'v': + switch { + case w.Flag('+'): + fmt.Fprintf(w, "%+[1]c %+[1]d %+[1]r %+[1]s", v) + case w.Flag('#'): + v.formatGoString(w) + default: + v.Format(w, 's') + } + } +} + +func (v Value) formatGoString(w fmt.State) { + io.WriteString(w, "parquet.") + switch v.Kind() { + case Boolean: + fmt.Fprintf(w, "BooleanValue(%t)", v.boolean()) + case Int32: + fmt.Fprintf(w, "Int32Value(%d)", v.int32()) + case Int64: + fmt.Fprintf(w, "Int64Value(%d)", v.int64()) + case Int96: + fmt.Fprintf(w, "Int96Value(%#v)", v.int96()) + case Float: + fmt.Fprintf(w, "FloatValue(%g)", v.float()) + case Double: + fmt.Fprintf(w, "DoubleValue(%g)", v.double()) + case ByteArray: + fmt.Fprintf(w, "ByteArrayValue(%q)", v.byteArray()) + case FixedLenByteArray: + fmt.Fprintf(w, "FixedLenByteArrayValue(%#v)", v.byteArray()) + default: + io.WriteString(w, "Value{}") + return + } + fmt.Fprintf(w, ".Level(%d,%d,%d)", + v.RepetitionLevel(), + v.DefinitionLevel(), + v.Column(), + ) +} + +// String returns a string representation of v. +func (v Value) String() string { + switch v.Kind() { + case Boolean: + return strconv.FormatBool(v.boolean()) + case Int32: + return strconv.FormatInt(int64(v.int32()), 10) + case Int64: + return strconv.FormatInt(v.int64(), 10) + case Int96: + return v.Int96().String() + case Float: + return strconv.FormatFloat(float64(v.float()), 'g', -1, 32) + case Double: + return strconv.FormatFloat(v.double(), 'g', -1, 32) + case ByteArray, FixedLenByteArray: + return string(v.byteArray()) + default: + return "" + } +} + +// GoString returns a Go value string representation of v. +func (v Value) GoString() string { return fmt.Sprintf("%#v", v) } + +// Level returns v with the repetition level, definition level, and column index +// set to the values passed as arguments. +// +// The method panics if either argument is negative. +func (v Value) Level(repetitionLevel, definitionLevel, columnIndex int) Value { + v.repetitionLevel = makeRepetitionLevel(repetitionLevel) + v.definitionLevel = makeDefinitionLevel(definitionLevel) + v.columnIndex = ^makeColumnIndex(columnIndex) + return v +} + +// Clone returns a copy of v which does not share any pointers with it. +func (v Value) Clone() Value { + switch k := v.Kind(); k { + case ByteArray, FixedLenByteArray: + v.ptr = unsafe.SliceData(copyBytes(v.byteArray())) + } + return v +} + +func makeInt96(bits []byte) (i96 deprecated.Int96) { + return deprecated.Int96{ + 2: binary.LittleEndian.Uint32(bits[8:12]), + 1: binary.LittleEndian.Uint32(bits[4:8]), + 0: binary.LittleEndian.Uint32(bits[0:4]), + } +} + +func parseValue(kind Kind, data []byte) (val Value, err error) { + switch kind { + case Boolean: + if len(data) == 1 { + val = makeValueBoolean(data[0] != 0) + } + case Int32: + if len(data) == 4 { + val = makeValueInt32(int32(binary.LittleEndian.Uint32(data))) + } + case Int64: + if len(data) == 8 { + val = makeValueInt64(int64(binary.LittleEndian.Uint64(data))) + } + case Int96: + if len(data) == 12 { + val = makeValueInt96(makeInt96(data)) + } + case Float: + if len(data) == 4 { + val = makeValueFloat(float32(math.Float32frombits(binary.LittleEndian.Uint32(data)))) + } + case Double: + if len(data) == 8 { + val = makeValueDouble(float64(math.Float64frombits(binary.LittleEndian.Uint64(data)))) + } + case ByteArray, FixedLenByteArray: + val = makeValueBytes(kind, data) + } + if val.isNull() { + err = fmt.Errorf("cannot decode %s value from input of length %d", kind, len(data)) + } + return val, err +} + +func copyBytes(b []byte) []byte { + c := make([]byte, len(b)) + copy(c, b) + return c +} + +// Equal returns true if v1 and v2 are equal. +// +// Values are considered equal if they are of the same physical type and hold +// the same Go values. For BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY, the content of +// the underlying byte arrays are tested for equality. +// +// Note that the repetition levels, definition levels, and column indexes are +// not compared by this function, use DeepEqual instead. +func Equal(v1, v2 Value) bool { + if v1.kind != v2.kind { + return false + } + switch ^Kind(v1.kind) { + case Boolean: + return v1.boolean() == v2.boolean() + case Int32: + return v1.int32() == v2.int32() + case Int64: + return v1.int64() == v2.int64() + case Int96: + return v1.int96() == v2.int96() + case Float: + return v1.float() == v2.float() + case Double: + return v1.double() == v2.double() + case ByteArray, FixedLenByteArray: + return bytes.Equal(v1.byteArray(), v2.byteArray()) + case -1: // null + return true + default: + return false + } +} + +// DeepEqual returns true if v1 and v2 are equal, including their repetition +// levels, definition levels, and column indexes. +// +// See Equal for details about how value equality is determined. +func DeepEqual(v1, v2 Value) bool { + return Equal(v1, v2) && + v1.repetitionLevel == v2.repetitionLevel && + v1.definitionLevel == v2.definitionLevel && + v1.columnIndex == v2.columnIndex +} + +var ( + _ fmt.Formatter = Value{} + _ fmt.Stringer = Value{} +) + +func clearValues(values []Value) { + for i := range values { + values[i] = Value{} + } +} + +// BooleanReader is an interface implemented by ValueReader instances which +// expose the content of a column of boolean values. +type BooleanReader interface { + // Read boolean values into the buffer passed as argument. + // + // The method returns io.EOF when all values have been read. + ReadBooleans(values []bool) (int, error) +} + +// BooleanWriter is an interface implemented by ValueWriter instances which +// support writing columns of boolean values. +type BooleanWriter interface { + // Write boolean values. + // + // The method returns the number of values written, and any error that + // occurred while writing the values. + WriteBooleans(values []bool) (int, error) +} + +// Int32Reader is an interface implemented by ValueReader instances which expose +// the content of a column of int32 values. +type Int32Reader interface { + // Read 32 bits integer values into the buffer passed as argument. + // + // The method returns io.EOF when all values have been read. + ReadInt32s(values []int32) (int, error) +} + +// Int32Writer is an interface implemented by ValueWriter instances which +// support writing columns of 32 bits signed integer values. +type Int32Writer interface { + // Write 32 bits signed integer values. + // + // The method returns the number of values written, and any error that + // occurred while writing the values. + WriteInt32s(values []int32) (int, error) +} + +// Int64Reader is an interface implemented by ValueReader instances which expose +// the content of a column of int64 values. +type Int64Reader interface { + // Read 64 bits integer values into the buffer passed as argument. + // + // The method returns io.EOF when all values have been read. + ReadInt64s(values []int64) (int, error) +} + +// Int64Writer is an interface implemented by ValueWriter instances which +// support writing columns of 64 bits signed integer values. +type Int64Writer interface { + // Write 64 bits signed integer values. + // + // The method returns the number of values written, and any error that + // occurred while writing the values. + WriteInt64s(values []int64) (int, error) +} + +// Int96Reader is an interface implemented by ValueReader instances which expose +// the content of a column of int96 values. +type Int96Reader interface { + // Read 96 bits integer values into the buffer passed as argument. + // + // The method returns io.EOF when all values have been read. + ReadInt96s(values []deprecated.Int96) (int, error) +} + +// Int96Writer is an interface implemented by ValueWriter instances which +// support writing columns of 96 bits signed integer values. +type Int96Writer interface { + // Write 96 bits signed integer values. + // + // The method returns the number of values written, and any error that + // occurred while writing the values. + WriteInt96s(values []deprecated.Int96) (int, error) +} + +// FloatReader is an interface implemented by ValueReader instances which expose +// the content of a column of single-precision floating point values. +type FloatReader interface { + // Read single-precision floating point values into the buffer passed as + // argument. + // + // The method returns io.EOF when all values have been read. + ReadFloats(values []float32) (int, error) +} + +// FloatWriter is an interface implemented by ValueWriter instances which +// support writing columns of single-precision floating point values. +type FloatWriter interface { + // Write single-precision floating point values. + // + // The method returns the number of values written, and any error that + // occurred while writing the values. + WriteFloats(values []float32) (int, error) +} + +// DoubleReader is an interface implemented by ValueReader instances which +// expose the content of a column of double-precision float point values. +type DoubleReader interface { + // Read double-precision floating point values into the buffer passed as + // argument. + // + // The method returns io.EOF when all values have been read. + ReadDoubles(values []float64) (int, error) +} + +// DoubleWriter is an interface implemented by ValueWriter instances which +// support writing columns of double-precision floating point values. +type DoubleWriter interface { + // Write double-precision floating point values. + // + // The method returns the number of values written, and any error that + // occurred while writing the values. + WriteDoubles(values []float64) (int, error) +} + +// ByteArrayReader is an interface implemented by ValueReader instances which +// expose the content of a column of variable length byte array values. +type ByteArrayReader interface { + // Read values into the byte buffer passed as argument, returning the number + // of values written to the buffer (not the number of bytes). Values are + // written using the PLAIN encoding, each byte array prefixed with its + // length encoded as a 4 bytes little endian unsigned integer. + // + // The method returns io.EOF when all values have been read. + // + // If the buffer was not empty, but too small to hold at least one value, + // io.ErrShortBuffer is returned. + ReadByteArrays(values []byte) (int, error) +} + +// ByteArrayWriter is an interface implemented by ValueWriter instances which +// support writing columns of variable length byte array values. +type ByteArrayWriter interface { + // Write variable length byte array values. + // + // The values passed as input must be laid out using the PLAIN encoding, + // with each byte array prefixed with the four bytes little endian unsigned + // integer length. + // + // The method returns the number of values written to the underlying column + // (not the number of bytes), or any error that occurred while attempting to + // write the values. + WriteByteArrays(values []byte) (int, error) +} + +// FixedLenByteArrayReader is an interface implemented by ValueReader instances +// which expose the content of a column of fixed length byte array values. +type FixedLenByteArrayReader interface { + // Read values into the byte buffer passed as argument, returning the number + // of values written to the buffer (not the number of bytes). + // + // The method returns io.EOF when all values have been read. + // + // If the buffer was not empty, but too small to hold at least one value, + // io.ErrShortBuffer is returned. + ReadFixedLenByteArrays(values []byte) (int, error) +} + +// FixedLenByteArrayWriter is an interface implemented by ValueWriter instances +// which support writing columns of fixed length byte array values. +type FixedLenByteArrayWriter interface { + // Writes the fixed length byte array values. + // + // The size of the values is assumed to be the same as the expected size of + // items in the column. The method errors if the length of the input values + // is not a multiple of the expected item size. + WriteFixedLenByteArrays(values []byte) (int, error) +} diff --git a/vendor/github.com/parquet-go/parquet-go/value_amd64.go b/vendor/github.com/parquet-go/parquet-go/value_amd64.go new file mode 100644 index 0000000000..fbd6432fd9 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/value_amd64.go @@ -0,0 +1,18 @@ +//go:build !purego + +package parquet + +import "golang.org/x/sys/cpu" + +//go:noescape +func memsetValuesAVX2(values []Value, model Value, _ uint64) + +func memsetValues(values []Value, model Value) { + if cpu.X86.HasAVX2 { + memsetValuesAVX2(values, model, 0) + } else { + for i := range values { + values[i] = model + } + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/value_amd64.s b/vendor/github.com/parquet-go/parquet-go/value_amd64.s new file mode 100644 index 0000000000..255117412f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/value_amd64.s @@ -0,0 +1,59 @@ +//go:build !purego + +#include "textflag.h" + +#define sizeOfValue 24 + +// This function is an optimized implementation of the memsetValues function +// which assigns the parquet.Value passed as second argument to all elements of +// the first slice argument. +// +// The optimizations relies on the fact that we can pack 4 parquet.Value values +// into 3 YMM registers (24 x 4 = 32 x 3 = 96). +// +// func memsetValuesAVX2(values []Value, model Value, _ uint64) +TEXT ยทmemsetValuesAVX2(SB), NOSPLIT, $0-56 // 48 + padding to load model in YMM + MOVQ values_base+0(FP), AX + MOVQ values_len+8(FP), BX + + MOVQ model_ptr+24(FP), R10 + MOVQ model_u64+32(FP), R11 + MOVQ model+40(FP), R12 // go vet complains about this line but it's OK + + XORQ SI, SI // byte index + MOVQ BX, DI // byte count + IMULQ $sizeOfValue, DI + + CMPQ BX, $4 + JB test + + MOVQ BX, R8 + SHRQ $2, R8 + SHLQ $2, R8 + IMULQ $sizeOfValue, R8 + + VMOVDQU model+24(FP), Y0 + VMOVDQU Y0, Y1 + VMOVDQU Y0, Y2 + + VPERMQ $0b00100100, Y0, Y0 + VPERMQ $0b01001001, Y1, Y1 + VPERMQ $0b10010010, Y2, Y2 +loop4: + VMOVDQU Y0, 0(AX)(SI*1) + VMOVDQU Y1, 32(AX)(SI*1) + VMOVDQU Y2, 64(AX)(SI*1) + ADDQ $4*sizeOfValue, SI + CMPQ SI, R8 + JNE loop4 + VZEROUPPER + JMP test +loop: + MOVQ R10, 0(AX)(SI*1) + MOVQ R11, 8(AX)(SI*1) + MOVQ R12, 16(AX)(SI*1) + ADDQ $sizeOfValue, SI +test: + CMPQ SI, DI + JNE loop + RET diff --git a/vendor/github.com/parquet-go/parquet-go/value_be.go b/vendor/github.com/parquet-go/parquet-go/value_be.go new file mode 100644 index 0000000000..5e7c038df8 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/value_be.go @@ -0,0 +1,19 @@ +//go:build s390x + +package parquet + +// On a big endian system, a boolean/byte value, which is in little endian byte +// format, is byte aligned to the 7th byte in a u64 (8 bytes) variable. +// Hence the data will be available at 7th byte when interpreted as a little +// endian byte format. So, in order to access a boolean/byte value out of u64 +// variable, we need to add an offset of "7". +// +// In the same way, an int32/uint32/float value, which is in little endian byte +// format, is byte aligned to the 4th byte in a u64 (8 bytes) variable. +// Hence the data will be available at 4th byte when interpreted as a little +// endian byte format. So, in order to access an int32/uint32/float value out of +// u64 variable, we need to add an offset of "4". +const ( + firstByteOffsetOfBooleanValue = 7 + firstByteOffsetOf32BitsValue = 4 +) diff --git a/vendor/github.com/parquet-go/parquet-go/value_le.go b/vendor/github.com/parquet-go/parquet-go/value_le.go new file mode 100644 index 0000000000..f11170ef73 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/value_le.go @@ -0,0 +1,8 @@ +//go:build !s390x + +package parquet + +const ( + firstByteOffsetOfBooleanValue = 0 + firstByteOffsetOf32BitsValue = 0 +) diff --git a/vendor/github.com/parquet-go/parquet-go/values_purego.go b/vendor/github.com/parquet-go/parquet-go/values_purego.go new file mode 100644 index 0000000000..8151134649 --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/values_purego.go @@ -0,0 +1,9 @@ +//go:build purego || !amd64 + +package parquet + +func memsetValues(values []Value, model Value) { + for i := range values { + values[i] = model + } +} diff --git a/vendor/github.com/parquet-go/parquet-go/writer.go b/vendor/github.com/parquet-go/parquet-go/writer.go new file mode 100644 index 0000000000..988fbd5e6f --- /dev/null +++ b/vendor/github.com/parquet-go/parquet-go/writer.go @@ -0,0 +1,1845 @@ +package parquet + +import ( + "bufio" + "bytes" + "cmp" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "math" + "math/bits" + "os" + "reflect" + "slices" + + "github.com/parquet-go/parquet-go/compress" + "github.com/parquet-go/parquet-go/encoding" + "github.com/parquet-go/parquet-go/encoding/plain" + "github.com/parquet-go/parquet-go/encoding/thrift" + "github.com/parquet-go/parquet-go/format" +) + +const ( + // The uncompressed page size is stored as int32 and must not be larger than the + // maximum int32 value (see format.PageHeader). + maxUncompressedPageSize = math.MaxInt32 +) + +// GenericWriter is similar to a Writer but uses a type parameter to define the +// Go type representing the schema of rows being written. +// +// Using this type over Writer has multiple advantages: +// +// - By leveraging type information, the Go compiler can provide greater +// guarantees that the code is correct. For example, the parquet.Writer.Write +// method accepts an argument of type interface{}, which delays type checking +// until runtime. The parquet.GenericWriter[T].Write method ensures at +// compile time that the values it receives will be of type T, reducing the +// risk of introducing errors. +// +// - Since type information is known at compile time, the implementation of +// parquet.GenericWriter[T] can make safe assumptions, removing the need for +// runtime validation of how the parameters are passed to its methods. +// Optimizations relying on type information are more effective, some of the +// writer's state can be precomputed at initialization, which was not possible +// with parquet.Writer. +// +// - The parquet.GenericWriter[T].Write method uses a data-oriented design, +// accepting an slice of T instead of a single value, creating more +// opportunities to amortize the runtime cost of abstractions. +// This optimization is not available for parquet.Writer because its Write +// method's argument would be of type []interface{}, which would require +// conversions back and forth from concrete types to empty interfaces (since +// a []T cannot be interpreted as []interface{} in Go), would make the API +// more difficult to use and waste compute resources in the type conversions, +// defeating the purpose of the optimization in the first place. +// +// Note that this type is only available when compiling with Go 1.18 or later. +type GenericWriter[T any] struct { + // At this time GenericWriter is expressed in terms of Writer to reuse the + // underlying logic. In the future, and if we accepted to break backward + // compatibility on the Write method, we could modify Writer to be an alias + // to GenericWriter with: + // + // type Writer = GenericWriter[any] + // + base Writer + // This function writes rows of type T to the writer, it gets generated by + // the NewGenericWriter function based on the type T and the underlying + // schema of the parquet file. + write writeFunc[T] + // This field is used to leverage the optimized writeRowsFunc algorithms. + columns []ColumnBuffer +} + +// NewGenericWriter is like NewWriter but returns a GenericWriter[T] suited to +// write rows of Go type T. +// +// The type parameter T should be a map, struct, or any. Any other types will +// cause a panic at runtime. Type checking is a lot more effective when the +// generic parameter is a struct type, using map and interface types is somewhat +// similar to using a Writer. +// +// If the option list may explicitly declare a schema, it must be compatible +// with the schema generated from T. +// +// Sorting columns may be set on the writer to configure the generated row +// groups metadata. However, rows are always written in the order they were +// seen, no reordering is performed, the writer expects the application to +// ensure proper correlation between the order of rows and the list of sorting +// columns. See SortingWriter[T] for a writer which handles reordering rows +// based on the configured sorting columns. +func NewGenericWriter[T any](output io.Writer, options ...WriterOption) *GenericWriter[T] { + config, err := NewWriterConfig(options...) + if err != nil { + panic(err) + } + + schema := config.Schema + t := typeOf[T]() + + var genWriteErr error + if schema == nil && t != nil { + schema = schemaOf(dereference(t)) + if len(schema.Columns()) == 0 { + genWriteErr = fmt.Errorf("cannot write %v: it has no columns (maybe it has no exported fields)", t) + } + config.Schema = schema + } else if schema != nil && len(schema.Columns()) == 0 { + genWriteErr = fmt.Errorf("cannot write %v: schema has no columns", t) + } + + if config.Schema == nil { + panic("generic writer must be instantiated with schema or concrete type.") + } + + var writeFn writeFunc[T] + if genWriteErr != nil { + writeFn = func(*GenericWriter[T], []T) (int, error) { return 0, genWriteErr } + } else { + writeFn = writeFuncOf[T](t, config.Schema) + } + + return &GenericWriter[T]{ + base: Writer{ + output: output, + config: config, + schema: schema, + writer: newWriter(output, config), + }, + write: writeFn, + } +} + +type writeFunc[T any] func(*GenericWriter[T], []T) (int, error) + +func writeFuncOf[T any](t reflect.Type, schema *Schema) writeFunc[T] { + if t == nil { + return (*GenericWriter[T]).writeAny + } + switch t.Kind() { + case reflect.Interface, reflect.Map: + return (*GenericWriter[T]).writeRows + + case reflect.Struct: + return makeWriteFunc[T](t, schema) + + case reflect.Pointer: + if e := t.Elem(); e.Kind() == reflect.Struct { + return makeWriteFunc[T](t, schema) + } + } + panic("cannot create writer for values of type " + t.String()) +} + +func makeWriteFunc[T any](t reflect.Type, schema *Schema) writeFunc[T] { + writeRows := writeRowsFuncOf(t, schema, nil) + return func(w *GenericWriter[T], rows []T) (n int, err error) { + if w.columns == nil { + w.columns = make([]ColumnBuffer, len(w.base.writer.columns)) + for i, c := range w.base.writer.columns { + // These fields are usually lazily initialized when writing rows, + // we need them to exist now tho. + c.columnBuffer = c.newColumnBuffer() + w.columns[i] = c.columnBuffer + } + } + err = writeRows(w.columns, makeArrayOf(rows), columnLevels{}) + if err == nil { + n = len(rows) + } + return n, err + } +} + +func (w *GenericWriter[T]) Close() error { + return w.base.Close() +} + +func (w *GenericWriter[T]) Flush() error { + return w.base.Flush() +} + +func (w *GenericWriter[T]) Reset(output io.Writer) { + w.base.Reset(output) +} + +func (w *GenericWriter[T]) Write(rows []T) (int, error) { + return w.base.writer.writeRows(len(rows), func(i, j int) (int, error) { + n, err := w.write(w, rows[i:j:j]) + if err != nil { + return n, err + } + + for _, c := range w.base.writer.columns { + if c.columnBuffer.Size() >= int64(c.bufferSize) { + if err := c.flush(); err != nil { + return n, err + } + } + } + + return n, nil + }) +} + +func (w *GenericWriter[T]) WriteRows(rows []Row) (int, error) { + return w.base.WriteRows(rows) +} + +func (w *GenericWriter[T]) WriteRowGroup(rowGroup RowGroup) (int64, error) { + return w.base.WriteRowGroup(rowGroup) +} + +// SetKeyValueMetadata sets a key/value pair in the Parquet file metadata. +// +// Keys are assumed to be unique, if the same key is repeated multiple times the +// last value is retained. While the parquet format does not require unique keys, +// this design decision was made to optimize for the most common use case where +// applications leverage this extension mechanism to associate single values to +// keys. This may create incompatibilities with other parquet libraries, or may +// cause some key/value pairs to be lost when open parquet files written with +// repeated keys. We can revisit this decision if it ever becomes a blocker. +func (w *GenericWriter[T]) SetKeyValueMetadata(key, value string) { + w.base.SetKeyValueMetadata(key, value) +} + +func (w *GenericWriter[T]) ReadRowsFrom(rows RowReader) (int64, error) { + return w.base.ReadRowsFrom(rows) +} + +func (w *GenericWriter[T]) Schema() *Schema { + return w.base.Schema() +} + +func (w *GenericWriter[T]) ColumnWriters() []*ColumnWriter { + return w.base.ColumnWriters() +} + +func (w *GenericWriter[T]) writeRows(rows []T) (int, error) { + if cap(w.base.rowbuf) < len(rows) { + w.base.rowbuf = make([]Row, len(rows)) + } else { + w.base.rowbuf = w.base.rowbuf[:len(rows)] + } + defer clearRows(w.base.rowbuf) + + schema := w.base.Schema() + for i := range rows { + w.base.rowbuf[i] = schema.Deconstruct(w.base.rowbuf[i], &rows[i]) + } + + return w.base.WriteRows(w.base.rowbuf) +} + +func (w *GenericWriter[T]) writeAny(rows []T) (n int, err error) { + for i := range rows { + if err = w.base.Write(rows[i]); err != nil { + return n, err + } + n++ + } + return n, nil +} + +// File returns a FileView of the written parquet file. +// Only available after Close is called. +func (w *GenericWriter[T]) File() FileView { + return w.base.File() +} + +var ( + _ RowWriterWithSchema = (*GenericWriter[any])(nil) + _ RowReaderFrom = (*GenericWriter[any])(nil) + _ RowGroupWriter = (*GenericWriter[any])(nil) + + _ RowWriterWithSchema = (*GenericWriter[struct{}])(nil) + _ RowReaderFrom = (*GenericWriter[struct{}])(nil) + _ RowGroupWriter = (*GenericWriter[struct{}])(nil) + + _ RowWriterWithSchema = (*GenericWriter[map[struct{}]struct{}])(nil) + _ RowReaderFrom = (*GenericWriter[map[struct{}]struct{}])(nil) + _ RowGroupWriter = (*GenericWriter[map[struct{}]struct{}])(nil) +) + +// Deprecated: A Writer uses a parquet schema and sequence of Go values to +// produce a parquet file to an io.Writer. +// +// This example showcases a typical use of parquet writers: +// +// writer := parquet.NewWriter(output) +// +// for _, row := range rows { +// if err := writer.Write(row); err != nil { +// ... +// } +// } +// +// if err := writer.Close(); err != nil { +// ... +// } +// +// The Writer type optimizes for minimal memory usage, each page is written as +// soon as it has been filled so only a single page per column needs to be held +// in memory and as a result, there are no opportunities to sort rows within an +// entire row group. Programs that need to produce parquet files with sorted +// row groups should use the Buffer type to buffer and sort the rows prior to +// writing them to a Writer. +// +// For programs building with Go 1.18 or later, the GenericWriter[T] type +// supersedes this one. +type Writer struct { + output io.Writer + config *WriterConfig + schema *Schema + writer *writer + rowbuf []Row +} + +// NewWriter constructs a parquet writer writing a file to the given io.Writer. +// +// The function panics if the writer configuration is invalid. Programs that +// cannot guarantee the validity of the options passed to NewWriter should +// construct the writer configuration independently prior to calling this +// function: +// +// config, err := parquet.NewWriterConfig(options...) +// if err != nil { +// // handle the configuration error +// ... +// } else { +// // this call to create a writer is guaranteed not to panic +// writer := parquet.NewWriter(output, config) +// ... +// } +func NewWriter(output io.Writer, options ...WriterOption) *Writer { + config, err := NewWriterConfig(options...) + if err != nil { + panic(err) + } + w := &Writer{ + output: output, + config: config, + } + if config.Schema != nil { + w.configure(config.Schema) + } + return w +} + +func (w *Writer) configure(schema *Schema) { + if schema != nil { + w.config.Schema = schema + w.schema = schema + w.writer = newWriter(w.output, w.config) + } +} + +// Close must be called after all values were produced to the writer in order to +// flush all buffers and write the parquet footer. +func (w *Writer) Close() error { + if w.writer != nil { + return w.writer.close() + } + return nil +} + +// Flush flushes all buffers into a row group to the underlying io.Writer. +// +// Flush is called automatically on Close, it is only useful to call explicitly +// if the application needs to limit the size of row groups or wants to produce +// multiple row groups per file. +// +// If the writer attempts to create more than MaxRowGroups row groups the method +// returns ErrTooManyRowGroups. +func (w *Writer) Flush() error { + if w.writer != nil { + return w.writer.flush() + } + return nil +} + +// Reset clears the state of the writer without flushing any of the buffers, +// and setting the output to the io.Writer passed as argument, allowing the +// writer to be reused to produce another parquet file. +// +// Reset may be called at any time, including after a writer was closed. +func (w *Writer) Reset(output io.Writer) { + if w.output = output; w.writer != nil { + w.writer.reset(w.output) + } +} + +// Write is called to write another row to the parquet file. +// +// The method uses the parquet schema configured on w to traverse the Go value +// and decompose it into a set of columns and values. If no schema were passed +// to NewWriter, it is deducted from the Go type of the row, which then have to +// be a struct or pointer to struct. +func (w *Writer) Write(row any) error { + if w.schema == nil { + w.configure(SchemaOf(row)) + } + if cap(w.rowbuf) == 0 { + w.rowbuf = make([]Row, 1) + } else { + w.rowbuf = w.rowbuf[:1] + } + defer clearRows(w.rowbuf) + w.rowbuf[0] = w.schema.Deconstruct(w.rowbuf[0][:0], row) + _, err := w.WriteRows(w.rowbuf) + return err +} + +// WriteRows is called to write rows to the parquet file. +// +// The Writer must have been given a schema when NewWriter was called, otherwise +// the structure of the parquet file cannot be determined from the row only. +// +// The row is expected to contain values for each column of the writer's schema, +// in the order produced by the parquet.(*Schema).Deconstruct method. +func (w *Writer) WriteRows(rows []Row) (int, error) { + return w.writer.WriteRows(rows) +} + +// WriteRowGroup writes a row group to the parquet file. +// +// Buffered rows will be flushed prior to writing rows from the group, unless +// the row group was empty in which case nothing is written to the file. +// +// The content of the row group is flushed to the writer; after the method +// returns successfully, the row group will be empty and in ready to be reused. +func (w *Writer) WriteRowGroup(rowGroup RowGroup) (int64, error) { + rowGroupSchema := rowGroup.Schema() + switch { + case rowGroupSchema == nil: + return 0, ErrRowGroupSchemaMissing + case w.schema == nil: + w.configure(rowGroupSchema) + case !EqualNodes(w.schema, rowGroupSchema): + return 0, ErrRowGroupSchemaMismatch + } + if err := w.writer.flush(); err != nil { + return 0, err + } + w.writer.configureBloomFilters(rowGroup.ColumnChunks()) + rows := rowGroup.Rows() + defer rows.Close() + n, err := CopyRows(w.writer, rows) + if err != nil { + return n, err + } + return w.writer.writeRowGroup(rowGroup.Schema(), rowGroup.SortingColumns()) +} + +// ReadRowsFrom reads rows from the reader passed as arguments and writes them +// to w. +// +// This is similar to calling WriteRow repeatedly, but will be more efficient +// if optimizations are supported by the reader. +func (w *Writer) ReadRowsFrom(rows RowReader) (written int64, err error) { + if w.schema == nil { + if r, ok := rows.(RowReaderWithSchema); ok { + w.configure(r.Schema()) + } + } + if cap(w.rowbuf) < defaultRowBufferSize { + w.rowbuf = make([]Row, defaultRowBufferSize) + } else { + w.rowbuf = w.rowbuf[:cap(w.rowbuf)] + } + return copyRows(w.writer, rows, w.rowbuf) +} + +// Schema returns the schema of rows written by w. +// +// The returned value will be nil if no schema has yet been configured on w. +func (w *Writer) Schema() *Schema { return w.schema } + +// SetKeyValueMetadata sets a key/value pair in the Parquet file metadata. +// +// Keys are assumed to be unique, if the same key is repeated multiple times the +// last value is retained. While the parquet format does not require unique keys, +// this design decision was made to optimize for the most common use case where +// applications leverage this extension mechanism to associate single values to +// keys. This may create incompatibilities with other parquet libraries, or may +// cause some key/value pairs to be lost when open parquet files written with +// repeated keys. We can revisit this decision if it ever becomes a blocker. +func (w *Writer) SetKeyValueMetadata(key, value string) { + for i, kv := range w.writer.metadata { + if kv.Key == key { + kv.Value = value + w.writer.metadata[i] = kv + return + } + } + w.writer.metadata = append(w.writer.metadata, format.KeyValue{ + Key: key, + Value: value, + }) +} + +// ColumnWriters returns writers for each column. This allows applications to +// write values directly to each column instead of having to first assemble +// values into rows to use WriteRows. +func (w *Writer) ColumnWriters() []*ColumnWriter { return w.writer.columns } + +type writerFileView struct { + writer *writer + schema *Schema +} + +// File returns a FileView of the written parquet file. +// Only available after Close is called. +func (w *Writer) File() FileView { + if w.writer == nil || w.schema == nil { + return nil + } + return &writerFileView{ + w.writer, + w.schema, + } +} + +func (w *writerFileView) Metadata() *format.FileMetaData { + return w.writer.fileMetaData +} + +func (w *writerFileView) Schema() *Schema { + return w.schema +} + +func (w *writerFileView) NumRows() int64 { + if w.writer.fileMetaData != nil { + return w.writer.fileMetaData.NumRows + } + return 0 +} + +func (w *writerFileView) Lookup(key string) (string, bool) { + if w.writer.fileMetaData != nil { + return lookupKeyValueMetadata(w.writer.fileMetaData.KeyValueMetadata, key) + } + return "", false +} + +func (w *writerFileView) Size() int64 { + return w.writer.writer.offset +} + +func (w *writerFileView) ColumnIndexes() []format.ColumnIndex { + return w.writer.columnIndex +} + +func (w *writerFileView) OffsetIndexes() []format.OffsetIndex { + return w.writer.offsetIndex +} + +func (w *writerFileView) Root() *Column { + if w.writer.fileMetaData != nil { + root, _ := openColumns(nil, w.writer.fileMetaData, w.writer.columnIndex, w.writer.offsetIndex) + return root + } + return nil +} + +func (w *writerFileView) RowGroups() []RowGroup { + if w.writer.fileMetaData != nil { + columns := makeLeafColumns(w.Root()) + file := &File{metadata: *w.writer.fileMetaData, schema: w.schema} + fileRowGroups := makeFileRowGroups(file, columns) + return makeRowGroups(fileRowGroups) + } + return nil +} + +type writer struct { + buffer *bufio.Writer + writer offsetTrackingWriter + values [][]Value + numRows int64 + maxRows int64 + + createdBy string + metadata []format.KeyValue + + columns []*ColumnWriter + columnChunk []format.ColumnChunk + columnIndex []format.ColumnIndex + offsetIndex []format.OffsetIndex + + columnOrders []format.ColumnOrder + schemaElements []format.SchemaElement + rowGroups []format.RowGroup + columnIndexes [][]format.ColumnIndex + offsetIndexes [][]format.OffsetIndex + sortingColumns []format.SortingColumn + + fileMetaData *format.FileMetaData +} + +func newWriter(output io.Writer, config *WriterConfig) *writer { + w := new(writer) + if config.WriteBufferSize <= 0 { + w.writer.Reset(output) + } else { + w.buffer = bufio.NewWriterSize(output, config.WriteBufferSize) + w.writer.Reset(w.buffer) + } + w.maxRows = config.MaxRowsPerRowGroup + w.createdBy = config.CreatedBy + w.metadata = make([]format.KeyValue, 0, len(config.KeyValueMetadata)) + for k, v := range config.KeyValueMetadata { + w.metadata = append(w.metadata, format.KeyValue{Key: k, Value: v}) + } + sortKeyValueMetadata(w.metadata) + w.sortingColumns = make([]format.SortingColumn, len(config.Sorting.SortingColumns)) + + config.Schema.forEachNode(func(name string, node Node) { + nodeType := node.Type() + + repetitionType := (*format.FieldRepetitionType)(nil) + if node != config.Schema { // the root has no repetition type + repetitionType = fieldRepetitionTypePtrOf(node) + } + // For backward compatibility with older readers, the parquet specification + // recommends to set the scale and precision on schema elements when the + // column is of logical type decimal. + logicalType := nodeType.LogicalType() + scale, precision := (*int32)(nil), (*int32)(nil) + if logicalType != nil && logicalType.Decimal != nil { + scale = &logicalType.Decimal.Scale + precision = &logicalType.Decimal.Precision + } + + typeLength := (*int32)(nil) + if n := int32(nodeType.Length()); n > 0 { + typeLength = &n + } + + w.schemaElements = append(w.schemaElements, format.SchemaElement{ + Type: nodeType.PhysicalType(), + TypeLength: typeLength, + RepetitionType: repetitionType, + Name: name, + NumChildren: int32(len(node.Fields())), + ConvertedType: nodeType.ConvertedType(), + Scale: scale, + Precision: precision, + FieldID: int32(node.ID()), + LogicalType: logicalType, + }) + }) + + dataPageType := format.DataPage + if config.DataPageVersion == 2 { + dataPageType = format.DataPageV2 + } + + defaultCompression := config.Compression + if defaultCompression == nil { + defaultCompression = &Uncompressed + } + + // Those buffers are scratch space used to generate the page header and + // content, they are shared by all column chunks because they are only + // used during calls to writeDictionaryPage or writeDataPage, which are + // not done concurrently. + buffers := new(writerBuffers) + + forEachLeafColumnOf(config.Schema, func(leaf leafColumn) { + encoding := encodingOf(leaf.node) + dictionary := Dictionary(nil) + columnType := leaf.node.Type() + columnIndex := int(leaf.columnIndex) + compression := leaf.node.Compression() + + if compression == nil { + compression = defaultCompression + } + + if isDictionaryEncoding(encoding) { + dictBuffer := columnType.NewValues( + make([]byte, 0, defaultDictBufferSize), + nil, + ) + dictionary = columnType.NewDictionary(columnIndex, 0, dictBuffer) + columnType = dictionary.Type() + } + + c := &ColumnWriter{ + buffers: buffers, + pool: config.ColumnPageBuffers, + columnPath: leaf.path, + columnType: columnType, + columnIndex: columnType.NewColumnIndexer(config.ColumnIndexSizeLimit), + columnFilter: searchBloomFilterColumn(config.BloomFilters, leaf.path), + compression: compression, + dictionary: dictionary, + dataPageType: dataPageType, + maxRepetitionLevel: leaf.maxRepetitionLevel, + maxDefinitionLevel: leaf.maxDefinitionLevel, + bufferIndex: int32(leaf.columnIndex), + bufferSize: int32(float64(config.PageBufferSize) * 0.98), + writePageStats: config.DataPageStatistics, + writePageBounds: !slices.ContainsFunc(config.SkipPageBounds, func(skip []string) bool { + return columnPath(skip).equal(leaf.path) + }), + encodings: make([]format.Encoding, 0, 3), + // Data pages in version 2 can omit compression when dictionary + // encoding is employed; only the dictionary page needs to be + // compressed, the data pages are encoded with the hybrid + // RLE/Bit-Pack encoding which doesn't benefit from an extra + // compression layer. + isCompressed: isCompressed(compression) && (dataPageType != format.DataPageV2 || dictionary == nil), + } + + c.header.encoder.Reset(c.header.protocol.NewWriter(&buffers.header)) + + if leaf.maxDefinitionLevel > 0 { + c.encodings = addEncoding(c.encodings, format.RLE) + } + + if isDictionaryEncoding(encoding) { + c.encodings = addEncoding(c.encodings, format.Plain) + } + + c.encoding = encoding + c.encodings = addEncoding(c.encodings, c.encoding.Encoding()) + sortPageEncodings(c.encodings) + + w.columns = append(w.columns, c) + + if sortingIndex := searchSortingColumn(config.Sorting.SortingColumns, leaf.path); sortingIndex < len(w.sortingColumns) { + w.sortingColumns[sortingIndex] = format.SortingColumn{ + ColumnIdx: int32(leaf.columnIndex), + Descending: config.Sorting.SortingColumns[sortingIndex].Descending(), + NullsFirst: config.Sorting.SortingColumns[sortingIndex].NullsFirst(), + } + } + }) + + // Pre-allocate the backing array so that in most cases where the rows + // contain a single value we will hit collocated memory areas when writing + // rows to the writer. This won't benefit repeated columns much but in that + // case we would just waste a bit of memory which we can afford. + values := make([]Value, len(w.columns)) + w.values = make([][]Value, len(w.columns)) + for i := range values { + w.values[i] = values[i : i : i+1] + } + + w.columnChunk = make([]format.ColumnChunk, len(w.columns)) + w.columnIndex = make([]format.ColumnIndex, len(w.columns)) + w.offsetIndex = make([]format.OffsetIndex, len(w.columns)) + w.columnOrders = make([]format.ColumnOrder, len(w.columns)) + + for i, c := range w.columns { + w.columnChunk[i] = format.ColumnChunk{ + MetaData: format.ColumnMetaData{ + Type: format.Type(c.columnType.Kind()), + Encoding: c.encodings, + PathInSchema: c.columnPath, + Codec: c.compression.CompressionCodec(), + KeyValueMetadata: nil, // TODO + }, + } + } + + for i, c := range w.columns { + c.columnChunk = &w.columnChunk[i] + c.offsetIndex = &w.offsetIndex[i] + } + + for i, c := range w.columns { + w.columnOrders[i] = *c.columnType.ColumnOrder() + } + + return w +} + +func (w *writer) reset(writer io.Writer) { + if w.buffer == nil { + w.writer.Reset(writer) + } else { + w.buffer.Reset(writer) + w.writer.Reset(w.buffer) + } + for _, c := range w.columns { + c.reset() + } + for i := range w.rowGroups { + w.rowGroups[i] = format.RowGroup{} + } + for i := range w.columnIndexes { + w.columnIndexes[i] = nil + } + for i := range w.offsetIndexes { + w.offsetIndexes[i] = nil + } + w.rowGroups = w.rowGroups[:0] + w.columnIndexes = w.columnIndexes[:0] + w.offsetIndexes = w.offsetIndexes[:0] + w.fileMetaData = nil +} + +func (w *writer) close() error { + if err := w.writeFileHeader(); err != nil { + return err + } + if err := w.flush(); err != nil { + return err + } + if err := w.writeFileFooter(); err != nil { + return err + } + if w.buffer != nil { + return w.buffer.Flush() + } + return nil +} + +func (w *writer) flush() error { + _, err := w.writeRowGroup(nil, nil) + return err +} + +func (w *writer) writeFileHeader() error { + if w.writer.writer == nil { + return io.ErrClosedPipe + } + if w.writer.offset == 0 { + _, err := w.writer.WriteString("PAR1") + return err + } + return nil +} + +func (w *writer) configureBloomFilters(columnChunks []ColumnChunk) { + for i, c := range w.columns { + if c.columnFilter != nil { + c.resizeBloomFilter(columnChunks[i].NumValues()) + } + } +} + +func (w *writer) writeFileFooter() error { + // The page index is composed of two sections: column and offset indexes. + // They are written after the row groups, right before the footer (which + // is written by the parent Writer.Close call). + // + // This section both writes the page index and generates the values of + // ColumnIndexOffset, ColumnIndexLength, OffsetIndexOffset, and + // OffsetIndexLength in the corresponding columns of the file metadata. + // + // Note: the page index is always written, even if we created data pages v1 + // because the parquet format is backward compatible in this case. Older + // readers will simply ignore this section since they do not know how to + // decode its content, nor have loaded any metadata to reference it. + protocol := new(thrift.CompactProtocol) + encoder := thrift.NewEncoder(protocol.NewWriter(&w.writer)) + + w.columnIndex = w.columnIndex[:0] + for i, columnIndexes := range w.columnIndexes { + rowGroup := &w.rowGroups[i] + for j := range columnIndexes { + column := &rowGroup.Columns[j] + column.ColumnIndexOffset = w.writer.offset + if err := encoder.Encode(&columnIndexes[j]); err != nil { + return err + } + column.ColumnIndexLength = int32(w.writer.offset - column.ColumnIndexOffset) + } + w.columnIndex = append(w.columnIndex, columnIndexes...) + } + + w.offsetIndex = w.offsetIndex[:0] + for i, offsetIndexes := range w.offsetIndexes { + rowGroup := &w.rowGroups[i] + for j := range offsetIndexes { + column := &rowGroup.Columns[j] + column.OffsetIndexOffset = w.writer.offset + if err := encoder.Encode(&offsetIndexes[j]); err != nil { + return err + } + column.OffsetIndexLength = int32(w.writer.offset - column.OffsetIndexOffset) + } + w.offsetIndex = append(w.offsetIndex, offsetIndexes...) + } + + numRows := int64(0) + for rowGroupIndex := range w.rowGroups { + numRows += w.rowGroups[rowGroupIndex].NumRows + } + + // We implemented the parquet specification version 2+, which is represented + // by the version number 2 in the file metadata. + // + // For reference, see: + // https://github.com/apache/arrow/blob/70b9ef5/go/parquet/metadata/file.go#L122-L127 + const parquetFileFormatVersion = 2 + + w.fileMetaData = &format.FileMetaData{ + Version: parquetFileFormatVersion, + Schema: w.schemaElements, + NumRows: numRows, + RowGroups: w.rowGroups, + KeyValueMetadata: w.metadata, + CreatedBy: w.createdBy, + ColumnOrders: w.columnOrders, + } + footer, err := thrift.Marshal(new(thrift.CompactProtocol), w.fileMetaData) + if err != nil { + return err + } + + length := len(footer) + footer = append(footer, 0, 0, 0, 0) + footer = append(footer, "PAR1"...) + binary.LittleEndian.PutUint32(footer[length:], uint32(length)) + + _, err = w.writer.Write(footer) + return err +} + +func (w *writer) writeRowGroup(rowGroupSchema *Schema, rowGroupSortingColumns []SortingColumn) (int64, error) { + if len(w.columns) == 0 { + return 0, nil + } + numRows := w.columns[0].totalRowCount() + if numRows == 0 { + return 0, nil + } + + if len(w.rowGroups) == MaxRowGroups { + return 0, ErrTooManyRowGroups + } + + defer func() { + w.numRows = 0 + for _, c := range w.columns { + c.reset() + } + for i := range w.columnIndex { + w.columnIndex[i] = format.ColumnIndex{} + } + }() + + for _, c := range w.columns { + if err := c.flush(); err != nil { + return 0, err + } + if err := c.flushFilterPages(); err != nil { + return 0, err + } + } + + if err := w.writeFileHeader(); err != nil { + return 0, err + } + fileOffset := w.writer.offset + + for i, c := range w.columns { + w.columnIndex[i] = format.ColumnIndex(c.columnIndex.ColumnIndex()) + + if c.dictionary != nil { + c.columnChunk.MetaData.DictionaryPageOffset = w.writer.offset + if err := c.writeDictionaryPage(&w.writer, c.dictionary); err != nil { + return 0, fmt.Errorf("writing dictionary page of row group colum %d: %w", i, err) + } + } + + dataPageOffset := w.writer.offset + c.columnChunk.MetaData.DataPageOffset = dataPageOffset + for j := range c.offsetIndex.PageLocations { + c.offsetIndex.PageLocations[j].Offset += dataPageOffset + } + + if offset, err := c.pageBuffer.Seek(0, io.SeekStart); err != nil { + return 0, err + } else if offset != 0 { + return 0, fmt.Errorf("resetting parquet page buffer to the start expected offset zero but got %d", offset) + } + if _, err := io.Copy(&w.writer, c.pageBuffer); err != nil { + return 0, fmt.Errorf("writing buffered pages of row group column %d: %w", i, err) + } + } + + for _, c := range w.columns { + if len(c.filter) > 0 { + c.columnChunk.MetaData.BloomFilterOffset = w.writer.offset + if err := c.writeBloomFilter(&w.writer); err != nil { + return 0, err + } + } + } + + totalByteSize := int64(0) + totalCompressedSize := int64(0) + + for i := range w.columnChunk { + c := &w.columnChunk[i].MetaData + sortPageEncodingStats(c.EncodingStats) + totalByteSize += int64(c.TotalUncompressedSize) + totalCompressedSize += int64(c.TotalCompressedSize) + } + + sortingColumns := w.sortingColumns + if len(sortingColumns) == 0 && len(rowGroupSortingColumns) > 0 { + sortingColumns = make([]format.SortingColumn, 0, len(rowGroupSortingColumns)) + forEachLeafColumnOf(rowGroupSchema, func(leaf leafColumn) { + if sortingIndex := searchSortingColumn(rowGroupSortingColumns, leaf.path); sortingIndex < len(sortingColumns) { + sortingColumns[sortingIndex] = format.SortingColumn{ + ColumnIdx: int32(leaf.columnIndex), + Descending: rowGroupSortingColumns[sortingIndex].Descending(), + NullsFirst: rowGroupSortingColumns[sortingIndex].NullsFirst(), + } + } + }) + } + + columns := make([]format.ColumnChunk, len(w.columnChunk)) + copy(columns, w.columnChunk) + + columnIndex := make([]format.ColumnIndex, len(w.columnIndex)) + copy(columnIndex, w.columnIndex) + + offsetIndex := make([]format.OffsetIndex, len(w.offsetIndex)) + copy(offsetIndex, w.offsetIndex) + + for i := range columns { + c := &columns[i] + c.MetaData.EncodingStats = make([]format.PageEncodingStats, len(c.MetaData.EncodingStats)) + copy(c.MetaData.EncodingStats, w.columnChunk[i].MetaData.EncodingStats) + } + + for i := range offsetIndex { + c := &offsetIndex[i] + c.PageLocations = make([]format.PageLocation, len(c.PageLocations)) + copy(c.PageLocations, w.offsetIndex[i].PageLocations) + } + + w.rowGroups = append(w.rowGroups, format.RowGroup{ + Columns: columns, + TotalByteSize: totalByteSize, + NumRows: numRows, + SortingColumns: sortingColumns, + FileOffset: fileOffset, + TotalCompressedSize: totalCompressedSize, + Ordinal: int16(len(w.rowGroups)), + }) + + w.columnIndexes = append(w.columnIndexes, columnIndex) + w.offsetIndexes = append(w.offsetIndexes, offsetIndex) + return numRows, nil +} + +func (w *writer) WriteRows(rows []Row) (int, error) { + return w.writeRows(len(rows), func(start, end int) (int, error) { + defer func() { + for i, values := range w.values { + clearValues(values) + w.values[i] = values[:0] + } + }() + + // TODO: if an error occurs in this method the writer may be left in an + // partially functional state. Applications are not expected to continue + // using the writer after getting an error, but maybe we could ensure that + // we are preventing further use as well? + for _, row := range rows[start:end] { + row.Range(func(columnIndex int, columnValues []Value) bool { + w.values[columnIndex] = append(w.values[columnIndex], columnValues...) + return true + }) + } + + for i, values := range w.values { + if len(values) > 0 { + if _, err := w.columns[i].WriteRowValues(values); err != nil { + return 0, err + } + } + } + + return end - start, nil + }) +} + +func (w *writer) writeRows(numRows int, write func(i, j int) (int, error)) (int, error) { + written := 0 + + for written < numRows { + remain := w.maxRows - w.numRows + length := numRows - written + + if remain == 0 { + remain = w.maxRows + + if err := w.flush(); err != nil { + return written, err + } + } + + if remain < int64(length) { + length = int(remain) + } + + // Since the writer cannot flush pages across row boundaries, calls to + // WriteRows with very large slices can result in greatly exceeding the + // target page size. To set a limit to the impact of these large writes + // we chunk the input in slices of 64 rows. + // + // Note that this mechanism isn't perfect; for example, values may hold + // large byte slices which could still cause the column buffers to grow + // beyond the target page size. + const maxRowsPerWrite = 64 + if length > maxRowsPerWrite { + length = maxRowsPerWrite + } + + n, err := write(written, written+length) + written += n + w.numRows += int64(n) + if err != nil { + return written, err + } + } + + return written, nil +} + +// The WriteValues method is intended to work in pair with WritePage to allow +// programs to target writing values to specific columns of of the writer. +func (w *writer) WriteValues(values []Value) (numValues int, err error) { + return w.columns[values[0].Column()].writeValues(values) +} + +// One writerBuffers is used by each writer instance, the memory buffers here +// are shared by all columns of the writer because serialization is not done +// concurrently, which helps keep memory utilization low, both in the total +// footprint and GC cost. +// +// The type also exposes helper methods to facilitate the generation of parquet +// pages. A scratch space is used when serialization requires combining multiple +// buffers or compressing the page data, with double-buffering technique being +// employed by swapping the scratch and page buffers to minimize memory copies. +type writerBuffers struct { + header bytes.Buffer // buffer where page headers are encoded + repetitions []byte // buffer used to encode repetition levels + definitions []byte // buffer used to encode definition levels + page []byte // page buffer holding the page data + scratch []byte // scratch space used for compression +} + +func (wb *writerBuffers) crc32() (checksum uint32) { + checksum = crc32.Update(checksum, crc32.IEEETable, wb.repetitions) + checksum = crc32.Update(checksum, crc32.IEEETable, wb.definitions) + checksum = crc32.Update(checksum, crc32.IEEETable, wb.page) + return checksum +} + +func (wb *writerBuffers) size() int { + return len(wb.repetitions) + len(wb.definitions) + len(wb.page) +} + +func (wb *writerBuffers) reset() { + wb.repetitions = wb.repetitions[:0] + wb.definitions = wb.definitions[:0] + wb.page = wb.page[:0] +} + +func encodeLevels(dst, src []byte, maxLevel byte) ([]byte, error) { + bitWidth := bits.Len8(maxLevel) + return levelEncodingsRLE[bitWidth-1].EncodeLevels(dst, src) +} + +func (wb *writerBuffers) encodeRepetitionLevels(page Page, maxRepetitionLevel byte) (err error) { + wb.repetitions, err = encodeLevels(wb.repetitions, page.RepetitionLevels(), maxRepetitionLevel) + return +} + +func (wb *writerBuffers) encodeDefinitionLevels(page Page, maxDefinitionLevel byte) (err error) { + wb.definitions, err = encodeLevels(wb.definitions, page.DefinitionLevels(), maxDefinitionLevel) + return +} + +func (wb *writerBuffers) prependLevelsToDataPageV1(maxRepetitionLevel, maxDefinitionLevel byte) { + hasRepetitionLevels := maxRepetitionLevel > 0 + hasDefinitionLevels := maxDefinitionLevel > 0 + + if hasRepetitionLevels || hasDefinitionLevels { + wb.scratch = wb.scratch[:0] + // In data pages v1, the repetition and definition levels are prefixed + // with the 4 bytes length of the sections. While the parquet-format + // documentation indicates that the length prefix is part of the hybrid + // RLE/Bit-Pack encoding, this is the only condition where it is used + // so we treat it as a special case rather than implementing it in the + // encoding. + // + // Reference https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 + if hasRepetitionLevels { + wb.scratch = plain.AppendInt32(wb.scratch, int32(len(wb.repetitions))) + wb.scratch = append(wb.scratch, wb.repetitions...) + wb.repetitions = wb.repetitions[:0] + } + if hasDefinitionLevels { + wb.scratch = plain.AppendInt32(wb.scratch, int32(len(wb.definitions))) + wb.scratch = append(wb.scratch, wb.definitions...) + wb.definitions = wb.definitions[:0] + } + wb.scratch = append(wb.scratch, wb.page...) + wb.swapPageAndScratchBuffers() + } +} + +func (wb *writerBuffers) encode(page Page, enc encoding.Encoding) (err error) { + pageType := page.Type() + pageData := page.Data() + wb.page, err = pageType.Encode(wb.page[:0], pageData, enc) + return err +} + +func (wb *writerBuffers) compress(codec compress.Codec) (err error) { + wb.scratch, err = codec.Encode(wb.scratch[:0], wb.page) + wb.swapPageAndScratchBuffers() + return err +} + +func (wb *writerBuffers) swapPageAndScratchBuffers() { + wb.page, wb.scratch = wb.scratch, wb.page[:0] +} + +// ColumnWriter writes values for a single column to underlying medium. +type ColumnWriter struct { + pool BufferPool + pageBuffer io.ReadWriteSeeker + numPages int + + columnPath columnPath + columnType Type + columnIndex ColumnIndexer + columnBuffer ColumnBuffer + columnFilter BloomFilterColumn + encoding encoding.Encoding + compression compress.Codec + dictionary Dictionary + + dataPageType format.PageType + maxRepetitionLevel byte + maxDefinitionLevel byte + + buffers *writerBuffers + + header struct { + protocol thrift.CompactProtocol + encoder thrift.Encoder + } + + filter []byte + numRows int64 + bufferIndex int32 + bufferSize int32 + writePageStats bool + writePageBounds bool + isCompressed bool + encodings []format.Encoding + + columnChunk *format.ColumnChunk + offsetIndex *format.OffsetIndex +} + +func (c *ColumnWriter) reset() { + if c.columnBuffer != nil { + c.columnBuffer.Reset() + } + if c.columnIndex != nil { + c.columnIndex.Reset() + } + if c.dictionary != nil { + c.dictionary.Reset() + } + if c.pageBuffer != nil { + c.pool.PutBuffer(c.pageBuffer) + c.pageBuffer = nil + } + c.numPages = 0 + // Bloom filters may change in size between row groups, but we retain the + // buffer to avoid reallocating large memory blocks. + c.filter = c.filter[:0] + c.numRows = 0 + // Reset the fields of column chunks that change between row groups, + // but keep the ones that remain unchanged. + c.columnChunk.MetaData.NumValues = 0 + c.columnChunk.MetaData.TotalUncompressedSize = 0 + c.columnChunk.MetaData.TotalCompressedSize = 0 + c.columnChunk.MetaData.DataPageOffset = 0 + c.columnChunk.MetaData.DictionaryPageOffset = 0 + c.columnChunk.MetaData.Statistics = format.Statistics{} + c.columnChunk.MetaData.EncodingStats = c.columnChunk.MetaData.EncodingStats[:0] + c.columnChunk.MetaData.BloomFilterOffset = 0 + c.offsetIndex.PageLocations = c.offsetIndex.PageLocations[:0] +} + +func (c *ColumnWriter) totalRowCount() int64 { + n := c.numRows + if c.columnBuffer != nil { + n += int64(c.columnBuffer.Len()) + } + return n +} + +func (c *ColumnWriter) flush() (err error) { + if c.columnBuffer == nil { + return nil + } + if c.columnBuffer.Len() > 0 { + defer c.columnBuffer.Reset() + _, err = c.writeDataPage(c.columnBuffer.Page()) + } + return err +} + +func (c *ColumnWriter) flushFilterPages() (err error) { + if c.columnFilter == nil { + return nil + } + + // If there is a dictionary, it contains all the values that we need to + // write to the filter. + if dict := c.dictionary; dict != nil { + // Need to always attempt to resize the filter, as the writer might + // be reused after resetting which would have reset the length of + // the filter to 0. + c.resizeBloomFilter(int64(dict.Len())) + return c.writePageToFilter(dict.Page()) + } + + // When the filter was already allocated, pages have been written to it as + // they were seen by the column writer. + if len(c.filter) > 0 { + return nil + } + + // When the filter was not allocated, the writer did not know how many + // values were going to be seen and therefore could not properly size the + // filter ahead of time. In this case, we read back all the pages that we + // have encoded and copy their values back to the filter. + // + // A prior implementation of the column writer used to create in-memory + // copies of the pages to avoid this decoding step; however, this unbounded + // allocation caused memory exhaustion in production applications. CPU being + // a somewhat more stretchable resource, we prefer spending time on this + // decoding step than having to trigger incident response when production + // systems are getting OOM-Killed. + c.resizeBloomFilter(c.columnChunk.MetaData.NumValues) + + column := &Column{ + // Set all the fields required by the decodeDataPage* methods. + typ: c.columnType, + encoding: c.encoding, + compression: c.compression, + maxRepetitionLevel: c.maxRepetitionLevel, + maxDefinitionLevel: c.maxDefinitionLevel, + index: int16(c.bufferIndex), + } + + var pageReader io.Reader = c.pageBuffer + if offset, err := c.pageBuffer.Seek(0, io.SeekStart); err != nil { + return err + } else if offset != 0 { + return fmt.Errorf("resetting parquet page buffer to the start expected offset zero but got %d", offset) + } + + if _, ok := pageReader.(*os.File); ok { + rbuf, pool := getBufioReader(pageReader, 1024) + defer func() { + putBufioReader(rbuf, pool) + }() + pageReader = rbuf + } + + pbuf := (*buffer)(nil) + defer func() { + if pbuf != nil { + pbuf.unref() + } + }() + + decoder := thrift.NewDecoder(c.header.protocol.NewReader(pageReader)) + + for range c.numPages { + header := new(format.PageHeader) + if err := decoder.Decode(header); err != nil { + return err + } + + if pbuf != nil { + pbuf.unref() + } + pbuf = buffers.get(int(header.CompressedPageSize)) + if _, err := io.ReadFull(pageReader, pbuf.data); err != nil { + return err + } + + var page Page + + switch header.Type { + case format.DataPage: + page, err = column.decodeDataPageV1(DataPageHeaderV1{header.DataPageHeader}, pbuf, nil, header.UncompressedPageSize) + case format.DataPageV2: + page, err = column.decodeDataPageV2(DataPageHeaderV2{header.DataPageHeaderV2}, pbuf, nil, header.UncompressedPageSize) + } + if page != nil { + err = c.writePageToFilter(page) + Release(page) + } + if err != nil { + return err + } + } + + return nil +} + +func (c *ColumnWriter) resizeBloomFilter(numValues int64) { + filterSize := c.columnFilter.Size(numValues) + if cap(c.filter) < filterSize { + c.filter = make([]byte, filterSize) + } else { + c.filter = c.filter[:filterSize] + for i := range c.filter { + c.filter[i] = 0 + } + } +} + +func (c *ColumnWriter) newColumnBuffer() ColumnBuffer { + column := c.columnType.NewColumnBuffer(int(c.bufferIndex), c.columnType.EstimateNumValues(int(c.bufferSize))) + switch { + case c.maxRepetitionLevel > 0: + column = newRepeatedColumnBuffer(column, c.maxRepetitionLevel, c.maxDefinitionLevel, nullsGoLast) + case c.maxDefinitionLevel > 0: + column = newOptionalColumnBuffer(column, c.maxDefinitionLevel, nullsGoLast) + } + return column +} + +// WriteRowValues writes entire rows to the column. On success, this returns the +// number of rows written (not the number of values). +// +// Unlike ValueWriter, where arbitrary values may be written regardless of row +// boundaries, this method requires whole rows. This is because the written +// values may be automatically flushed to a data page, based on the writer's +// configured page buffer size, and a single row is not permitted to span two +// pages. +func (c *ColumnWriter) WriteRowValues(rows []Value) (int, error) { + var startingRows int64 + if c.columnBuffer == nil { + // Lazily create the row group column so we don't need to allocate it if + // rows are not written individually to the column. + c.columnBuffer = c.newColumnBuffer() + } else { + startingRows = int64(c.columnBuffer.Len()) + } + if _, err := c.columnBuffer.WriteValues(rows); err != nil { + return 0, err + } + numRows := int(int64(c.columnBuffer.Len()) - startingRows) + if c.columnBuffer.Size() >= int64(c.bufferSize) { + return numRows, c.flush() + } + return numRows, nil +} + +func (c *ColumnWriter) writeValues(values []Value) (numValues int, err error) { + if c.columnBuffer == nil { + c.columnBuffer = c.newColumnBuffer() + } + return c.columnBuffer.WriteValues(values) +} + +func (c *ColumnWriter) writeBloomFilter(w io.Writer) error { + e := thrift.NewEncoder(c.header.protocol.NewWriter(w)) + h := bloomFilterHeader(c.columnFilter) + h.NumBytes = int32(len(c.filter)) + if err := e.Encode(&h); err != nil { + return err + } + _, err := w.Write(c.filter) + return err +} + +func (c *ColumnWriter) writeDataPage(page Page) (int64, error) { + numValues := page.NumValues() + if numValues == 0 { + return 0, nil + } + + buf := c.buffers + buf.reset() + + if c.maxRepetitionLevel > 0 { + buf.encodeRepetitionLevels(page, c.maxRepetitionLevel) + } + if c.maxDefinitionLevel > 0 { + buf.encodeDefinitionLevels(page, c.maxDefinitionLevel) + } + + if err := buf.encode(page, c.encoding); err != nil { + return 0, fmt.Errorf("encoding parquet data page: %w", err) + } + if c.dataPageType == format.DataPage { + buf.prependLevelsToDataPageV1(c.maxDefinitionLevel, c.maxDefinitionLevel) + } + + uncompressedPageSize := buf.size() + if uncompressedPageSize > maxUncompressedPageSize { + return 0, fmt.Errorf("page size limit exceeded: %d>%d", uncompressedPageSize, maxUncompressedPageSize) + } + if c.isCompressed { + if err := buf.compress(c.compression); err != nil { + return 0, fmt.Errorf("compressing parquet data page: %w", err) + } + } + + if page.Dictionary() == nil && len(c.filter) > 0 { + // When the writer knows the number of values in advance (e.g. when + // writing a full row group), the filter encoding is set and the page + // can be directly applied to the filter, which minimizes memory usage + // since there is no need to buffer the values in order to determine + // the size of the filter. + if err := c.writePageToFilter(page); err != nil { + return 0, err + } + } + + statistics := format.Statistics{} + if c.writePageStats { + statistics = c.makePageStatistics(page) + } + + pageHeader := &format.PageHeader{ + Type: c.dataPageType, + UncompressedPageSize: int32(uncompressedPageSize), + CompressedPageSize: int32(buf.size()), + CRC: int32(buf.crc32()), + } + + numRows := page.NumRows() + numNulls := page.NumNulls() + switch c.dataPageType { + case format.DataPage: + pageHeader.DataPageHeader = &format.DataPageHeader{ + NumValues: int32(numValues), + Encoding: c.encoding.Encoding(), + DefinitionLevelEncoding: format.RLE, + RepetitionLevelEncoding: format.RLE, + Statistics: statistics, + } + case format.DataPageV2: + pageHeader.DataPageHeaderV2 = &format.DataPageHeaderV2{ + NumValues: int32(numValues), + NumNulls: int32(numNulls), + NumRows: int32(numRows), + Encoding: c.encoding.Encoding(), + DefinitionLevelsByteLength: int32(len(buf.definitions)), + RepetitionLevelsByteLength: int32(len(buf.repetitions)), + IsCompressed: &c.isCompressed, + Statistics: statistics, + } + } + + buf.header.Reset() + if err := c.header.encoder.Encode(pageHeader); err != nil { + return 0, err + } + + size := int64(buf.header.Len()) + + int64(len(buf.repetitions)) + + int64(len(buf.definitions)) + + int64(len(buf.page)) + + err := c.writePageTo(size, func(output io.Writer) (written int64, err error) { + for _, data := range [...][]byte{ + buf.header.Bytes(), + buf.repetitions, + buf.definitions, + buf.page, + } { + wn, err := output.Write(data) + written += int64(wn) + if err != nil { + return written, err + } + } + return written, nil + }) + if err != nil { + return 0, err + } + + c.recordPageStats(int32(buf.header.Len()), pageHeader, page) + return numValues, nil +} + +func (c *ColumnWriter) writeDictionaryPage(output io.Writer, dict Dictionary) (err error) { + buf := c.buffers + buf.reset() + + if err := buf.encode(dict.Page(), &Plain); err != nil { + return fmt.Errorf("writing parquet dictionary page: %w", err) + } + + uncompressedPageSize := buf.size() + if uncompressedPageSize > maxUncompressedPageSize { + return fmt.Errorf("page size limit exceeded: %d>%d", uncompressedPageSize, maxUncompressedPageSize) + } + if isCompressed(c.compression) { + if err := buf.compress(c.compression); err != nil { + return fmt.Errorf("copmressing parquet dictionary page: %w", err) + } + } + + pageHeader := &format.PageHeader{ + Type: format.DictionaryPage, + UncompressedPageSize: int32(uncompressedPageSize), + CompressedPageSize: int32(buf.size()), + CRC: int32(buf.crc32()), + DictionaryPageHeader: &format.DictionaryPageHeader{ + NumValues: int32(dict.Len()), + Encoding: format.Plain, + IsSorted: false, + }, + } + + header := &c.buffers.header + header.Reset() + if err := c.header.encoder.Encode(pageHeader); err != nil { + return err + } + if _, err := output.Write(header.Bytes()); err != nil { + return err + } + if _, err := output.Write(buf.page); err != nil { + return err + } + c.recordPageStats(int32(header.Len()), pageHeader, nil) + return nil +} + +func (w *ColumnWriter) writePageToFilter(page Page) (err error) { + pageType := page.Type() + pageData := page.Data() + w.filter, err = pageType.Encode(w.filter, pageData, w.columnFilter.Encoding()) + return err +} + +func (c *ColumnWriter) writePageTo(size int64, writeTo func(io.Writer) (int64, error)) (err error) { + if c.pageBuffer == nil { + c.pageBuffer = c.pool.GetBuffer() + defer func() { + if err != nil { + c.pool.PutBuffer(c.pageBuffer) + c.pageBuffer = nil + } + }() + if _, err = c.pageBuffer.Seek(0, io.SeekStart); err != nil { + return err + } + } + written, err := writeTo(c.pageBuffer) + if err != nil { + return err + } + if written != size { + return fmt.Errorf("writing parquet column page expected %dB but got %dB: %w", size, written, io.ErrShortWrite) + } + c.numPages++ + return nil +} + +func (c *ColumnWriter) makePageStatistics(page Page) format.Statistics { + numNulls := page.NumNulls() + minValue, maxValue, _ := page.Bounds() + minValueBytes := minValue.Bytes() + maxValueBytes := maxValue.Bytes() + return format.Statistics{ + Min: minValueBytes, // deprecated + Max: maxValueBytes, // deprecated + NullCount: numNulls, + MinValue: minValueBytes, + MaxValue: maxValueBytes, + } +} + +func (c *ColumnWriter) recordPageStats(headerSize int32, header *format.PageHeader, page Page) { + uncompressedSize := headerSize + header.UncompressedPageSize + compressedSize := headerSize + header.CompressedPageSize + + if page != nil { + numNulls := page.NumNulls() + numValues := page.NumValues() + + var minValue, maxValue Value + var pageHasBounds bool + if c.writePageBounds { + minValue, maxValue, pageHasBounds = page.Bounds() + } + + c.columnIndex.IndexPage(numValues, numNulls, minValue, maxValue) + c.columnChunk.MetaData.NumValues += numValues + c.columnChunk.MetaData.Statistics.NullCount += numNulls + + if pageHasBounds { + var existingMaxValue, existingMinValue Value + + if c.columnChunk.MetaData.Statistics.MaxValue != nil && c.columnChunk.MetaData.Statistics.MinValue != nil { + existingMaxValue = c.columnType.Kind().Value(c.columnChunk.MetaData.Statistics.MaxValue) + existingMinValue = c.columnType.Kind().Value(c.columnChunk.MetaData.Statistics.MinValue) + } + + if existingMaxValue.isNull() || c.columnType.Compare(maxValue, existingMaxValue) > 0 { + buf := c.columnChunk.MetaData.Statistics.MaxValue[:0] + // if maxValue is empty string, c.columnChunk.MetaData.Statistics.MaxValue should be []bytes{}, but nil + if buf == nil && maxValue.Kind() == ByteArray && len(maxValue.ByteArray()) == 0 { + buf = make([]byte, 0) + } + c.columnChunk.MetaData.Statistics.MaxValue = maxValue.AppendBytes(buf) + } + + if existingMinValue.isNull() || c.columnType.Compare(minValue, existingMinValue) < 0 { + buf := c.columnChunk.MetaData.Statistics.MinValue[:0] + // same as above + if buf == nil && minValue.Kind() == ByteArray && len(minValue.ByteArray()) == 0 { + buf = make([]byte, 0) + } + c.columnChunk.MetaData.Statistics.MinValue = minValue.AppendBytes(buf) + } + } + + c.offsetIndex.PageLocations = append(c.offsetIndex.PageLocations, format.PageLocation{ + Offset: c.columnChunk.MetaData.TotalCompressedSize, + CompressedPageSize: compressedSize, + FirstRowIndex: c.numRows, + }) + + c.numRows += page.NumRows() + } + + pageType := header.Type + encoding := format.Encoding(-1) + switch pageType { + case format.DataPageV2: + encoding = header.DataPageHeaderV2.Encoding + case format.DataPage: + encoding = header.DataPageHeader.Encoding + case format.DictionaryPage: + encoding = header.DictionaryPageHeader.Encoding + } + + c.columnChunk.MetaData.TotalUncompressedSize += int64(uncompressedSize) + c.columnChunk.MetaData.TotalCompressedSize += int64(compressedSize) + c.columnChunk.MetaData.EncodingStats = addPageEncodingStats(c.columnChunk.MetaData.EncodingStats, format.PageEncodingStats{ + PageType: pageType, + Encoding: encoding, + Count: 1, + }) +} + +func addEncoding(encodings []format.Encoding, add format.Encoding) []format.Encoding { + if slices.Contains(encodings, add) { + return encodings + } + return append(encodings, add) +} + +func addPageEncodingStats(stats []format.PageEncodingStats, pages ...format.PageEncodingStats) []format.PageEncodingStats { +addPages: + for _, add := range pages { + for i, st := range stats { + if st.PageType == add.PageType && st.Encoding == add.Encoding { + stats[i].Count += add.Count + continue addPages + } + } + stats = append(stats, add) + } + return stats +} + +func sortPageEncodings(encodings []format.Encoding) { + slices.Sort(encodings) +} + +func sortPageEncodingStats(stats []format.PageEncodingStats) { + slices.SortFunc(stats, func(s1, s2 format.PageEncodingStats) int { + if k := cmp.Compare(s1.PageType, s2.PageType); k != 0 { + return k + } + return cmp.Compare(s1.Encoding, s2.Encoding) + }) +} + +type offsetTrackingWriter struct { + writer io.Writer + offset int64 +} + +func (w *offsetTrackingWriter) Reset(writer io.Writer) { + w.writer = writer + w.offset = 0 +} + +func (w *offsetTrackingWriter) Write(b []byte) (int, error) { + n, err := w.writer.Write(b) + w.offset += int64(n) + return n, err +} + +func (w *offsetTrackingWriter) WriteString(s string) (int, error) { + n, err := io.WriteString(w.writer, s) + w.offset += int64(n) + return n, err +} + +func (w *offsetTrackingWriter) ReadFrom(r io.Reader) (int64, error) { + // io.Copy will make use of io.ReaderFrom if w.writer implements it. + n, err := io.Copy(w.writer, r) + w.offset += n + return n, err +} + +var ( + _ RowWriterWithSchema = (*Writer)(nil) + _ RowReaderFrom = (*Writer)(nil) + _ RowGroupWriter = (*Writer)(nil) + + _ RowWriter = (*writer)(nil) + _ ValueWriter = (*writer)(nil) + + _ io.ReaderFrom = (*offsetTrackingWriter)(nil) + _ io.StringWriter = (*offsetTrackingWriter)(nil) +) diff --git a/vendor/github.com/pierrec/lz4/v4/.gitignore b/vendor/github.com/pierrec/lz4/v4/.gitignore new file mode 100644 index 0000000000..5d7e88de0a --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/.gitignore @@ -0,0 +1,36 @@ +# Created by https://www.gitignore.io/api/macos + +### macOS ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# End of https://www.gitignore.io/api/macos + +cmd/*/*exe +.idea + +fuzz/*.zip diff --git a/vendor/github.com/pierrec/lz4/v4/LICENSE b/vendor/github.com/pierrec/lz4/v4/LICENSE new file mode 100644 index 0000000000..bd899d8353 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2015, Pierre Curto +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of xxHash nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/pierrec/lz4/v4/README.md b/vendor/github.com/pierrec/lz4/v4/README.md new file mode 100644 index 0000000000..dee77545b0 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/README.md @@ -0,0 +1,92 @@ +# lz4 : LZ4 compression in pure Go + +[![Go Reference](https://pkg.go.dev/badge/github.com/pierrec/lz4/v4.svg)](https://pkg.go.dev/github.com/pierrec/lz4/v4) +[![CI](https://github.com/pierrec/lz4/workflows/ci/badge.svg)](https://github.com/pierrec/lz4/actions) +[![Go Report Card](https://goreportcard.com/badge/github.com/pierrec/lz4)](https://goreportcard.com/report/github.com/pierrec/lz4) +[![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/pierrec/lz4.svg?style=social)](https://github.com/pierrec/lz4/tags) + +## Overview + +This package provides a streaming interface to [LZ4 data streams](http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html) as well as low level compress and uncompress functions for LZ4 data blocks. +The implementation is based on the reference C [one](https://github.com/lz4/lz4). + +## Install + +Assuming you have the go toolchain installed: + +``` +go get github.com/pierrec/lz4/v4 +``` + +There is a command line interface tool to compress and decompress LZ4 files. + +``` +go install github.com/pierrec/lz4/v4/cmd/lz4c@latest +``` + +Usage + +``` +Usage of lz4c: + -version + print the program version + +Subcommands: +Compress the given files or from stdin to stdout. +compress [arguments] [ ...] + -bc + enable block checksum + -l int + compression level (0=fastest) + -sc + disable stream checksum + -size string + block max size [64K,256K,1M,4M] (default "4M") + +Uncompress the given files or from stdin to stdout. +uncompress [arguments] [ ...] + +``` + + +## Example + +``` +// Compress and uncompress an input string. +s := "hello world" +r := strings.NewReader(s) + +// The pipe will uncompress the data from the writer. +pr, pw := io.Pipe() +zw := lz4.NewWriter(pw) +zr := lz4.NewReader(pr) + +go func() { + // Compress the input string. + _, _ = io.Copy(zw, r) + _ = zw.Close() // Make sure the writer is closed + _ = pw.Close() // Terminate the pipe +}() + +_, _ = io.Copy(os.Stdout, zr) + +// Output: +// hello world +``` + +## Contributing + +Contributions are very welcome for bug fixing, performance improvements...! + +- Open an issue with a proper description +- Send a pull request with appropriate test case(s) + +## Contributors + +Thanks to all [contributors](https://github.com/pierrec/lz4/graphs/contributors) so far! + +Special thanks to [@Zariel](https://github.com/Zariel) for his asm implementation of the decoder. + +Special thanks to [@greatroar](https://github.com/greatroar) for his work on the asm implementations of the decoder for amd64 and arm64. + +Special thanks to [@klauspost](https://github.com/klauspost) for his work on optimizing the code. diff --git a/vendor/github.com/pierrec/lz4/v4/compressing_reader.go b/vendor/github.com/pierrec/lz4/v4/compressing_reader.go new file mode 100644 index 0000000000..8df0dc76d0 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/compressing_reader.go @@ -0,0 +1,222 @@ +package lz4 + +import ( + "errors" + "io" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/lz4stream" +) + +type crState int + +const ( + crStateInitial crState = iota + crStateReading + crStateFlushing + crStateDone +) + +type CompressingReader struct { + state crState + src io.ReadCloser // source reader + level lz4block.CompressionLevel // how hard to try + frame *lz4stream.Frame // frame being built + in []byte + out ovWriter + handler func(int) +} + +// NewCompressingReader creates a reader which reads compressed data from +// raw stream. This makes it a logical opposite of a normal lz4.Reader. +// We require an io.ReadCloser as an underlying source for compatibility +// with Go's http.Request. +func NewCompressingReader(src io.ReadCloser) *CompressingReader { + zrd := &CompressingReader { + frame: lz4stream.NewFrame(), + } + + _ = zrd.Apply(DefaultBlockSizeOption, DefaultChecksumOption, defaultOnBlockDone) + zrd.Reset(src) + + return zrd +} + +// Source exposes the underlying source stream for introspection and control. +func (zrd *CompressingReader) Source() io.ReadCloser { + return zrd.src +} + +// Close simply invokes the underlying stream Close method. This method is +// provided for the benefit of Go http client/server, which relies on Close +// for goroutine termination. +func (zrd *CompressingReader) Close() error { + return zrd.src.Close() +} + +// Apply applies useful options to the lz4 encoder. +func (zrd *CompressingReader) Apply(options ...Option) (err error) { + if zrd.state != crStateInitial { + return lz4errors.ErrOptionClosedOrError + } + + zrd.Reset(zrd.src) + + for _, o := range options { + if err = o(zrd); err != nil { + return + } + } + return +} + +func (*CompressingReader) private() {} + +func (zrd *CompressingReader) init() error { + zrd.frame.InitW(&zrd.out, 1, false) + size := zrd.frame.Descriptor.Flags.BlockSizeIndex() + zrd.in = size.Get() + return zrd.frame.Descriptor.Write(zrd.frame, &zrd.out) +} + +// Read allows reading of lz4 compressed data +func (zrd *CompressingReader) Read(p []byte) (n int, err error) { + defer func() { + if err != nil { + zrd.state = crStateDone + } + }() + + if !zrd.out.reset(p) { + return len(p), nil + } + + switch zrd.state { + case crStateInitial: + err = zrd.init() + if err != nil { + return + } + zrd.state = crStateReading + case crStateDone: + return 0, errors.New("This reader is done") + case crStateFlushing: + if zrd.out.dataPos > 0 { + n = zrd.out.dataPos + zrd.out.data = nil + zrd.out.dataPos = 0 + return + } else { + zrd.state = crStateDone + return 0, io.EOF + } + } + + for zrd.state == crStateReading { + block := zrd.frame.Blocks.Block + + var rCount int + rCount, err = io.ReadFull(zrd.src, zrd.in) + switch err { + case nil: + err = block.Compress( + zrd.frame, zrd.in[ : rCount], zrd.level, + ).Write(zrd.frame, &zrd.out) + zrd.handler(len(block.Data)) + if err != nil { + return + } + + if zrd.out.dataPos == len(zrd.out.data) { + n = zrd.out.dataPos + zrd.out.dataPos = 0 + zrd.out.data = nil + return + } + case io.EOF, io.ErrUnexpectedEOF: // read may be partial + if rCount > 0 { + err = block.Compress( + zrd.frame, zrd.in[ : rCount], zrd.level, + ).Write(zrd.frame, &zrd.out) + zrd.handler(len(block.Data)) + if err != nil { + return + } + } + + err = zrd.frame.CloseW(&zrd.out, 1) + if err != nil { + return + } + zrd.state = crStateFlushing + + n = zrd.out.dataPos + zrd.out.dataPos = 0 + zrd.out.data = nil + return + default: + return + } + } + + err = lz4errors.ErrInternalUnhandledState + return +} + +// Reset makes the stream usable again; mostly handy to reuse lz4 encoder +// instances. +func (zrd *CompressingReader) Reset(src io.ReadCloser) { + zrd.frame.Reset(1) + zrd.state = crStateInitial + zrd.src = src + zrd.out.clear() +} + +type ovWriter struct { + data []byte + ov []byte + dataPos int + ovPos int +} + +func (wr *ovWriter) Write(p []byte) (n int, err error) { + count := copy(wr.data[wr.dataPos : ], p) + wr.dataPos += count + + if count < len(p) { + wr.ov = append(wr.ov, p[count : ]...) + } + + return len(p), nil +} + +func (wr *ovWriter) reset(out []byte) bool { + ovRem := len(wr.ov) - wr.ovPos + + if ovRem >= len(out) { + wr.ovPos += copy(out, wr.ov[wr.ovPos : ]) + return false + } + + if ovRem > 0 { + copy(out, wr.ov[wr.ovPos : ]) + wr.ov = wr.ov[ : 0] + wr.ovPos = 0 + wr.dataPos = ovRem + } else if wr.ovPos > 0 { + wr.ov = wr.ov[ : 0] + wr.ovPos = 0 + wr.dataPos = 0 + } + + wr.data = out + return true +} + +func (wr *ovWriter) clear() { + wr.data = nil + wr.dataPos = 0 + wr.ov = wr.ov[ : 0] + wr.ovPos = 0 +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go new file mode 100644 index 0000000000..fec8adb03a --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/block.go @@ -0,0 +1,481 @@ +package lz4block + +import ( + "encoding/binary" + "math/bits" + "sync" + + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +const ( + // The following constants are used to setup the compression algorithm. + minMatch = 4 // the minimum size of the match sequence size (4 bytes) + winSizeLog = 16 // LZ4 64Kb window size limit + winSize = 1 << winSizeLog + winMask = winSize - 1 // 64Kb window of previous data for dependent blocks + + // hashLog determines the size of the hash table used to quickly find a previous match position. + // Its value influences the compression speed and memory usage, the lower the faster, + // but at the expense of the compression ratio. + // 16 seems to be the best compromise for fast compression. + hashLog = 16 + htSize = 1 << hashLog + + mfLimit = 10 + minMatch // The last match cannot start within the last 14 bytes. +) + +func recoverBlock(e *error) { + if r := recover(); r != nil && *e == nil { + *e = lz4errors.ErrInvalidSourceShortBuffer + } +} + +// blockHash hashes the lower 6 bytes into a value < htSize. +func blockHash(x uint64) uint32 { + const prime6bytes = 227718039650203 + return uint32(((x << (64 - 48)) * prime6bytes) >> (64 - hashLog)) +} + +func CompressBlockBound(n int) int { + return n + n/255 + 16 +} + +func UncompressBlock(src, dst, dict []byte) (int, error) { + if len(src) == 0 { + return 0, nil + } + if di := decodeBlock(dst, src, dict); di >= 0 { + return di, nil + } + return 0, lz4errors.ErrInvalidSourceShortBuffer +} + +type Compressor struct { + // Offsets are at most 64kiB, so we can store only the lower 16 bits of + // match positions: effectively, an offset from some 64kiB block boundary. + // + // When we retrieve such an offset, we interpret it as relative to the last + // block boundary si &^ 0xffff, or the one before, (si &^ 0xffff) - 0x10000, + // depending on which of these is inside the current window. If a table + // entry was generated more than 64kiB back in the input, we find out by + // inspecting the input stream. + table [htSize]uint16 + + // Bitmap indicating which positions in the table are in use. + // This allows us to quickly reset the table for reuse, + // without having to zero everything. + inUse [htSize / 32]uint32 +} + +// Get returns the position of a presumptive match for the hash h. +// The match may be a false positive due to a hash collision or an old entry. +// If si < winSize, the return value may be negative. +func (c *Compressor) get(h uint32, si int) int { + h &= htSize - 1 + i := 0 + if c.inUse[h/32]&(1<<(h%32)) != 0 { + i = int(c.table[h]) + } + i += si &^ winMask + if i >= si { + // Try previous 64kiB block (negative when in first block). + i -= winSize + } + return i +} + +func (c *Compressor) put(h uint32, si int) { + h &= htSize - 1 + c.table[h] = uint16(si) + c.inUse[h/32] |= 1 << (h % 32) +} + +func (c *Compressor) reset() { c.inUse = [htSize / 32]uint32{} } + +var compressorPool = sync.Pool{New: func() interface{} { return new(Compressor) }} + +func CompressBlock(src, dst []byte) (int, error) { + c := compressorPool.Get().(*Compressor) + n, err := c.CompressBlock(src, dst) + compressorPool.Put(c) + return n, err +} + +func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { + // Zero out reused table to avoid non-deterministic output (issue #65). + c.reset() + + // Return 0, nil only if the destination buffer size is < CompressBlockBound. + isNotCompressible := len(dst) < CompressBlockBound(len(src)) + + // adaptSkipLog sets how quickly the compressor begins skipping blocks when data is incompressible. + // This significantly speeds up incompressible data and usually has very small impact on compression. + // bytes to skip = 1 + (bytes since last match >> adaptSkipLog) + const adaptSkipLog = 7 + + // si: Current position of the search. + // anchor: Position of the current literals. + var si, di, anchor int + sn := len(src) - mfLimit + if sn <= 0 { + goto lastLiterals + } + + // Fast scan strategy: the hash table only stores the last 4 bytes sequences. + for si < sn { + // Hash the next 6 bytes (sequence)... + match := binary.LittleEndian.Uint64(src[si:]) + h := blockHash(match) + h2 := blockHash(match >> 8) + + // We check a match at s, s+1 and s+2 and pick the first one we get. + // Checking 3 only requires us to load the source one. + ref := c.get(h, si) + ref2 := c.get(h2, si+1) + c.put(h, si) + c.put(h2, si+1) + + offset := si - ref + + if offset <= 0 || offset >= winSize || uint32(match) != binary.LittleEndian.Uint32(src[ref:]) { + // No match. Start calculating another hash. + // The processor can usually do this out-of-order. + h = blockHash(match >> 16) + ref3 := c.get(h, si+2) + + // Check the second match at si+1 + si += 1 + offset = si - ref2 + + if offset <= 0 || offset >= winSize || uint32(match>>8) != binary.LittleEndian.Uint32(src[ref2:]) { + // No match. Check the third match at si+2 + si += 1 + offset = si - ref3 + c.put(h, si) + + if offset <= 0 || offset >= winSize || uint32(match>>16) != binary.LittleEndian.Uint32(src[ref3:]) { + // Skip one extra byte (at si+3) before we check 3 matches again. + si += 2 + (si-anchor)>>adaptSkipLog + continue + } + } + } + + // Match found. + lLen := si - anchor // Literal length. + // We already matched 4 bytes. + mLen := 4 + + // Extend backwards if we can, reducing literals. + tOff := si - offset - 1 + for lLen > 0 && tOff >= 0 && src[si-1] == src[tOff] { + si-- + tOff-- + lLen-- + mLen++ + } + + // Add the match length, so we continue search at the end. + // Use mLen to store the offset base. + si, mLen = si+mLen, si+minMatch + + // Find the longest match by looking by batches of 8 bytes. + for si+8 <= sn { + x := binary.LittleEndian.Uint64(src[si:]) ^ binary.LittleEndian.Uint64(src[si-offset:]) + if x == 0 { + si += 8 + } else { + // Stop is first non-zero byte. + si += bits.TrailingZeros64(x) >> 3 + break + } + } + + mLen = si - mLen + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + if mLen < 0xF { + dst[di] = byte(mLen) + } else { + dst[di] = 0xF + } + + // Encode literals length. + if lLen < 0xF { + dst[di] |= byte(lLen << 4) + } else { + dst[di] |= 0xF0 + di++ + l := lLen - 0xF + for ; l >= 0xFF && di < len(dst); l -= 0xFF { + dst[di] = 0xFF + di++ + } + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di] = byte(l) + } + di++ + + // Literals. + if di+lLen > len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + copy(dst[di:di+lLen], src[anchor:anchor+lLen]) + di += lLen + 2 + anchor = si + + // Encode offset. + if di > len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di-2], dst[di-1] = byte(offset), byte(offset>>8) + + // Encode match length part 2. + if mLen >= 0xF { + for mLen -= 0xF; mLen >= 0xFF && di < len(dst); mLen -= 0xFF { + dst[di] = 0xFF + di++ + } + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di] = byte(mLen) + di++ + } + // Check if we can load next values. + if si >= sn { + break + } + // Hash match end-2 + h = blockHash(binary.LittleEndian.Uint64(src[si-2:])) + c.put(h, si-2) + } + +lastLiterals: + if isNotCompressible && anchor == 0 { + // Incompressible. + return 0, nil + } + + // Last literals. + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + lLen := len(src) - anchor + if lLen < 0xF { + dst[di] = byte(lLen << 4) + } else { + dst[di] = 0xF0 + di++ + for lLen -= 0xF; lLen >= 0xFF && di < len(dst); lLen -= 0xFF { + dst[di] = 0xFF + di++ + } + if di >= len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + dst[di] = byte(lLen) + } + di++ + + // Write the last literals. + if isNotCompressible && di >= anchor { + // Incompressible. + return 0, nil + } + if di+len(src)-anchor > len(dst) { + return 0, lz4errors.ErrInvalidSourceShortBuffer + } + di += copy(dst[di:di+len(src)-anchor], src[anchor:]) + return di, nil +} + +// blockHash hashes 4 bytes into a value < winSize. +func blockHashHC(x uint32) uint32 { + const hasher uint32 = 2654435761 // Knuth multiplicative hash. + return x * hasher >> (32 - winSizeLog) +} + +type CompressorHC struct { + // hashTable: stores the last position found for a given hash + // chainTable: stores previous positions for a given hash + hashTable, chainTable [htSize]int + needsReset bool +} + +var compressorHCPool = sync.Pool{New: func() interface{} { return new(CompressorHC) }} + +func CompressBlockHC(src, dst []byte, depth CompressionLevel) (int, error) { + c := compressorHCPool.Get().(*CompressorHC) + n, err := c.CompressBlock(src, dst, depth) + compressorHCPool.Put(c) + return n, err +} + +func (c *CompressorHC) CompressBlock(src, dst []byte, depth CompressionLevel) (_ int, err error) { + if c.needsReset { + // Zero out reused table to avoid non-deterministic output (issue #65). + c.hashTable = [htSize]int{} + c.chainTable = [htSize]int{} + } + c.needsReset = true // Only false on first call. + + defer recoverBlock(&err) + + // Return 0, nil only if the destination buffer size is < CompressBlockBound. + isNotCompressible := len(dst) < CompressBlockBound(len(src)) + + // adaptSkipLog sets how quickly the compressor begins skipping blocks when data is incompressible. + // This significantly speeds up incompressible data and usually has very small impact on compression. + // bytes to skip = 1 + (bytes since last match >> adaptSkipLog) + const adaptSkipLog = 7 + + var si, di, anchor int + sn := len(src) - mfLimit + if sn <= 0 { + goto lastLiterals + } + + if depth == 0 { + depth = winSize + } + + for si < sn { + // Hash the next 4 bytes (sequence). + match := binary.LittleEndian.Uint32(src[si:]) + h := blockHashHC(match) + + // Follow the chain until out of window and give the longest match. + mLen := 0 + offset := 0 + for next, try := c.hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next, try = c.chainTable[next&winMask], try-1 { + // The first (mLen==0) or next byte (mLen>=minMatch) at current match length + // must match to improve on the match length. + if src[next+mLen] != src[si+mLen] { + continue + } + ml := 0 + // Compare the current position with a previous with the same hash. + for ml < sn-si { + x := binary.LittleEndian.Uint64(src[next+ml:]) ^ binary.LittleEndian.Uint64(src[si+ml:]) + if x == 0 { + ml += 8 + } else { + // Stop is first non-zero byte. + ml += bits.TrailingZeros64(x) >> 3 + break + } + } + if ml < minMatch || ml <= mLen { + // Match too small (>adaptSkipLog + continue + } + + // Match found. + // Update hash/chain tables with overlapping bytes: + // si already hashed, add everything from si+1 up to the match length. + winStart := si + 1 + if ws := si + mLen - winSize; ws > winStart { + winStart = ws + } + for si, ml := winStart, si+mLen; si < ml; { + match >>= 8 + match |= uint32(src[si+3]) << 24 + h := blockHashHC(match) + c.chainTable[si&winMask] = c.hashTable[h] + c.hashTable[h] = si + si++ + } + + lLen := si - anchor + si += mLen + mLen -= minMatch // Match length does not include minMatch. + + if mLen < 0xF { + dst[di] = byte(mLen) + } else { + dst[di] = 0xF + } + + // Encode literals length. + if lLen < 0xF { + dst[di] |= byte(lLen << 4) + } else { + dst[di] |= 0xF0 + di++ + l := lLen - 0xF + for ; l >= 0xFF; l -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(l) + } + di++ + + // Literals. + copy(dst[di:di+lLen], src[anchor:anchor+lLen]) + di += lLen + anchor = si + + // Encode offset. + di += 2 + dst[di-2], dst[di-1] = byte(offset), byte(offset>>8) + + // Encode match length part 2. + if mLen >= 0xF { + for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(mLen) + di++ + } + } + + if isNotCompressible && anchor == 0 { + // Incompressible. + return 0, nil + } + + // Last literals. +lastLiterals: + lLen := len(src) - anchor + if lLen < 0xF { + dst[di] = byte(lLen << 4) + } else { + dst[di] = 0xF0 + di++ + lLen -= 0xF + for ; lLen >= 0xFF; lLen -= 0xFF { + dst[di] = 0xFF + di++ + } + dst[di] = byte(lLen) + } + di++ + + // Write the last literals. + if isNotCompressible && di >= anchor { + // Incompressible. + return 0, nil + } + di += copy(dst[di:di+len(src)-anchor], src[anchor:]) + return di, nil +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go new file mode 100644 index 0000000000..138083d947 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/blocks.go @@ -0,0 +1,87 @@ +// Package lz4block provides LZ4 BlockSize types and pools of buffers. +package lz4block + +import "sync" + +const ( + Block64Kb uint32 = 1 << (16 + iota*2) + Block256Kb + Block1Mb + Block4Mb + Block8Mb = 2 * Block4Mb +) + +var ( + BlockPool64K = sync.Pool{New: func() interface{} { return make([]byte, Block64Kb) }} + BlockPool256K = sync.Pool{New: func() interface{} { return make([]byte, Block256Kb) }} + BlockPool1M = sync.Pool{New: func() interface{} { return make([]byte, Block1Mb) }} + BlockPool4M = sync.Pool{New: func() interface{} { return make([]byte, Block4Mb) }} + BlockPool8M = sync.Pool{New: func() interface{} { return make([]byte, Block8Mb) }} +) + +func Index(b uint32) BlockSizeIndex { + switch b { + case Block64Kb: + return 4 + case Block256Kb: + return 5 + case Block1Mb: + return 6 + case Block4Mb: + return 7 + case Block8Mb: // only valid in legacy mode + return 3 + } + return 0 +} + +func IsValid(b uint32) bool { + return Index(b) > 0 +} + +type BlockSizeIndex uint8 + +func (b BlockSizeIndex) IsValid() bool { + switch b { + case 4, 5, 6, 7: + return true + } + return false +} + +func (b BlockSizeIndex) Get() []byte { + var buf interface{} + switch b { + case 4: + buf = BlockPool64K.Get() + case 5: + buf = BlockPool256K.Get() + case 6: + buf = BlockPool1M.Get() + case 7: + buf = BlockPool4M.Get() + case 3: + buf = BlockPool8M.Get() + } + return buf.([]byte) +} + +func Put(buf []byte) { + // Safeguard: do not allow invalid buffers. + switch c := cap(buf); uint32(c) { + case Block64Kb: + BlockPool64K.Put(buf[:c]) + case Block256Kb: + BlockPool256K.Put(buf[:c]) + case Block1Mb: + BlockPool1M.Put(buf[:c]) + case Block4Mb: + BlockPool4M.Put(buf[:c]) + case Block8Mb: + BlockPool8M.Put(buf[:c]) + } +} + +type CompressionLevel uint32 + +const Fast CompressionLevel = 0 diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s new file mode 100644 index 0000000000..1d00133fac --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_amd64.s @@ -0,0 +1,448 @@ +// +build !appengine +// +build gc +// +build !noasm + +#include "go_asm.h" +#include "textflag.h" + +// AX scratch +// BX scratch +// CX literal and match lengths +// DX token, match offset +// +// DI &dst +// SI &src +// R8 &dst + len(dst) +// R9 &src + len(src) +// R11 &dst +// R12 short output end +// R13 short input end +// R14 &dict +// R15 len(dict) + +// func decodeBlock(dst, src, dict []byte) int +TEXT ยทdecodeBlock(SB), NOSPLIT, $48-80 + MOVQ dst_base+0(FP), DI + MOVQ DI, R11 + MOVQ dst_len+8(FP), R8 + ADDQ DI, R8 + + MOVQ src_base+24(FP), SI + MOVQ src_len+32(FP), R9 + CMPQ R9, $0 + JE err_corrupt + ADDQ SI, R9 + + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + + // shortcut ends + // short output end + MOVQ R8, R12 + SUBQ $32, R12 + // short input end + MOVQ R9, R13 + SUBQ $16, R13 + + XORL CX, CX + +loop: + // token := uint32(src[si]) + MOVBLZX (SI), DX + INCQ SI + + // lit_len = token >> 4 + // if lit_len > 0 + // CX = lit_len + MOVL DX, CX + SHRL $4, CX + + // if lit_len != 0xF + CMPL CX, $0xF + JEQ lit_len_loop + CMPQ DI, R12 + JAE copy_literal + CMPQ SI, R13 + JAE copy_literal + + // copy shortcut + + // A two-stage shortcut for the most common case: + // 1) If the literal length is 0..14, and there is enough space, + // enter the shortcut and copy 16 bytes on behalf of the literals + // (in the fast mode, only 8 bytes can be safely copied this way). + // 2) Further if the match length is 4..18, copy 18 bytes in a similar + // manner; but we ensure that there's enough space in the output for + // those 18 bytes earlier, upon entering the shortcut (in other words, + // there is a combined check for both stages). + + // copy literal + MOVOU (SI), X0 + MOVOU X0, (DI) + ADDQ CX, DI + ADDQ CX, SI + + MOVL DX, CX + ANDL $0xF, CX + + // The second stage: prepare for match copying, decode full info. + // If it doesn't work out, the info won't be wasted. + // offset := uint16(data[:2]) + MOVWLZX (SI), DX + TESTL DX, DX + JE err_corrupt + ADDQ $2, SI + JC err_short_buf + + MOVQ DI, AX + SUBQ DX, AX + JC err_corrupt + CMPQ AX, DI + JA err_short_buf + + // if we can't do the second stage then jump straight to read the + // match length, we already have the offset. + CMPL CX, $0xF + JEQ match_len_loop_pre + CMPL DX, $8 + JLT match_len_loop_pre + CMPQ AX, R11 + JB match_len_loop_pre + + // memcpy(op + 0, match + 0, 8); + MOVQ (AX), BX + MOVQ BX, (DI) + // memcpy(op + 8, match + 8, 8); + MOVQ 8(AX), BX + MOVQ BX, 8(DI) + // memcpy(op +16, match +16, 2); + MOVW 16(AX), BX + MOVW BX, 16(DI) + + LEAQ const_minMatch(DI)(CX*1), DI + + // shortcut complete, load next token + JMP loopcheck + + // Read the rest of the literal length: + // do { BX = src[si++]; lit_len += BX } while (BX == 0xFF). +lit_len_loop: + CMPQ SI, R9 + JAE err_short_buf + + MOVBLZX (SI), BX + INCQ SI + ADDQ BX, CX + + CMPB BX, $0xFF + JE lit_len_loop + +copy_literal: + // bounds check src and dst + MOVQ SI, AX + ADDQ CX, AX + JC err_short_buf + CMPQ AX, R9 + JA err_short_buf + + MOVQ DI, BX + ADDQ CX, BX + JC err_short_buf + CMPQ BX, R8 + JA err_short_buf + + // Copy literals of <=48 bytes through the XMM registers. + CMPQ CX, $48 + JGT memmove_lit + + // if len(dst[di:]) < 48 + MOVQ R8, AX + SUBQ DI, AX + CMPQ AX, $48 + JLT memmove_lit + + // if len(src[si:]) < 48 + MOVQ R9, BX + SUBQ SI, BX + CMPQ BX, $48 + JLT memmove_lit + + MOVOU (SI), X0 + MOVOU 16(SI), X1 + MOVOU 32(SI), X2 + MOVOU X0, (DI) + MOVOU X1, 16(DI) + MOVOU X2, 32(DI) + + ADDQ CX, SI + ADDQ CX, DI + + JMP finish_lit_copy + +memmove_lit: + // memmove(to, from, len) + MOVQ DI, 0(SP) + MOVQ SI, 8(SP) + MOVQ CX, 16(SP) + + // Spill registers. Increment SI, DI now so we don't need to save CX. + ADDQ CX, DI + ADDQ CX, SI + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + MOVL DX, 40(SP) + + CALL runtimeยทmemmove(SB) + + // restore registers + MOVQ 24(SP), DI + MOVQ 32(SP), SI + MOVL 40(SP), DX + + // recalc initial values + MOVQ dst_base+0(FP), R8 + MOVQ R8, R11 + ADDQ dst_len+8(FP), R8 + MOVQ src_base+24(FP), R9 + ADDQ src_len+32(FP), R9 + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + MOVQ R8, R12 + SUBQ $32, R12 + MOVQ R9, R13 + SUBQ $16, R13 + +finish_lit_copy: + // CX := mLen + // free up DX to use for offset + MOVL DX, CX + ANDL $0xF, CX + + CMPQ SI, R9 + JAE end + + // offset + // si += 2 + // DX := int(src[si-2]) | int(src[si-1])<<8 + ADDQ $2, SI + JC err_short_buf + CMPQ SI, R9 + JA err_short_buf + MOVWQZX -2(SI), DX + + // 0 offset is invalid + TESTL DX, DX + JEQ err_corrupt + +match_len_loop_pre: + // if mlen != 0xF + CMPB CX, $0xF + JNE copy_match + + // do { BX = src[si++]; mlen += BX } while (BX == 0xFF). +match_len_loop: + CMPQ SI, R9 + JAE err_short_buf + + MOVBLZX (SI), BX + INCQ SI + ADDQ BX, CX + + CMPB BX, $0xFF + JE match_len_loop + +copy_match: + ADDQ $const_minMatch, CX + + // check we have match_len bytes left in dst + // di+match_len < len(dst) + MOVQ DI, AX + ADDQ CX, AX + JC err_short_buf + CMPQ AX, R8 + JA err_short_buf + + // DX = offset + // CX = match_len + // BX = &dst + (di - offset) + MOVQ DI, BX + SUBQ DX, BX + + // check BX is within dst + // if BX < &dst + JC copy_match_from_dict + CMPQ BX, R11 + JBE copy_match_from_dict + + // if offset + match_len < di + LEAQ (BX)(CX*1), AX + CMPQ DI, AX + JA copy_interior_match + + // AX := len(dst[:di]) + // MOVQ DI, AX + // SUBQ R11, AX + + // copy 16 bytes at a time + // if di-offset < 16 copy 16-(di-offset) bytes to di + // then do the remaining + +copy_match_loop: + // for match_len >= 0 + // dst[di] = dst[i] + // di++ + // i++ + MOVB (BX), AX + MOVB AX, (DI) + INCQ DI + INCQ BX + DECQ CX + JNZ copy_match_loop + + JMP loopcheck + +copy_interior_match: + CMPQ CX, $16 + JGT memmove_match + + // if len(dst[di:]) < 16 + MOVQ R8, AX + SUBQ DI, AX + CMPQ AX, $16 + JLT memmove_match + + MOVOU (BX), X0 + MOVOU X0, (DI) + + ADDQ CX, DI + XORL CX, CX + JMP loopcheck + +copy_match_from_dict: + // CX = match_len + // BX = &dst + (di - offset) + + // AX = offset - di = dict_bytes_available => count of bytes potentially covered by the dictionary + MOVQ R11, AX + SUBQ BX, AX + + // BX = len(dict) - dict_bytes_available + MOVQ R15, BX + SUBQ AX, BX + JS err_short_dict + + ADDQ R14, BX + + // if match_len > dict_bytes_available, match fits entirely within external dictionary : just copy + CMPQ CX, AX + JLT memmove_match + + // The match stretches over the dictionary and our block + // 1) copy what comes from the dictionary + // AX = dict_bytes_available = copy_size + // BX = &dict_end - copy_size + // CX = match_len + + // memmove(to, from, len) + MOVQ DI, 0(SP) + MOVQ BX, 8(SP) + MOVQ AX, 16(SP) + // store extra stuff we want to recover + // spill + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + MOVQ CX, 40(SP) + CALL runtimeยทmemmove(SB) + + // restore registers + MOVQ 16(SP), AX // copy_size + MOVQ 24(SP), DI + MOVQ 32(SP), SI + MOVQ 40(SP), CX // match_len + + // recalc initial values + MOVQ dst_base+0(FP), R8 + MOVQ R8, R11 // TODO: make these sensible numbers + ADDQ dst_len+8(FP), R8 + MOVQ src_base+24(FP), R9 + ADDQ src_len+32(FP), R9 + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + MOVQ R8, R12 + SUBQ $32, R12 + MOVQ R9, R13 + SUBQ $16, R13 + + // di+=copy_size + ADDQ AX, DI + + // 2) copy the rest from the current block + // CX = match_len - copy_size = rest_size + SUBQ AX, CX + MOVQ R11, BX + + // check if we have a copy overlap + // AX = &dst + rest_size + MOVQ CX, AX + ADDQ BX, AX + // if &dst + rest_size > di, copy byte by byte + CMPQ AX, DI + + JA copy_match_loop + +memmove_match: + // memmove(to, from, len) + MOVQ DI, 0(SP) + MOVQ BX, 8(SP) + MOVQ CX, 16(SP) + + // Spill registers. Increment DI now so we don't need to save CX. + ADDQ CX, DI + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + + CALL runtimeยทmemmove(SB) + + // restore registers + MOVQ 24(SP), DI + MOVQ 32(SP), SI + + // recalc initial values + MOVQ dst_base+0(FP), R8 + MOVQ R8, R11 // TODO: make these sensible numbers + ADDQ dst_len+8(FP), R8 + MOVQ src_base+24(FP), R9 + ADDQ src_len+32(FP), R9 + MOVQ R8, R12 + SUBQ $32, R12 + MOVQ R9, R13 + SUBQ $16, R13 + MOVQ dict_base+48(FP), R14 + MOVQ dict_len+56(FP), R15 + XORL CX, CX + +loopcheck: + // for si < len(src) + CMPQ SI, R9 + JB loop + +end: + // Remaining length must be zero. + TESTQ CX, CX + JNE err_corrupt + + SUBQ R11, DI + MOVQ DI, ret+72(FP) + RET + +err_corrupt: + MOVQ $-1, ret+72(FP) + RET + +err_short_buf: + MOVQ $-2, ret+72(FP) + RET + +err_short_dict: + MOVQ $-3, ret+72(FP) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s new file mode 100644 index 0000000000..20b21fcf15 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm.s @@ -0,0 +1,231 @@ +// +build gc +// +build !noasm + +#include "go_asm.h" +#include "textflag.h" + +// Register allocation. +#define dst R0 +#define dstorig R1 +#define src R2 +#define dstend R3 +#define srcend R4 +#define match R5 // Match address. +#define dictend R6 +#define token R7 +#define len R8 // Literal and match lengths. +#define offset R7 // Match offset; overlaps with token. +#define tmp1 R9 +#define tmp2 R11 +#define tmp3 R12 + +// func decodeBlock(dst, src, dict []byte) int +TEXT ยทdecodeBlock(SB), NOFRAME+NOSPLIT, $-4-40 + MOVW dst_base +0(FP), dst + MOVW dst_len +4(FP), dstend + MOVW src_base +12(FP), src + MOVW src_len +16(FP), srcend + + CMP $0, srcend + BEQ shortSrc + + ADD dst, dstend + ADD src, srcend + + MOVW dst, dstorig + +loop: + // Read token. Extract literal length. + MOVBU.P 1(src), token + MOVW token >> 4, len + CMP $15, len + BNE readLitlenDone + +readLitlenLoop: + CMP src, srcend + BEQ shortSrc + MOVBU.P 1(src), tmp1 + ADD.S tmp1, len + BVS shortDst + CMP $255, tmp1 + BEQ readLitlenLoop + +readLitlenDone: + CMP $0, len + BEQ copyLiteralDone + + // Bounds check dst+len and src+len. + ADD.S dst, len, tmp1 + ADD.CC.S src, len, tmp2 + BCS shortSrc + CMP dstend, tmp1 + //BHI shortDst // Uncomment for distinct error codes. + CMP.LS srcend, tmp2 + BHI shortSrc + + // Copy literal. + CMP $4, len + BLO copyLiteralFinish + + // Copy 0-3 bytes until src is aligned. + TST $1, src + MOVBU.NE.P 1(src), tmp1 + MOVB.NE.P tmp1, 1(dst) + SUB.NE $1, len + + TST $2, src + MOVHU.NE.P 2(src), tmp2 + MOVB.NE.P tmp2, 1(dst) + MOVW.NE tmp2 >> 8, tmp1 + MOVB.NE.P tmp1, 1(dst) + SUB.NE $2, len + + B copyLiteralLoopCond + +copyLiteralLoop: + // Aligned load, unaligned write. + MOVW.P 4(src), tmp1 + MOVW tmp1 >> 8, tmp2 + MOVB tmp2, 1(dst) + MOVW tmp1 >> 16, tmp3 + MOVB tmp3, 2(dst) + MOVW tmp1 >> 24, tmp2 + MOVB tmp2, 3(dst) + MOVB.P tmp1, 4(dst) +copyLiteralLoopCond: + // Loop until len-4 < 0. + SUB.S $4, len + BPL copyLiteralLoop + +copyLiteralFinish: + // Copy remaining 0-3 bytes. + // At this point, len may be < 0, but len&3 is still accurate. + TST $1, len + MOVB.NE.P 1(src), tmp3 + MOVB.NE.P tmp3, 1(dst) + TST $2, len + MOVB.NE.P 2(src), tmp1 + MOVB.NE.P tmp1, 2(dst) + MOVB.NE -1(src), tmp2 + MOVB.NE tmp2, -1(dst) + +copyLiteralDone: + // Initial part of match length. + // This frees up the token register for reuse as offset. + AND $15, token, len + + CMP src, srcend + BEQ end + + // Read offset. + ADD.S $2, src + BCS shortSrc + CMP srcend, src + BHI shortSrc + MOVBU -2(src), offset + MOVBU -1(src), tmp1 + ORR.S tmp1 << 8, offset + BEQ corrupt + + // Read rest of match length. + CMP $15, len + BNE readMatchlenDone + +readMatchlenLoop: + CMP src, srcend + BEQ shortSrc + MOVBU.P 1(src), tmp1 + ADD.S tmp1, len + BVS shortDst + CMP $255, tmp1 + BEQ readMatchlenLoop + +readMatchlenDone: + // Bounds check dst+len+minMatch. + ADD.S dst, len, tmp1 + ADD.CC.S $const_minMatch, tmp1 + BCS shortDst + CMP dstend, tmp1 + BHI shortDst + + RSB dst, offset, match + CMP dstorig, match + BGE copyMatch4 + + // match < dstorig means the match starts in the dictionary, + // at len(dict) - offset + (dst - dstorig). + MOVW dict_base+24(FP), match + MOVW dict_len +28(FP), dictend + + ADD $const_minMatch, len + + RSB dst, dstorig, tmp1 + RSB dictend, offset, tmp2 + ADD.S tmp2, tmp1 + BMI shortDict + ADD match, dictend + ADD tmp1, match + +copyDict: + MOVBU.P 1(match), tmp1 + MOVB.P tmp1, 1(dst) + SUB.S $1, len + CMP.NE match, dictend + BNE copyDict + + // If the match extends beyond the dictionary, the rest is at dstorig. + CMP $0, len + BEQ copyMatchDone + MOVW dstorig, match + B copyMatch + + // Copy a regular match. + // Since len+minMatch is at least four, we can do a 4ร— unrolled + // byte copy loop. Using MOVW instead of four byte loads is faster, + // but to remain portable we'd have to align match first, which is + // too expensive. By alternating loads and stores, we also handle + // the case offset < 4. +copyMatch4: + SUB.S $4, len + MOVBU.P 4(match), tmp1 + MOVB.P tmp1, 4(dst) + MOVBU -3(match), tmp2 + MOVB tmp2, -3(dst) + MOVBU -2(match), tmp3 + MOVB tmp3, -2(dst) + MOVBU -1(match), tmp1 + MOVB tmp1, -1(dst) + BPL copyMatch4 + + // Restore len, which is now negative. + ADD.S $4, len + BEQ copyMatchDone + +copyMatch: + // Finish with a byte-at-a-time copy. + SUB.S $1, len + MOVBU.P 1(match), tmp2 + MOVB.P tmp2, 1(dst) + BNE copyMatch + +copyMatchDone: + CMP src, srcend + BNE loop + +end: + CMP $0, len + BNE corrupt + SUB dstorig, dst, tmp1 + MOVW tmp1, ret+36(FP) + RET + + // The error cases have distinct labels so we can put different + // return codes here when debugging, or if the error returns need to + // be changed. +shortDict: +shortDst: +shortSrc: +corrupt: + MOVW $-1, tmp1 + MOVW tmp1, ret+36(FP) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm64.s b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm64.s new file mode 100644 index 0000000000..d2fe11b8ea --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_arm64.s @@ -0,0 +1,241 @@ +// +build gc +// +build !noasm + +// This implementation assumes that strict alignment checking is turned off. +// The Go compiler makes the same assumption. + +#include "go_asm.h" +#include "textflag.h" + +// Register allocation. +#define dst R0 +#define dstorig R1 +#define src R2 +#define dstend R3 +#define dstend16 R4 // dstend - 16 +#define srcend R5 +#define srcend16 R6 // srcend - 16 +#define match R7 // Match address. +#define dict R8 +#define dictlen R9 +#define dictend R10 +#define token R11 +#define len R12 // Literal and match lengths. +#define lenRem R13 +#define offset R14 // Match offset. +#define tmp1 R15 +#define tmp2 R16 +#define tmp3 R17 +#define tmp4 R19 + +// func decodeBlock(dst, src, dict []byte) int +TEXT ยทdecodeBlock(SB), NOFRAME+NOSPLIT, $0-80 + LDP dst_base+0(FP), (dst, dstend) + ADD dst, dstend + MOVD dst, dstorig + + LDP src_base+24(FP), (src, srcend) + CBZ srcend, shortSrc + ADD src, srcend + + // dstend16 = max(dstend-16, 0) and similarly for srcend16. + SUBS $16, dstend, dstend16 + CSEL LO, ZR, dstend16, dstend16 + SUBS $16, srcend, srcend16 + CSEL LO, ZR, srcend16, srcend16 + + LDP dict_base+48(FP), (dict, dictlen) + ADD dict, dictlen, dictend + +loop: + // Read token. Extract literal length. + MOVBU.P 1(src), token + LSR $4, token, len + CMP $15, len + BNE readLitlenDone + +readLitlenLoop: + CMP src, srcend + BEQ shortSrc + MOVBU.P 1(src), tmp1 + ADDS tmp1, len + BVS shortDst + CMP $255, tmp1 + BEQ readLitlenLoop + +readLitlenDone: + CBZ len, copyLiteralDone + + // Bounds check dst+len and src+len. + ADDS dst, len, tmp1 + BCS shortSrc + ADDS src, len, tmp2 + BCS shortSrc + CMP dstend, tmp1 + BHI shortDst + CMP srcend, tmp2 + BHI shortSrc + + // Copy literal. + SUBS $16, len + BLO copyLiteralShort + +copyLiteralLoop: + LDP.P 16(src), (tmp1, tmp2) + STP.P (tmp1, tmp2), 16(dst) + SUBS $16, len + BPL copyLiteralLoop + + // Copy (final part of) literal of length 0-15. + // If we have >=16 bytes left in src and dst, just copy 16 bytes. +copyLiteralShort: + CMP dstend16, dst + CCMP LO, src, srcend16, $0b0010 // 0010 = preserve carry (LO). + BHS copyLiteralShortEnd + + AND $15, len + + LDP (src), (tmp1, tmp2) + ADD len, src + STP (tmp1, tmp2), (dst) + ADD len, dst + + B copyLiteralDone + + // Safe but slow copy near the end of src, dst. +copyLiteralShortEnd: + TBZ $3, len, 3(PC) + MOVD.P 8(src), tmp1 + MOVD.P tmp1, 8(dst) + TBZ $2, len, 3(PC) + MOVW.P 4(src), tmp2 + MOVW.P tmp2, 4(dst) + TBZ $1, len, 3(PC) + MOVH.P 2(src), tmp3 + MOVH.P tmp3, 2(dst) + TBZ $0, len, 3(PC) + MOVBU.P 1(src), tmp4 + MOVB.P tmp4, 1(dst) + +copyLiteralDone: + // Initial part of match length. + AND $15, token, len + + CMP src, srcend + BEQ end + + // Read offset. + ADDS $2, src + BCS shortSrc + CMP srcend, src + BHI shortSrc + MOVHU -2(src), offset + CBZ offset, corrupt + + // Read rest of match length. + CMP $15, len + BNE readMatchlenDone + +readMatchlenLoop: + CMP src, srcend + BEQ shortSrc + MOVBU.P 1(src), tmp1 + ADDS tmp1, len + BVS shortDst + CMP $255, tmp1 + BEQ readMatchlenLoop + +readMatchlenDone: + ADD $const_minMatch, len + + // Bounds check dst+len. + ADDS dst, len, tmp2 + BCS shortDst + CMP dstend, tmp2 + BHI shortDst + + SUB offset, dst, match + CMP dstorig, match + BHS copyMatchTry8 + + // match < dstorig means the match starts in the dictionary, + // at len(dict) - offset + (dst - dstorig). + SUB dstorig, dst, tmp1 + SUB offset, dictlen, tmp2 + ADDS tmp2, tmp1 + BMI shortDict + ADD dict, tmp1, match + +copyDict: + MOVBU.P 1(match), tmp3 + MOVB.P tmp3, 1(dst) + SUBS $1, len + CCMP NE, dictend, match, $0b0100 // 0100 sets the Z (EQ) flag. + BNE copyDict + + CBZ len, copyMatchDone + + // If the match extends beyond the dictionary, the rest is at dstorig. + // Recompute the offset for the next check. + MOVD dstorig, match + SUB dstorig, dst, offset + +copyMatchTry8: + // Copy doublewords if both len and offset are at least eight. + // A 16-at-a-time loop doesn't provide a further speedup. + CMP $8, len + CCMP HS, offset, $8, $0 + BLO copyMatchTry4 + + AND $7, len, lenRem + SUB $8, len +copyMatchLoop8: + MOVD.P 8(match), tmp1 + MOVD.P tmp1, 8(dst) + SUBS $8, len + BPL copyMatchLoop8 + + MOVD (match)(len), tmp2 // match+len == match+lenRem-8. + ADD lenRem, dst + MOVD $0, len + MOVD tmp2, -8(dst) + B copyMatchDone + +copyMatchTry4: + // Copy words if both len and offset are at least four. + CMP $4, len + CCMP HS, offset, $4, $0 + BLO copyMatchLoop1 + + MOVWU.P 4(match), tmp2 + MOVWU.P tmp2, 4(dst) + SUBS $4, len + BEQ copyMatchDone + +copyMatchLoop1: + // Byte-at-a-time copy for small offsets <= 3. + MOVBU.P 1(match), tmp2 + MOVB.P tmp2, 1(dst) + SUBS $1, len + BNE copyMatchLoop1 + +copyMatchDone: + CMP src, srcend + BNE loop + +end: + CBNZ len, corrupt + SUB dstorig, dst, tmp1 + MOVD tmp1, ret+72(FP) + RET + + // The error cases have distinct labels so we can put different + // return codes here when debugging, or if the error returns need to + // be changed. +shortDict: +shortDst: +shortSrc: +corrupt: + MOVD $-1, tmp1 + MOVD tmp1, ret+72(FP) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go new file mode 100644 index 0000000000..8d9023d100 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_asm.go @@ -0,0 +1,10 @@ +//go:build (amd64 || arm || arm64) && !appengine && gc && !noasm +// +build amd64 arm arm64 +// +build !appengine +// +build gc +// +build !noasm + +package lz4block + +//go:noescape +func decodeBlock(dst, src, dict []byte) int diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go new file mode 100644 index 0000000000..9f568fbb1a --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4block/decode_other.go @@ -0,0 +1,139 @@ +//go:build (!amd64 && !arm && !arm64) || appengine || !gc || noasm +// +build !amd64,!arm,!arm64 appengine !gc noasm + +package lz4block + +import ( + "encoding/binary" +) + +func decodeBlock(dst, src, dict []byte) (ret int) { + // Restrict capacities so we don't read or write out of bounds. + dst = dst[:len(dst):len(dst)] + src = src[:len(src):len(src)] + + const hasError = -2 + + if len(src) == 0 { + return hasError + } + + defer func() { + if recover() != nil { + ret = hasError + } + }() + + var si, di uint + for si < uint(len(src)) { + // Literals and match lengths (token). + b := uint(src[si]) + si++ + + // Literals. + if lLen := b >> 4; lLen > 0 { + switch { + case lLen < 0xF && si+16 < uint(len(src)): + // Shortcut 1 + // if we have enough room in src and dst, and the literals length + // is small enough (0..14) then copy all 16 bytes, even if not all + // are part of the literals. + copy(dst[di:], src[si:si+16]) + si += lLen + di += lLen + if mLen := b & 0xF; mLen < 0xF { + // Shortcut 2 + // if the match length (4..18) fits within the literals, then copy + // all 18 bytes, even if not all are part of the literals. + mLen += 4 + if offset := u16(src[si:]); mLen <= offset && offset < di { + i := di - offset + // The remaining buffer may not hold 18 bytes. + // See https://github.com/pierrec/lz4/issues/51. + if end := i + 18; end <= uint(len(dst)) { + copy(dst[di:], dst[i:end]) + si += 2 + di += mLen + continue + } + } + } + case lLen == 0xF: + for { + x := uint(src[si]) + if lLen += x; int(lLen) < 0 { + return hasError + } + si++ + if x != 0xFF { + break + } + } + fallthrough + default: + copy(dst[di:di+lLen], src[si:si+lLen]) + si += lLen + di += lLen + } + } + + mLen := b & 0xF + if si == uint(len(src)) && mLen == 0 { + break + } else if si >= uint(len(src)) { + return hasError + } + + offset := u16(src[si:]) + if offset == 0 { + return hasError + } + si += 2 + + // Match. + mLen += minMatch + if mLen == minMatch+0xF { + for { + x := uint(src[si]) + if mLen += x; int(mLen) < 0 { + return hasError + } + si++ + if x != 0xFF { + break + } + } + } + + // Copy the match. + if di < offset { + // The match is beyond our block, meaning the first part + // is in the dictionary. + fromDict := dict[uint(len(dict))+di-offset:] + n := uint(copy(dst[di:di+mLen], fromDict)) + di += n + if mLen -= n; mLen == 0 { + continue + } + // We copied n = offset-di bytes from the dictionary, + // then set di = di+n = offset, so the following code + // copies from dst[di-offset:] = dst[0:]. + } + + expanded := dst[di-offset:] + if mLen > offset { + // Efficiently copy the match dst[di-offset:di] into the dst slice. + bytesToCopy := offset * (mLen / offset) + for n := offset; n <= bytesToCopy+offset; n *= 2 { + copy(expanded[n:], expanded[:n]) + } + di += bytesToCopy + mLen -= bytesToCopy + } + di += uint(copy(dst[di:di+mLen], expanded[:mLen])) + } + + return int(di) +} + +func u16(p []byte) uint { return uint(binary.LittleEndian.Uint16(p)) } diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go new file mode 100644 index 0000000000..710ea42812 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4errors/errors.go @@ -0,0 +1,19 @@ +package lz4errors + +type Error string + +func (e Error) Error() string { return string(e) } + +const ( + ErrInvalidSourceShortBuffer Error = "lz4: invalid source or destination buffer too short" + ErrInvalidFrame Error = "lz4: bad magic number" + ErrInternalUnhandledState Error = "lz4: unhandled state" + ErrInvalidHeaderChecksum Error = "lz4: invalid header checksum" + ErrInvalidBlockChecksum Error = "lz4: invalid block checksum" + ErrInvalidFrameChecksum Error = "lz4: invalid frame checksum" + ErrOptionInvalidCompressionLevel Error = "lz4: invalid compression level" + ErrOptionClosedOrError Error = "lz4: cannot apply options on closed or in error object" + ErrOptionInvalidBlockSize Error = "lz4: invalid block size" + ErrOptionNotApplicable Error = "lz4: option not applicable" + ErrWriterNotClosed Error = "lz4: writer not closed" +) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go new file mode 100644 index 0000000000..e96465460c --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/block.go @@ -0,0 +1,348 @@ +package lz4stream + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/xxh32" +) + +type Blocks struct { + Block *FrameDataBlock + Blocks chan chan *FrameDataBlock + mu sync.Mutex + err error +} + +func (b *Blocks) initW(f *Frame, dst io.Writer, num int) { + if num == 1 { + b.Blocks = nil + b.Block = NewFrameDataBlock(f) + return + } + b.Block = nil + if cap(b.Blocks) != num { + b.Blocks = make(chan chan *FrameDataBlock, num) + } + // goroutine managing concurrent block compression goroutines. + go func() { + // Process next block compression item. + for c := range b.Blocks { + // Read the next compressed block result. + // Waiting here ensures that the blocks are output in the order they were sent. + // The incoming channel is always closed as it indicates to the caller that + // the block has been processed. + block := <-c + if block == nil { + // Notify the block compression routine that we are done with its result. + // This is used when a sentinel block is sent to terminate the compression. + close(c) + return + } + // Do not attempt to write the block upon any previous failure. + if b.err == nil { + // Write the block. + if err := block.Write(f, dst); err != nil { + // Keep the first error. + b.err = err + // All pending compression goroutines need to shut down, so we need to keep going. + } + } + close(c) + } + }() +} + +func (b *Blocks) close(f *Frame, num int) error { + if num == 1 { + if b.Block != nil { + b.Block.Close(f) + } + err := b.err + b.err = nil + return err + } + if b.Blocks == nil { + err := b.err + b.err = nil + return err + } + c := make(chan *FrameDataBlock) + b.Blocks <- c + c <- nil + <-c + err := b.err + b.err = nil + return err +} + +// ErrorR returns any error set while uncompressing a stream. +func (b *Blocks) ErrorR() error { + b.mu.Lock() + defer b.mu.Unlock() + return b.err +} + +// initR returns a channel that streams the uncompressed blocks if in concurrent +// mode and no error. When the channel is closed, check for any error with b.ErrorR. +// +// If not in concurrent mode, the uncompressed block is b.Block and the returned error +// needs to be checked. +func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) { + size := f.Descriptor.Flags.BlockSizeIndex() + if num == 1 { + b.Blocks = nil + b.Block = NewFrameDataBlock(f) + return nil, nil + } + b.Block = nil + blocks := make(chan chan []byte, num) + // data receives the uncompressed blocks. + data := make(chan []byte) + // Read blocks from the source sequentially + // and uncompress them concurrently. + + // In legacy mode, accrue the uncompress sizes in cum. + var cum uint32 + go func() { + var cumx uint32 + var err error + for b.ErrorR() == nil { + block := NewFrameDataBlock(f) + cumx, err = block.Read(f, src, 0) + if err != nil { + block.Close(f) + break + } + // Recheck for an error as reading may be slow and uncompressing is expensive. + if b.ErrorR() != nil { + block.Close(f) + break + } + c := make(chan []byte) + blocks <- c + go func() { + defer block.Close(f) + data, err := block.Uncompress(f, size.Get(), nil, false) + if err != nil { + b.closeR(err) + // Close the block channel to indicate an error. + close(c) + } else { + c <- data + } + }() + } + // End the collection loop and the data channel. + c := make(chan []byte) + blocks <- c + c <- nil // signal the collection loop that we are done + <-c // wait for the collect loop to complete + if f.isLegacy() && cum == cumx { + err = io.EOF + } + b.closeR(err) + close(data) + }() + // Collect the uncompressed blocks and make them available + // on the returned channel. + go func(leg bool) { + defer close(blocks) + skipBlocks := false + for c := range blocks { + buf, ok := <-c + if !ok { + // A closed channel indicates an error. + // All remaining channels should be discarded. + skipBlocks = true + continue + } + if buf == nil { + // Signal to end the loop. + close(c) + return + } + if skipBlocks { + // A previous error has occurred, skipping remaining channels. + continue + } + // Perform checksum now as the blocks are received in order. + if f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(buf) + } + if leg { + cum += uint32(len(buf)) + } + data <- buf + close(c) + } + }(f.isLegacy()) + return data, nil +} + +// closeR safely sets the error on b if not already set. +func (b *Blocks) closeR(err error) { + b.mu.Lock() + if b.err == nil { + b.err = err + } + b.mu.Unlock() +} + +func NewFrameDataBlock(f *Frame) *FrameDataBlock { + buf := f.Descriptor.Flags.BlockSizeIndex().Get() + return &FrameDataBlock{Data: buf, data: buf} +} + +type FrameDataBlock struct { + Size DataBlockSize + Data []byte // compressed or uncompressed data (.data or .src) + Checksum uint32 + data []byte // buffer for compressed data + src []byte // uncompressed data + err error // used in concurrent mode +} + +func (b *FrameDataBlock) Close(f *Frame) { + b.Size = 0 + b.Checksum = 0 + b.err = nil + if b.data != nil { + // Block was not already closed. + lz4block.Put(b.data) + b.Data = nil + b.data = nil + b.src = nil + } +} + +// Block compression errors are ignored since the buffer is sized appropriately. +func (b *FrameDataBlock) Compress(f *Frame, src []byte, level lz4block.CompressionLevel) *FrameDataBlock { + data := b.data + if f.isLegacy() { + data = data[:cap(data)] + } else { + data = data[:len(src)] // trigger the incompressible flag in CompressBlock + } + var n int + switch level { + case lz4block.Fast: + n, _ = lz4block.CompressBlock(src, data) + default: + n, _ = lz4block.CompressBlockHC(src, data, level) + } + if n == 0 { + b.Size.UncompressedSet(true) + b.Data = src + } else { + b.Size.UncompressedSet(false) + b.Data = data[:n] + } + b.Size.sizeSet(len(b.Data)) + b.src = src // keep track of the source for content checksum + + if f.Descriptor.Flags.BlockChecksum() { + b.Checksum = xxh32.ChecksumZero(src) + } + return b +} + +func (b *FrameDataBlock) Write(f *Frame, dst io.Writer) error { + // Write is called in the same order as blocks are compressed, + // so content checksum must be done here. + if f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(b.src) + } + buf := f.buf[:] + binary.LittleEndian.PutUint32(buf, uint32(b.Size)) + if _, err := dst.Write(buf[:4]); err != nil { + return err + } + + if _, err := dst.Write(b.Data); err != nil { + return err + } + + if b.Checksum == 0 { + return nil + } + binary.LittleEndian.PutUint32(buf, b.Checksum) + _, err := dst.Write(buf[:4]) + return err +} + +// Read updates b with the next block data, size and checksum if available. +func (b *FrameDataBlock) Read(f *Frame, src io.Reader, cum uint32) (uint32, error) { + x, err := f.readUint32(src) + if err != nil { + return 0, err + } + if f.isLegacy() { + switch x { + case frameMagicLegacy: + // Concatenated legacy frame. + return b.Read(f, src, cum) + case cum: + // Only works in non concurrent mode, for concurrent mode + // it is handled separately. + // Linux kernel format appends the total uncompressed size at the end. + return 0, io.EOF + } + } else if x == 0 { + // Marker for end of stream. + return 0, io.EOF + } + b.Size = DataBlockSize(x) + + size := b.Size.size() + if size > cap(b.data) { + return x, lz4errors.ErrOptionInvalidBlockSize + } + b.data = b.data[:size] + if _, err := io.ReadFull(src, b.data); err != nil { + return x, err + } + if f.Descriptor.Flags.BlockChecksum() { + sum, err := f.readUint32(src) + if err != nil { + return 0, err + } + b.Checksum = sum + } + return x, nil +} + +func (b *FrameDataBlock) Uncompress(f *Frame, dst, dict []byte, sum bool) ([]byte, error) { + if b.Size.Uncompressed() { + n := copy(dst, b.data) + dst = dst[:n] + } else { + n, err := lz4block.UncompressBlock(b.data, dst, dict) + if err != nil { + return nil, err + } + dst = dst[:n] + } + if f.Descriptor.Flags.BlockChecksum() { + if c := xxh32.ChecksumZero(dst); c != b.Checksum { + err := fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidBlockChecksum, c, b.Checksum) + return nil, err + } + } + if sum && f.Descriptor.Flags.ContentChecksum() { + _, _ = f.checksum.Write(dst) + } + return dst, nil +} + +func (f *Frame) readUint32(r io.Reader) (x uint32, err error) { + if _, err = io.ReadFull(r, f.buf[:4]); err != nil { + return + } + x = binary.LittleEndian.Uint32(f.buf[:4]) + return +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go new file mode 100644 index 0000000000..18192a9433 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame.go @@ -0,0 +1,204 @@ +// Package lz4stream provides the types that support reading and writing LZ4 data streams. +package lz4stream + +import ( + "encoding/binary" + "fmt" + "io" + "io/ioutil" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/xxh32" +) + +//go:generate go run gen.go + +const ( + frameMagic uint32 = 0x184D2204 + frameSkipMagic uint32 = 0x184D2A50 + frameMagicLegacy uint32 = 0x184C2102 +) + +func NewFrame() *Frame { + return &Frame{} +} + +type Frame struct { + buf [15]byte // frame descriptor needs at most 4(magic)+4+8+1=11 bytes + Magic uint32 + Descriptor FrameDescriptor + Blocks Blocks + Checksum uint32 + checksum xxh32.XXHZero +} + +// Reset allows reusing the Frame. +// The Descriptor configuration is not modified. +func (f *Frame) Reset(num int) { + f.Magic = 0 + f.Descriptor.Checksum = 0 + f.Descriptor.ContentSize = 0 + _ = f.Blocks.close(f, num) + f.Checksum = 0 +} + +func (f *Frame) InitW(dst io.Writer, num int, legacy bool) { + if legacy { + f.Magic = frameMagicLegacy + idx := lz4block.Index(lz4block.Block8Mb) + f.Descriptor.Flags.BlockSizeIndexSet(idx) + } else { + f.Magic = frameMagic + f.Descriptor.initW() + } + f.Blocks.initW(f, dst, num) + f.checksum.Reset() +} + +func (f *Frame) CloseW(dst io.Writer, num int) error { + if err := f.Blocks.close(f, num); err != nil { + return err + } + if f.isLegacy() { + return nil + } + buf := f.buf[:0] + // End mark (data block size of uint32(0)). + buf = append(buf, 0, 0, 0, 0) + if f.Descriptor.Flags.ContentChecksum() { + buf = f.checksum.Sum(buf) + } + _, err := dst.Write(buf) + return err +} + +func (f *Frame) isLegacy() bool { + return f.Magic == frameMagicLegacy +} + +func (f *Frame) ParseHeaders(src io.Reader) error { + if f.Magic > 0 { + // Header already read. + return nil + } + +newFrame: + var err error + if f.Magic, err = f.readUint32(src); err != nil { + return err + } + switch m := f.Magic; { + case m == frameMagic || m == frameMagicLegacy: + // All 16 values of frameSkipMagic are valid. + case m>>8 == frameSkipMagic>>8: + skip, err := f.readUint32(src) + if err != nil { + return err + } + if _, err := io.CopyN(ioutil.Discard, src, int64(skip)); err != nil { + return err + } + goto newFrame + default: + return lz4errors.ErrInvalidFrame + } + if err := f.Descriptor.initR(f, src); err != nil { + return err + } + f.checksum.Reset() + return nil +} + +func (f *Frame) InitR(src io.Reader, num int) (chan []byte, error) { + return f.Blocks.initR(f, num, src) +} + +func (f *Frame) CloseR(src io.Reader) (err error) { + if f.isLegacy() { + return nil + } + if !f.Descriptor.Flags.ContentChecksum() { + return nil + } + if f.Checksum, err = f.readUint32(src); err != nil { + return err + } + if c := f.checksum.Sum32(); c != f.Checksum { + return fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidFrameChecksum, c, f.Checksum) + } + return nil +} + +type FrameDescriptor struct { + Flags DescriptorFlags + ContentSize uint64 + Checksum uint8 +} + +func (fd *FrameDescriptor) initW() { + fd.Flags.VersionSet(1) + fd.Flags.BlockIndependenceSet(true) +} + +func (fd *FrameDescriptor) Write(f *Frame, dst io.Writer) error { + if fd.Checksum > 0 { + // Header already written. + return nil + } + + buf := f.buf[:4] + // Write the magic number here even though it belongs to the Frame. + binary.LittleEndian.PutUint32(buf, f.Magic) + if !f.isLegacy() { + buf = buf[:4+2] + binary.LittleEndian.PutUint16(buf[4:], uint16(fd.Flags)) + + if fd.Flags.Size() { + buf = buf[:4+2+8] + binary.LittleEndian.PutUint64(buf[4+2:], fd.ContentSize) + } + fd.Checksum = descriptorChecksum(buf[4:]) + buf = append(buf, fd.Checksum) + } + + _, err := dst.Write(buf) + return err +} + +func (fd *FrameDescriptor) initR(f *Frame, src io.Reader) error { + if f.isLegacy() { + idx := lz4block.Index(lz4block.Block8Mb) + f.Descriptor.Flags.BlockSizeIndexSet(idx) + return nil + } + // Read the flags and the checksum, hoping that there is not content size. + buf := f.buf[:3] + if _, err := io.ReadFull(src, buf); err != nil { + return err + } + descr := binary.LittleEndian.Uint16(buf) + fd.Flags = DescriptorFlags(descr) + if fd.Flags.Size() { + // Append the 8 missing bytes. + buf = buf[:3+8] + if _, err := io.ReadFull(src, buf[3:]); err != nil { + return err + } + fd.ContentSize = binary.LittleEndian.Uint64(buf[2:]) + } + fd.Checksum = buf[len(buf)-1] // the checksum is the last byte + buf = buf[:len(buf)-1] // all descriptor fields except checksum + if c := descriptorChecksum(buf); fd.Checksum != c { + return fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidHeaderChecksum, c, fd.Checksum) + } + // Validate the elements that can be. + if idx := fd.Flags.BlockSizeIndex(); !idx.IsValid() { + return lz4errors.ErrOptionInvalidBlockSize + } + return nil +} + +func descriptorChecksum(buf []byte) byte { + return byte(xxh32.ChecksumZero(buf) >> 8) +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go new file mode 100644 index 0000000000..d33a6be95c --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/lz4stream/frame_gen.go @@ -0,0 +1,103 @@ +// Code generated by `gen.exe`. DO NOT EDIT. + +package lz4stream + +import "github.com/pierrec/lz4/v4/internal/lz4block" + +// DescriptorFlags is defined as follow: +// field bits +// ----- ---- +// _ 2 +// ContentChecksum 1 +// Size 1 +// BlockChecksum 1 +// BlockIndependence 1 +// Version 2 +// _ 4 +// BlockSizeIndex 3 +// _ 1 +type DescriptorFlags uint16 + +// Getters. +func (x DescriptorFlags) ContentChecksum() bool { return x>>2&1 != 0 } +func (x DescriptorFlags) Size() bool { return x>>3&1 != 0 } +func (x DescriptorFlags) BlockChecksum() bool { return x>>4&1 != 0 } +func (x DescriptorFlags) BlockIndependence() bool { return x>>5&1 != 0 } +func (x DescriptorFlags) Version() uint16 { return uint16(x >> 6 & 0x3) } +func (x DescriptorFlags) BlockSizeIndex() lz4block.BlockSizeIndex { + return lz4block.BlockSizeIndex(x >> 12 & 0x7) +} + +// Setters. +func (x *DescriptorFlags) ContentChecksumSet(v bool) *DescriptorFlags { + const b = 1 << 2 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) SizeSet(v bool) *DescriptorFlags { + const b = 1 << 3 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) BlockChecksumSet(v bool) *DescriptorFlags { + const b = 1 << 4 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) BlockIndependenceSet(v bool) *DescriptorFlags { + const b = 1 << 5 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} +func (x *DescriptorFlags) VersionSet(v uint16) *DescriptorFlags { + *x = *x&^(0x3<<6) | (DescriptorFlags(v) & 0x3 << 6) + return x +} +func (x *DescriptorFlags) BlockSizeIndexSet(v lz4block.BlockSizeIndex) *DescriptorFlags { + *x = *x&^(0x7<<12) | (DescriptorFlags(v) & 0x7 << 12) + return x +} + +// Code generated by `gen.exe`. DO NOT EDIT. + +// DataBlockSize is defined as follow: +// field bits +// ----- ---- +// size 31 +// Uncompressed 1 +type DataBlockSize uint32 + +// Getters. +func (x DataBlockSize) size() int { return int(x & 0x7FFFFFFF) } +func (x DataBlockSize) Uncompressed() bool { return x>>31&1 != 0 } + +// Setters. +func (x *DataBlockSize) sizeSet(v int) *DataBlockSize { + *x = *x&^0x7FFFFFFF | DataBlockSize(v)&0x7FFFFFFF + return x +} +func (x *DataBlockSize) UncompressedSet(v bool) *DataBlockSize { + const b = 1 << 31 + if v { + *x = *x&^b | b + } else { + *x &^= b + } + return x +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go new file mode 100644 index 0000000000..651d10c104 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero.go @@ -0,0 +1,212 @@ +// Package xxh32 implements the very fast XXH hashing algorithm (32 bits version). +// (ported from the reference implementation https://github.com/Cyan4973/xxHash/) +package xxh32 + +import ( + "encoding/binary" +) + +const ( + prime1 uint32 = 2654435761 + prime2 uint32 = 2246822519 + prime3 uint32 = 3266489917 + prime4 uint32 = 668265263 + prime5 uint32 = 374761393 + + primeMask = 0xFFFFFFFF + prime1plus2 = uint32((uint64(prime1) + uint64(prime2)) & primeMask) // 606290984 + prime1minus = uint32((-int64(prime1)) & primeMask) // 1640531535 +) + +// XXHZero represents an xxhash32 object with seed 0. +type XXHZero struct { + v [4]uint32 + totalLen uint64 + buf [16]byte + bufused int +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +func (xxh XXHZero) Sum(b []byte) []byte { + h32 := xxh.Sum32() + return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24)) +} + +// Reset resets the Hash to its initial state. +func (xxh *XXHZero) Reset() { + xxh.v[0] = prime1plus2 + xxh.v[1] = prime2 + xxh.v[2] = 0 + xxh.v[3] = prime1minus + xxh.totalLen = 0 + xxh.bufused = 0 +} + +// Size returns the number of bytes returned by Sum(). +func (xxh *XXHZero) Size() int { + return 4 +} + +// BlockSizeIndex gives the minimum number of bytes accepted by Write(). +func (xxh *XXHZero) BlockSize() int { + return 1 +} + +// Write adds input bytes to the Hash. +// It never returns an error. +func (xxh *XXHZero) Write(input []byte) (int, error) { + if xxh.totalLen == 0 { + xxh.Reset() + } + n := len(input) + m := xxh.bufused + + xxh.totalLen += uint64(n) + + r := len(xxh.buf) - m + if n < r { + copy(xxh.buf[m:], input) + xxh.bufused += len(input) + return n, nil + } + + var buf *[16]byte + if m != 0 { + // some data left from previous update + buf = &xxh.buf + c := copy(buf[m:], input) + n -= c + input = input[c:] + } + update(&xxh.v, buf, input) + xxh.bufused = copy(xxh.buf[:], input[n-n%16:]) + + return n, nil +} + +// Portable version of update. This updates v by processing all of buf +// (if not nil) and all full 16-byte blocks of input. +func updateGo(v *[4]uint32, buf *[16]byte, input []byte) { + // Causes compiler to work directly from registers instead of stack: + v1, v2, v3, v4 := v[0], v[1], v[2], v[3] + + if buf != nil { + v1 = rol13(v1+binary.LittleEndian.Uint32(buf[:])*prime2) * prime1 + v2 = rol13(v2+binary.LittleEndian.Uint32(buf[4:])*prime2) * prime1 + v3 = rol13(v3+binary.LittleEndian.Uint32(buf[8:])*prime2) * prime1 + v4 = rol13(v4+binary.LittleEndian.Uint32(buf[12:])*prime2) * prime1 + } + + for ; len(input) >= 16; input = input[16:] { + sub := input[:16] //BCE hint for compiler + v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1 + v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1 + v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1 + v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1 + } + v[0], v[1], v[2], v[3] = v1, v2, v3, v4 +} + +// Sum32 returns the 32 bits Hash value. +func (xxh *XXHZero) Sum32() uint32 { + h32 := uint32(xxh.totalLen) + if h32 >= 16 { + h32 += rol1(xxh.v[0]) + rol7(xxh.v[1]) + rol12(xxh.v[2]) + rol18(xxh.v[3]) + } else { + h32 += prime5 + } + + p := 0 + n := xxh.bufused + buf := xxh.buf + for n := n - 4; p <= n; p += 4 { + h32 += binary.LittleEndian.Uint32(buf[p:p+4]) * prime3 + h32 = rol17(h32) * prime4 + } + for ; p < n; p++ { + h32 += uint32(buf[p]) * prime5 + h32 = rol11(h32) * prime1 + } + + h32 ^= h32 >> 15 + h32 *= prime2 + h32 ^= h32 >> 13 + h32 *= prime3 + h32 ^= h32 >> 16 + + return h32 +} + +// Portable version of ChecksumZero. +func checksumZeroGo(input []byte) uint32 { + n := len(input) + h32 := uint32(n) + + if n < 16 { + h32 += prime5 + } else { + v1 := prime1plus2 + v2 := prime2 + v3 := uint32(0) + v4 := prime1minus + p := 0 + for n := n - 16; p <= n; p += 16 { + sub := input[p:][:16] //BCE hint for compiler + v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime2) * prime1 + v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime2) * prime1 + v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime2) * prime1 + v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime2) * prime1 + } + input = input[p:] + n -= p + h32 += rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + } + + p := 0 + for n := n - 4; p <= n; p += 4 { + h32 += binary.LittleEndian.Uint32(input[p:p+4]) * prime3 + h32 = rol17(h32) * prime4 + } + for p < n { + h32 += uint32(input[p]) * prime5 + h32 = rol11(h32) * prime1 + p++ + } + + h32 ^= h32 >> 15 + h32 *= prime2 + h32 ^= h32 >> 13 + h32 *= prime3 + h32 ^= h32 >> 16 + + return h32 +} + +func rol1(u uint32) uint32 { + return u<<1 | u>>31 +} + +func rol7(u uint32) uint32 { + return u<<7 | u>>25 +} + +func rol11(u uint32) uint32 { + return u<<11 | u>>21 +} + +func rol12(u uint32) uint32 { + return u<<12 | u>>20 +} + +func rol13(u uint32) uint32 { + return u<<13 | u>>19 +} + +func rol17(u uint32) uint32 { + return u<<17 | u>>15 +} + +func rol18(u uint32) uint32 { + return u<<18 | u>>14 +} diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go new file mode 100644 index 0000000000..0978b2665b --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.go @@ -0,0 +1,11 @@ +// +build !noasm + +package xxh32 + +// ChecksumZero returns the 32-bit hash of input. +// +//go:noescape +func ChecksumZero(input []byte) uint32 + +//go:noescape +func update(v *[4]uint32, buf *[16]byte, input []byte) diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s new file mode 100644 index 0000000000..c18ffd5743 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_arm.s @@ -0,0 +1,251 @@ +// +build !noasm + +#include "go_asm.h" +#include "textflag.h" + +// Register allocation. +#define p R0 +#define n R1 +#define h R2 +#define v1 R2 // Alias for h. +#define v2 R3 +#define v3 R4 +#define v4 R5 +#define x1 R6 +#define x2 R7 +#define x3 R8 +#define x4 R9 + +// We need the primes in registers. The 16-byte loop only uses prime{1,2}. +#define prime1r R11 +#define prime2r R12 +#define prime3r R3 // The rest can alias v{2-4}. +#define prime4r R4 +#define prime5r R5 + +// Update round macros. These read from and increment p. + +#define round16aligned \ + MOVM.IA.W (p), [x1, x2, x3, x4] \ + \ + MULA x1, prime2r, v1, v1 \ + MULA x2, prime2r, v2, v2 \ + MULA x3, prime2r, v3, v3 \ + MULA x4, prime2r, v4, v4 \ + \ + MOVW v1 @> 19, v1 \ + MOVW v2 @> 19, v2 \ + MOVW v3 @> 19, v3 \ + MOVW v4 @> 19, v4 \ + \ + MUL prime1r, v1 \ + MUL prime1r, v2 \ + MUL prime1r, v3 \ + MUL prime1r, v4 \ + +#define round16unaligned \ + MOVBU.P 16(p), x1 \ + MOVBU -15(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -14(p), x3 \ + MOVBU -13(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v1, v1 \ + MOVW v1 @> 19, v1 \ + MUL prime1r, v1 \ + \ + MOVBU -12(p), x1 \ + MOVBU -11(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -10(p), x3 \ + MOVBU -9(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v2, v2 \ + MOVW v2 @> 19, v2 \ + MUL prime1r, v2 \ + \ + MOVBU -8(p), x1 \ + MOVBU -7(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -6(p), x3 \ + MOVBU -5(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v3, v3 \ + MOVW v3 @> 19, v3 \ + MUL prime1r, v3 \ + \ + MOVBU -4(p), x1 \ + MOVBU -3(p), x2 \ + ORR x2 << 8, x1 \ + MOVBU -2(p), x3 \ + MOVBU -1(p), x4 \ + ORR x4 << 8, x3 \ + ORR x3 << 16, x1 \ + \ + MULA x1, prime2r, v4, v4 \ + MOVW v4 @> 19, v4 \ + MUL prime1r, v4 \ + + +// func ChecksumZero([]byte) uint32 +TEXT ยทChecksumZero(SB), NOFRAME|NOSPLIT, $-4-16 + MOVW input_base+0(FP), p + MOVW input_len+4(FP), n + + MOVW $const_prime1, prime1r + MOVW $const_prime2, prime2r + + // Set up h for n < 16. It's tempting to say {ADD prime5, n, h} + // here, but that's a pseudo-op that generates a load through R11. + MOVW $const_prime5, prime5r + ADD prime5r, n, h + CMP $0, n + BEQ end + + // We let n go negative so we can do comparisons with SUB.S + // instead of separate CMP. + SUB.S $16, n + BMI loop16done + + ADD prime1r, prime2r, v1 + MOVW prime2r, v2 + MOVW $0, v3 + RSB $0, prime1r, v4 + + TST $3, p + BNE loop16unaligned + +loop16aligned: + SUB.S $16, n + round16aligned + BPL loop16aligned + B loop16finish + +loop16unaligned: + SUB.S $16, n + round16unaligned + BPL loop16unaligned + +loop16finish: + MOVW v1 @> 31, h + ADD v2 @> 25, h + ADD v3 @> 20, h + ADD v4 @> 14, h + + // h += len(input) with v2 as temporary. + MOVW input_len+4(FP), v2 + ADD v2, h + +loop16done: + ADD $16, n // Restore number of bytes left. + + SUB.S $4, n + MOVW $const_prime3, prime3r + BMI loop4done + MOVW $const_prime4, prime4r + + TST $3, p + BNE loop4unaligned + +loop4aligned: + SUB.S $4, n + + MOVW.P 4(p), x1 + MULA prime3r, x1, h, h + MOVW h @> 15, h + MUL prime4r, h + + BPL loop4aligned + B loop4done + +loop4unaligned: + SUB.S $4, n + + MOVBU.P 4(p), x1 + MOVBU -3(p), x2 + ORR x2 << 8, x1 + MOVBU -2(p), x3 + ORR x3 << 16, x1 + MOVBU -1(p), x4 + ORR x4 << 24, x1 + + MULA prime3r, x1, h, h + MOVW h @> 15, h + MUL prime4r, h + + BPL loop4unaligned + +loop4done: + ADD.S $4, n // Restore number of bytes left. + BEQ end + + MOVW $const_prime5, prime5r + +loop1: + SUB.S $1, n + + MOVBU.P 1(p), x1 + MULA prime5r, x1, h, h + MOVW h @> 21, h + MUL prime1r, h + + BNE loop1 + +end: + MOVW $const_prime3, prime3r + EOR h >> 15, h + MUL prime2r, h + EOR h >> 13, h + MUL prime3r, h + EOR h >> 16, h + + MOVW h, ret+12(FP) + RET + + +// func update(v *[4]uint64, buf *[16]byte, p []byte) +TEXT ยทupdate(SB), NOFRAME|NOSPLIT, $-4-20 + MOVW v+0(FP), p + MOVM.IA (p), [v1, v2, v3, v4] + + MOVW $const_prime1, prime1r + MOVW $const_prime2, prime2r + + // Process buf, if not nil. + MOVW buf+4(FP), p + CMP $0, p + BEQ noBuffered + + round16aligned + +noBuffered: + MOVW input_base +8(FP), p + MOVW input_len +12(FP), n + + SUB.S $16, n + BMI end + + TST $3, p + BNE loop16unaligned + +loop16aligned: + SUB.S $16, n + round16aligned + BPL loop16aligned + B end + +loop16unaligned: + SUB.S $16, n + round16unaligned + BPL loop16unaligned + +end: + MOVW v+0(FP), p + MOVM.IA [v1, v2, v3, v4], (p) + RET diff --git a/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go new file mode 100644 index 0000000000..c96b59b8c3 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/internal/xxh32/xxh32zero_other.go @@ -0,0 +1,10 @@ +// +build !arm noasm + +package xxh32 + +// ChecksumZero returns the 32-bit hash of input. +func ChecksumZero(input []byte) uint32 { return checksumZeroGo(input) } + +func update(v *[4]uint32, buf *[16]byte, input []byte) { + updateGo(v, buf, input) +} diff --git a/vendor/github.com/pierrec/lz4/v4/lz4.go b/vendor/github.com/pierrec/lz4/v4/lz4.go new file mode 100644 index 0000000000..a62022e088 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/lz4.go @@ -0,0 +1,157 @@ +// Package lz4 implements reading and writing lz4 compressed data. +// +// The package supports both the LZ4 stream format, +// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html, +// and the LZ4 block format, defined at +// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html. +// +// See https://github.com/lz4/lz4 for the reference C implementation. +package lz4 + +import ( + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +func _() { + // Safety checks for duplicated elements. + var x [1]struct{} + _ = x[lz4block.CompressionLevel(Fast)-lz4block.Fast] + _ = x[Block64Kb-BlockSize(lz4block.Block64Kb)] + _ = x[Block256Kb-BlockSize(lz4block.Block256Kb)] + _ = x[Block1Mb-BlockSize(lz4block.Block1Mb)] + _ = x[Block4Mb-BlockSize(lz4block.Block4Mb)] +} + +// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible. +func CompressBlockBound(n int) int { + return lz4block.CompressBlockBound(n) +} + +// UncompressBlock uncompresses the source buffer into the destination one, +// and returns the uncompressed size. +// +// The destination buffer must be sized appropriately. +// +// An error is returned if the source data is invalid or the destination buffer is too small. +func UncompressBlock(src, dst []byte) (int, error) { + return lz4block.UncompressBlock(src, dst, nil) +} + +// UncompressBlockWithDict uncompresses the source buffer into the destination one using a +// dictionary, and returns the uncompressed size. +// +// The destination buffer must be sized appropriately. +// +// An error is returned if the source data is invalid or the destination buffer is too small. +func UncompressBlockWithDict(src, dst, dict []byte) (int, error) { + return lz4block.UncompressBlock(src, dst, dict) +} + +// A Compressor compresses data into the LZ4 block format. +// It uses a fast compression algorithm. +// +// A Compressor is not safe for concurrent use by multiple goroutines. +// +// Use a Writer to compress into the LZ4 stream format. +type Compressor struct{ c lz4block.Compressor } + +// CompressBlock compresses the source buffer src into the destination dst. +// +// If compression is successful, the first return value is the size of the +// compressed data, which is always >0. +// +// If dst has length at least CompressBlockBound(len(src)), compression always +// succeeds. Otherwise, the first return value is zero. The error return is +// non-nil if the compressed data does not fit in dst, but it might fit in a +// larger buffer that is still smaller than CompressBlockBound(len(src)). The +// return value (0, nil) means the data is likely incompressible and a buffer +// of length CompressBlockBound(len(src)) should be passed in. +func (c *Compressor) CompressBlock(src, dst []byte) (int, error) { + return c.c.CompressBlock(src, dst) +} + +// CompressBlock compresses the source buffer into the destination one. +// This is the fast version of LZ4 compression and also the default one. +// +// The argument hashTable is scratch space for a hash table used by the +// compressor. If provided, it should have length at least 1<<16. If it is +// shorter (or nil), CompressBlock allocates its own hash table. +// +// The size of the compressed data is returned. +// +// If the destination buffer size is lower than CompressBlockBound and +// the compressed size is 0 and no error, then the data is incompressible. +// +// An error is returned if the destination buffer is too small. + +// CompressBlock is equivalent to Compressor.CompressBlock. +// The final argument is ignored and should be set to nil. +// +// This function is deprecated. Use a Compressor instead. +func CompressBlock(src, dst []byte, _ []int) (int, error) { + return lz4block.CompressBlock(src, dst) +} + +// A CompressorHC compresses data into the LZ4 block format. +// Its compression ratio is potentially better than that of a Compressor, +// but it is also slower and requires more memory. +// +// A Compressor is not safe for concurrent use by multiple goroutines. +// +// Use a Writer to compress into the LZ4 stream format. +type CompressorHC struct { + // Level is the maximum search depth for compression. + // Values <= 0 mean no maximum. + Level CompressionLevel + c lz4block.CompressorHC +} + +// CompressBlock compresses the source buffer src into the destination dst. +// +// If compression is successful, the first return value is the size of the +// compressed data, which is always >0. +// +// If dst has length at least CompressBlockBound(len(src)), compression always +// succeeds. Otherwise, the first return value is zero. The error return is +// non-nil if the compressed data does not fit in dst, but it might fit in a +// larger buffer that is still smaller than CompressBlockBound(len(src)). The +// return value (0, nil) means the data is likely incompressible and a buffer +// of length CompressBlockBound(len(src)) should be passed in. +func (c *CompressorHC) CompressBlock(src, dst []byte) (int, error) { + return c.c.CompressBlock(src, dst, lz4block.CompressionLevel(c.Level)) +} + +// CompressBlockHC is equivalent to CompressorHC.CompressBlock. +// The final two arguments are ignored and should be set to nil. +// +// This function is deprecated. Use a CompressorHC instead. +func CompressBlockHC(src, dst []byte, depth CompressionLevel, _, _ []int) (int, error) { + return lz4block.CompressBlockHC(src, dst, lz4block.CompressionLevel(depth)) +} + +const ( + // ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed + // block is corrupted or the destination buffer is not large enough for the uncompressed data. + ErrInvalidSourceShortBuffer = lz4errors.ErrInvalidSourceShortBuffer + // ErrInvalidFrame is returned when reading an invalid LZ4 archive. + ErrInvalidFrame = lz4errors.ErrInvalidFrame + // ErrInternalUnhandledState is an internal error. + ErrInternalUnhandledState = lz4errors.ErrInternalUnhandledState + // ErrInvalidHeaderChecksum is returned when reading a frame. + ErrInvalidHeaderChecksum = lz4errors.ErrInvalidHeaderChecksum + // ErrInvalidBlockChecksum is returned when reading a frame. + ErrInvalidBlockChecksum = lz4errors.ErrInvalidBlockChecksum + // ErrInvalidFrameChecksum is returned when reading a frame. + ErrInvalidFrameChecksum = lz4errors.ErrInvalidFrameChecksum + // ErrOptionInvalidCompressionLevel is returned when the supplied compression level is invalid. + ErrOptionInvalidCompressionLevel = lz4errors.ErrOptionInvalidCompressionLevel + // ErrOptionClosedOrError is returned when an option is applied to a closed or in error object. + ErrOptionClosedOrError = lz4errors.ErrOptionClosedOrError + // ErrOptionInvalidBlockSize is returned when + ErrOptionInvalidBlockSize = lz4errors.ErrOptionInvalidBlockSize + // ErrOptionNotApplicable is returned when trying to apply an option to an object not supporting it. + ErrOptionNotApplicable = lz4errors.ErrOptionNotApplicable + // ErrWriterNotClosed is returned when attempting to reset an unclosed writer. + ErrWriterNotClosed = lz4errors.ErrWriterNotClosed +) diff --git a/vendor/github.com/pierrec/lz4/v4/options.go b/vendor/github.com/pierrec/lz4/v4/options.go new file mode 100644 index 0000000000..57a44e767d --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/options.go @@ -0,0 +1,242 @@ +package lz4 + +import ( + "fmt" + "reflect" + "runtime" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +//go:generate go run golang.org/x/tools/cmd/stringer -type=BlockSize,CompressionLevel -output options_gen.go + +type ( + applier interface { + Apply(...Option) error + private() + } + // Option defines the parameters to setup an LZ4 Writer or Reader. + Option func(applier) error +) + +// String returns a string representation of the option with its parameter(s). +func (o Option) String() string { + return o(nil).Error() +} + +// Default options. +var ( + DefaultBlockSizeOption = BlockSizeOption(Block4Mb) + DefaultChecksumOption = ChecksumOption(true) + DefaultConcurrency = ConcurrencyOption(1) + defaultOnBlockDone = OnBlockDoneOption(nil) +) + +const ( + Block64Kb BlockSize = 1 << (16 + iota*2) + Block256Kb + Block1Mb + Block4Mb +) + +// BlockSizeIndex defines the size of the blocks to be compressed. +type BlockSize uint32 + +// BlockSizeOption defines the maximum size of compressed blocks (default=Block4Mb). +func BlockSizeOption(size BlockSize) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("BlockSizeOption(%s)", size) + return lz4errors.Error(s) + case *Writer: + size := uint32(size) + if !lz4block.IsValid(size) { + return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidBlockSize, size) + } + w.frame.Descriptor.Flags.BlockSizeIndexSet(lz4block.Index(size)) + return nil + case *CompressingReader: + size := uint32(size) + if !lz4block.IsValid(size) { + return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidBlockSize, size) + } + w.frame.Descriptor.Flags.BlockSizeIndexSet(lz4block.Index(size)) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// BlockChecksumOption enables or disables block checksum (default=false). +func BlockChecksumOption(flag bool) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("BlockChecksumOption(%v)", flag) + return lz4errors.Error(s) + case *Writer: + w.frame.Descriptor.Flags.BlockChecksumSet(flag) + return nil + case *CompressingReader: + w.frame.Descriptor.Flags.BlockChecksumSet(flag) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// ChecksumOption enables/disables all blocks or content checksum (default=true). +func ChecksumOption(flag bool) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("ChecksumOption(%v)", flag) + return lz4errors.Error(s) + case *Writer: + w.frame.Descriptor.Flags.ContentChecksumSet(flag) + return nil + case *CompressingReader: + w.frame.Descriptor.Flags.ContentChecksumSet(flag) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// SizeOption sets the size of the original uncompressed data (default=0). It is useful to know the size of the +// whole uncompressed data stream. +func SizeOption(size uint64) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("SizeOption(%d)", size) + return lz4errors.Error(s) + case *Writer: + w.frame.Descriptor.Flags.SizeSet(size > 0) + w.frame.Descriptor.ContentSize = size + return nil + case *CompressingReader: + w.frame.Descriptor.Flags.SizeSet(size > 0) + w.frame.Descriptor.ContentSize = size + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// ConcurrencyOption sets the number of go routines used for compression. +// If n <= 0, then the output of runtime.GOMAXPROCS(0) is used. +func ConcurrencyOption(n int) Option { + if n <= 0 { + n = runtime.GOMAXPROCS(0) + } + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("ConcurrencyOption(%d)", n) + return lz4errors.Error(s) + case *Writer: + rw.num = n + return nil + case *Reader: + rw.num = n + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// CompressionLevel defines the level of compression to use. The higher the better, but slower, compression. +type CompressionLevel uint32 + +const ( + Fast CompressionLevel = 0 + Level1 CompressionLevel = 1 << (8 + iota) + Level2 + Level3 + Level4 + Level5 + Level6 + Level7 + Level8 + Level9 +) + +// CompressionLevelOption defines the compression level (default=Fast). +func CompressionLevelOption(level CompressionLevel) Option { + return func(a applier) error { + switch w := a.(type) { + case nil: + s := fmt.Sprintf("CompressionLevelOption(%s)", level) + return lz4errors.Error(s) + case *Writer: + switch level { + case Fast, Level1, Level2, Level3, Level4, Level5, Level6, Level7, Level8, Level9: + default: + return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidCompressionLevel, level) + } + w.level = lz4block.CompressionLevel(level) + return nil + case *CompressingReader: + switch level { + case Fast, Level1, Level2, Level3, Level4, Level5, Level6, Level7, Level8, Level9: + default: + return fmt.Errorf("%w: %d", lz4errors.ErrOptionInvalidCompressionLevel, level) + } + w.level = lz4block.CompressionLevel(level) + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +func onBlockDone(int) {} + +// OnBlockDoneOption is triggered when a block has been processed. For a Writer, it is when is has been compressed, +// for a Reader, it is when it has been uncompressed. +func OnBlockDoneOption(handler func(size int)) Option { + if handler == nil { + handler = onBlockDone + } + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("OnBlockDoneOption(%s)", reflect.TypeOf(handler).String()) + return lz4errors.Error(s) + case *Writer: + rw.handler = handler + return nil + case *Reader: + rw.handler = handler + return nil + case *CompressingReader: + rw.handler = handler + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} + +// LegacyOption provides support for writing LZ4 frames in the legacy format. +// +// See https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md#legacy-frame. +// +// NB. compressed Linux kernel images use a tweaked LZ4 legacy format where +// the compressed stream is followed by the original (uncompressed) size of +// the kernel (https://events.static.linuxfound.org/sites/events/files/lcjpcojp13_klee.pdf). +// This is also supported as a special case. +func LegacyOption(legacy bool) Option { + return func(a applier) error { + switch rw := a.(type) { + case nil: + s := fmt.Sprintf("LegacyOption(%v)", legacy) + return lz4errors.Error(s) + case *Writer: + rw.legacy = legacy + return nil + } + return lz4errors.ErrOptionNotApplicable + } +} diff --git a/vendor/github.com/pierrec/lz4/v4/options_gen.go b/vendor/github.com/pierrec/lz4/v4/options_gen.go new file mode 100644 index 0000000000..2de814909e --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/options_gen.go @@ -0,0 +1,92 @@ +// Code generated by "stringer -type=BlockSize,CompressionLevel -output options_gen.go"; DO NOT EDIT. + +package lz4 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Block64Kb-65536] + _ = x[Block256Kb-262144] + _ = x[Block1Mb-1048576] + _ = x[Block4Mb-4194304] +} + +const ( + _BlockSize_name_0 = "Block64Kb" + _BlockSize_name_1 = "Block256Kb" + _BlockSize_name_2 = "Block1Mb" + _BlockSize_name_3 = "Block4Mb" +) + +func (i BlockSize) String() string { + switch { + case i == 65536: + return _BlockSize_name_0 + case i == 262144: + return _BlockSize_name_1 + case i == 1048576: + return _BlockSize_name_2 + case i == 4194304: + return _BlockSize_name_3 + default: + return "BlockSize(" + strconv.FormatInt(int64(i), 10) + ")" + } +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Fast-0] + _ = x[Level1-512] + _ = x[Level2-1024] + _ = x[Level3-2048] + _ = x[Level4-4096] + _ = x[Level5-8192] + _ = x[Level6-16384] + _ = x[Level7-32768] + _ = x[Level8-65536] + _ = x[Level9-131072] +} + +const ( + _CompressionLevel_name_0 = "Fast" + _CompressionLevel_name_1 = "Level1" + _CompressionLevel_name_2 = "Level2" + _CompressionLevel_name_3 = "Level3" + _CompressionLevel_name_4 = "Level4" + _CompressionLevel_name_5 = "Level5" + _CompressionLevel_name_6 = "Level6" + _CompressionLevel_name_7 = "Level7" + _CompressionLevel_name_8 = "Level8" + _CompressionLevel_name_9 = "Level9" +) + +func (i CompressionLevel) String() string { + switch { + case i == 0: + return _CompressionLevel_name_0 + case i == 512: + return _CompressionLevel_name_1 + case i == 1024: + return _CompressionLevel_name_2 + case i == 2048: + return _CompressionLevel_name_3 + case i == 4096: + return _CompressionLevel_name_4 + case i == 8192: + return _CompressionLevel_name_5 + case i == 16384: + return _CompressionLevel_name_6 + case i == 32768: + return _CompressionLevel_name_7 + case i == 65536: + return _CompressionLevel_name_8 + case i == 131072: + return _CompressionLevel_name_9 + default: + return "CompressionLevel(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/vendor/github.com/pierrec/lz4/v4/reader.go b/vendor/github.com/pierrec/lz4/v4/reader.go new file mode 100644 index 0000000000..275daad7cb --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/reader.go @@ -0,0 +1,275 @@ +package lz4 + +import ( + "bytes" + "io" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/lz4stream" +) + +var readerStates = []aState{ + noState: newState, + errorState: newState, + newState: readState, + readState: closedState, + closedState: newState, +} + +// NewReader returns a new LZ4 frame decoder. +func NewReader(r io.Reader) *Reader { + return newReader(r, false) +} + +func newReader(r io.Reader, legacy bool) *Reader { + zr := &Reader{frame: lz4stream.NewFrame()} + zr.state.init(readerStates) + _ = zr.Apply(DefaultConcurrency, defaultOnBlockDone) + zr.Reset(r) + return zr +} + +// Reader allows reading an LZ4 stream. +type Reader struct { + state _State + src io.Reader // source reader + num int // concurrency level + frame *lz4stream.Frame // frame being read + data []byte // block buffer allocated in non concurrent mode + reads chan []byte // pending data + idx int // size of pending data + handler func(int) + cum uint32 + dict []byte +} + +func (*Reader) private() {} + +func (r *Reader) Apply(options ...Option) (err error) { + defer r.state.check(&err) + switch r.state.state { + case newState: + case errorState: + return r.state.err + default: + return lz4errors.ErrOptionClosedOrError + } + for _, o := range options { + if err = o(r); err != nil { + return + } + } + return +} + +// Size returns the size of the underlying uncompressed data, if set in the stream. +func (r *Reader) Size() int { + switch r.state.state { + case readState, closedState: + if r.frame.Descriptor.Flags.Size() { + return int(r.frame.Descriptor.ContentSize) + } + } + return 0 +} + +func (r *Reader) isNotConcurrent() bool { + return r.num == 1 +} + +func (r *Reader) init() error { + err := r.frame.ParseHeaders(r.src) + if err != nil { + return err + } + if !r.frame.Descriptor.Flags.BlockIndependence() { + // We can't decompress dependent blocks concurrently. + // Instead of throwing an error to the user, silently drop concurrency + r.num = 1 + } + data, err := r.frame.InitR(r.src, r.num) + if err != nil { + return err + } + r.reads = data + r.idx = 0 + size := r.frame.Descriptor.Flags.BlockSizeIndex() + r.data = size.Get() + r.cum = 0 + return nil +} + +func (r *Reader) Read(buf []byte) (n int, err error) { + defer r.state.check(&err) + switch r.state.state { + case readState: + case closedState, errorState: + return 0, r.state.err + case newState: + // First initialization. + if err = r.init(); r.state.next(err) { + return + } + default: + return 0, r.state.fail() + } + for len(buf) > 0 { + var bn int + if r.idx == 0 { + if r.isNotConcurrent() { + bn, err = r.read(buf) + } else { + lz4block.Put(r.data) + r.data = <-r.reads + if len(r.data) == 0 { + // No uncompressed data: something went wrong or we are done. + err = r.frame.Blocks.ErrorR() + } + } + switch err { + case nil: + case io.EOF: + if er := r.frame.CloseR(r.src); er != nil { + err = er + } + lz4block.Put(r.data) + r.data = nil + return + default: + return + } + } + if bn == 0 { + // Fill buf with buffered data. + bn = copy(buf, r.data[r.idx:]) + r.idx += bn + if r.idx == len(r.data) { + // All data read, get ready for the next Read. + r.idx = 0 + } + } + buf = buf[bn:] + n += bn + r.handler(bn) + } + return +} + +// read uncompresses the next block as follow: +// - if buf has enough room, the block is uncompressed into it directly +// and the lenght of used space is returned +// - else, the uncompress data is stored in r.data and 0 is returned +func (r *Reader) read(buf []byte) (int, error) { + block := r.frame.Blocks.Block + _, err := block.Read(r.frame, r.src, r.cum) + if err != nil { + return 0, err + } + var direct bool + dst := r.data[:cap(r.data)] + if len(buf) >= len(dst) { + // Uncompress directly into buf. + direct = true + dst = buf + } + dst, err = block.Uncompress(r.frame, dst, r.dict, true) + if err != nil { + return 0, err + } + if !r.frame.Descriptor.Flags.BlockIndependence() { + if len(r.dict)+len(dst) > 128*1024 { + preserveSize := 64*1024 - len(dst) + if preserveSize < 0 { + preserveSize = 0 + } + r.dict = r.dict[len(r.dict)-preserveSize:] + } + r.dict = append(r.dict, dst...) + } + r.cum += uint32(len(dst)) + if direct { + return len(dst), nil + } + r.data = dst + return 0, nil +} + +// Reset clears the state of the Reader r such that it is equivalent to its +// initial state from NewReader, but instead reading from reader. +// No access to reader is performed. +func (r *Reader) Reset(reader io.Reader) { + if r.data != nil { + lz4block.Put(r.data) + r.data = nil + } + r.frame.Reset(r.num) + r.state.reset() + r.src = reader + r.reads = nil +} + +// WriteTo efficiently uncompresses the data from the Reader underlying source to w. +func (r *Reader) WriteTo(w io.Writer) (n int64, err error) { + switch r.state.state { + case closedState, errorState: + return 0, r.state.err + case newState: + if err = r.init(); r.state.next(err) { + return + } + default: + return 0, r.state.fail() + } + defer r.state.nextd(&err) + + var data []byte + if r.isNotConcurrent() { + size := r.frame.Descriptor.Flags.BlockSizeIndex() + data = size.Get() + defer lz4block.Put(data) + } + for { + var bn int + var dst []byte + if r.isNotConcurrent() { + bn, err = r.read(data) + dst = data[:bn] + } else { + lz4block.Put(dst) + dst = <-r.reads + bn = len(dst) + if bn == 0 { + // No uncompressed data: something went wrong or we are done. + err = r.frame.Blocks.ErrorR() + } + } + switch err { + case nil: + case io.EOF: + err = r.frame.CloseR(r.src) + return + default: + return + } + r.handler(bn) + bn, err = w.Write(dst) + n += int64(bn) + if err != nil { + return + } + } +} + +// ValidFrameHeader returns a bool indicating if the given bytes slice matches a LZ4 header. +func ValidFrameHeader(in []byte) (bool, error) { + f := lz4stream.NewFrame() + err := f.ParseHeaders(bytes.NewReader(in)) + if err == nil { + return true, nil + } + if err == lz4errors.ErrInvalidFrame { + return false, nil + } + return false, err +} diff --git a/vendor/github.com/pierrec/lz4/v4/state.go b/vendor/github.com/pierrec/lz4/v4/state.go new file mode 100644 index 0000000000..d94f04d05e --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/state.go @@ -0,0 +1,75 @@ +package lz4 + +import ( + "errors" + "fmt" + "io" + + "github.com/pierrec/lz4/v4/internal/lz4errors" +) + +//go:generate go run golang.org/x/tools/cmd/stringer -type=aState -output state_gen.go + +const ( + noState aState = iota // uninitialized reader + errorState // unrecoverable error encountered + newState // instantiated object + readState // reading data + writeState // writing data + closedState // all done +) + +type ( + aState uint8 + _State struct { + states []aState + state aState + err error + } +) + +func (s *_State) init(states []aState) { + s.states = states + s.state = states[0] +} + +func (s *_State) reset() { + s.state = s.states[0] + s.err = nil +} + +// next sets the state to the next one unless it is passed a non nil error. +// It returns whether or not it is in error. +func (s *_State) next(err error) bool { + if err != nil { + s.err = fmt.Errorf("%s: %w", s.state, err) + s.state = errorState + return true + } + s.state = s.states[s.state] + return false +} + +// nextd is like next but for defers. +func (s *_State) nextd(errp *error) bool { + return errp != nil && s.next(*errp) +} + +// check sets s in error if not already in error and if the error is not nil or io.EOF, +func (s *_State) check(errp *error) { + if s.state == errorState || errp == nil { + return + } + if err := *errp; err != nil { + s.err = fmt.Errorf("%w[%s]", err, s.state) + if !errors.Is(err, io.EOF) { + s.state = errorState + } + } +} + +func (s *_State) fail() error { + s.state = errorState + s.err = fmt.Errorf("%w[%s]", lz4errors.ErrInternalUnhandledState, s.state) + return s.err +} diff --git a/vendor/github.com/pierrec/lz4/v4/state_gen.go b/vendor/github.com/pierrec/lz4/v4/state_gen.go new file mode 100644 index 0000000000..75fb828924 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/state_gen.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=aState -output state_gen.go"; DO NOT EDIT. + +package lz4 + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[noState-0] + _ = x[errorState-1] + _ = x[newState-2] + _ = x[readState-3] + _ = x[writeState-4] + _ = x[closedState-5] +} + +const _aState_name = "noStateerrorStatenewStatereadStatewriteStateclosedState" + +var _aState_index = [...]uint8{0, 7, 17, 25, 34, 44, 55} + +func (i aState) String() string { + if i >= aState(len(_aState_index)-1) { + return "aState(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _aState_name[_aState_index[i]:_aState_index[i+1]] +} diff --git a/vendor/github.com/pierrec/lz4/v4/writer.go b/vendor/github.com/pierrec/lz4/v4/writer.go new file mode 100644 index 0000000000..4358adee10 --- /dev/null +++ b/vendor/github.com/pierrec/lz4/v4/writer.go @@ -0,0 +1,242 @@ +package lz4 + +import ( + "io" + + "github.com/pierrec/lz4/v4/internal/lz4block" + "github.com/pierrec/lz4/v4/internal/lz4errors" + "github.com/pierrec/lz4/v4/internal/lz4stream" +) + +var writerStates = []aState{ + noState: newState, + newState: writeState, + writeState: closedState, + closedState: newState, + errorState: newState, +} + +// NewWriter returns a new LZ4 frame encoder. +func NewWriter(w io.Writer) *Writer { + zw := &Writer{frame: lz4stream.NewFrame()} + zw.state.init(writerStates) + _ = zw.Apply(DefaultBlockSizeOption, DefaultChecksumOption, DefaultConcurrency, defaultOnBlockDone) + zw.Reset(w) + return zw +} + +// Writer allows writing an LZ4 stream. +type Writer struct { + state _State + src io.Writer // destination writer + level lz4block.CompressionLevel // how hard to try + num int // concurrency level + frame *lz4stream.Frame // frame being built + data []byte // pending data + idx int // size of pending data + handler func(int) + legacy bool +} + +func (*Writer) private() {} + +func (w *Writer) Apply(options ...Option) (err error) { + defer w.state.check(&err) + switch w.state.state { + case newState: + case errorState: + return w.state.err + default: + return lz4errors.ErrOptionClosedOrError + } + w.Reset(w.src) + for _, o := range options { + if err = o(w); err != nil { + return + } + } + return +} + +func (w *Writer) isNotConcurrent() bool { + return w.num == 1 +} + +// init sets up the Writer when in newState. It does not change the Writer state. +func (w *Writer) init() error { + w.frame.InitW(w.src, w.num, w.legacy) + size := w.frame.Descriptor.Flags.BlockSizeIndex() + w.data = size.Get() + w.idx = 0 + return w.frame.Descriptor.Write(w.frame, w.src) +} + +func (w *Writer) Write(buf []byte) (n int, err error) { + defer w.state.check(&err) + switch w.state.state { + case writeState: + case closedState, errorState: + return 0, w.state.err + case newState: + if err = w.init(); w.state.next(err) { + return + } + default: + return 0, w.state.fail() + } + + zn := len(w.data) + for len(buf) > 0 { + if w.isNotConcurrent() && w.idx == 0 && len(buf) >= zn { + // Avoid a copy as there is enough data for a block. + if err = w.write(buf[:zn], false); err != nil { + return + } + n += zn + buf = buf[zn:] + continue + } + // Accumulate the data to be compressed. + m := copy(w.data[w.idx:], buf) + n += m + w.idx += m + buf = buf[m:] + + if w.idx < len(w.data) { + // Buffer not filled. + return + } + + // Buffer full. + if err = w.write(w.data, true); err != nil { + return + } + if !w.isNotConcurrent() { + size := w.frame.Descriptor.Flags.BlockSizeIndex() + w.data = size.Get() + } + w.idx = 0 + } + return +} + +func (w *Writer) write(data []byte, safe bool) error { + if w.isNotConcurrent() { + block := w.frame.Blocks.Block + err := block.Compress(w.frame, data, w.level).Write(w.frame, w.src) + w.handler(len(block.Data)) + return err + } + c := make(chan *lz4stream.FrameDataBlock) + w.frame.Blocks.Blocks <- c + go func(c chan *lz4stream.FrameDataBlock, data []byte, safe bool) { + b := lz4stream.NewFrameDataBlock(w.frame) + c <- b.Compress(w.frame, data, w.level) + <-c + w.handler(len(b.Data)) + b.Close(w.frame) + if safe { + // safe to put it back as the last usage of it was FrameDataBlock.Write() called before c is closed + lz4block.Put(data) + } + }(c, data, safe) + + return nil +} + +// Flush any buffered data to the underlying writer immediately. +func (w *Writer) Flush() (err error) { + switch w.state.state { + case writeState: + case errorState: + return w.state.err + case newState: + if err = w.init(); w.state.next(err) { + return + } + default: + return nil + } + + if w.idx > 0 { + // Flush pending data, disable w.data freeing as it is done later on. + if err = w.write(w.data[:w.idx], false); err != nil { + return err + } + w.idx = 0 + } + return nil +} + +// Close closes the Writer, flushing any unwritten data to the underlying writer +// without closing it. +func (w *Writer) Close() error { + if err := w.Flush(); err != nil { + return err + } + err := w.frame.CloseW(w.src, w.num) + // It is now safe to free the buffer. + if w.data != nil { + lz4block.Put(w.data) + w.data = nil + } + return err +} + +// Reset clears the state of the Writer w such that it is equivalent to its +// initial state from NewWriter, but instead writing to writer. +// Reset keeps the previous options unless overwritten by the supplied ones. +// No access to writer is performed. +// +// w.Close must be called before Reset or pending data may be dropped. +func (w *Writer) Reset(writer io.Writer) { + w.frame.Reset(w.num) + w.state.reset() + w.src = writer +} + +// ReadFrom efficiently reads from r and compressed into the Writer destination. +func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { + switch w.state.state { + case closedState, errorState: + return 0, w.state.err + case newState: + if err = w.init(); w.state.next(err) { + return + } + default: + return 0, w.state.fail() + } + defer w.state.check(&err) + + size := w.frame.Descriptor.Flags.BlockSizeIndex() + var done bool + var rn int + data := size.Get() + if w.isNotConcurrent() { + // Keep the same buffer for the whole process. + defer lz4block.Put(data) + } + for !done { + rn, err = io.ReadFull(r, data) + switch err { + case nil: + case io.EOF, io.ErrUnexpectedEOF: // read may be partial + done = true + default: + return + } + n += int64(rn) + err = w.write(data[:rn], true) + if err != nil { + return + } + w.handler(rn) + if !done && !w.isNotConcurrent() { + // The buffer will be returned automatically by go routines (safe=true) + // so get a new one fo the next round. + data = size.Get() + } + } + return +} diff --git a/vendor/github.com/rivo/uniseg/LICENSE.txt b/vendor/github.com/rivo/uniseg/LICENSE.txt new file mode 100644 index 0000000000..5040f1ef80 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Oliver Kuederle + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/rivo/uniseg/README.md b/vendor/github.com/rivo/uniseg/README.md new file mode 100644 index 0000000000..a8191b8154 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/README.md @@ -0,0 +1,137 @@ +# Unicode Text Segmentation for Go + +[![Go Reference](https://pkg.go.dev/badge/github.com/rivo/uniseg.svg)](https://pkg.go.dev/github.com/rivo/uniseg) +[![Go Report](https://img.shields.io/badge/go%20report-A%2B-brightgreen.svg)](https://goreportcard.com/report/github.com/rivo/uniseg) + +This Go package implements Unicode Text Segmentation according to [Unicode Standard Annex #29](https://unicode.org/reports/tr29/), Unicode Line Breaking according to [Unicode Standard Annex #14](https://unicode.org/reports/tr14/) (Unicode version 15.0.0), and monospace font string width calculation similar to [wcwidth](https://man7.org/linux/man-pages/man3/wcwidth.3.html). + +## Background + +### Grapheme Clusters + +In Go, [strings are read-only slices of bytes](https://go.dev/blog/strings). They can be turned into Unicode code points using the `for` loop or by casting: `[]rune(str)`. However, multiple code points may be combined into one user-perceived character or what the Unicode specification calls "grapheme cluster". Here are some examples: + +|String|Bytes (UTF-8)|Code points (runes)|Grapheme clusters| +|-|-|-|-| +|Kaฬˆse|6 bytes: `4b 61 cc 88 73 65`|5 code points: `4b 61 308 73 65`|4 clusters: `[4b],[61 308],[73],[65]`| +|๐Ÿณ๏ธโ€๐ŸŒˆ|14 bytes: `f0 9f 8f b3 ef b8 8f e2 80 8d f0 9f 8c 88`|4 code points: `1f3f3 fe0f 200d 1f308`|1 cluster: `[1f3f3 fe0f 200d 1f308]`| +|๐Ÿ‡ฉ๐Ÿ‡ช|8 bytes: `f0 9f 87 a9 f0 9f 87 aa`|2 code points: `1f1e9 1f1ea`|1 cluster: `[1f1e9 1f1ea]`| + +This package provides tools to iterate over these grapheme clusters. This may be used to determine the number of user-perceived characters, to split strings in their intended places, or to extract individual characters which form a unit. + +### Word Boundaries + +Word boundaries are used in a number of different contexts. The most familiar ones are selection (double-click mouse selection), cursor movement ("move to next word" control-arrow keys), and the dialog option "Whole Word Search" for search and replace. They are also used in database queries, to determine whether elements are within a certain number of words of one another. Searching may also use word boundaries in determining matching items. This package provides tools to determine word boundaries within strings. + +### Sentence Boundaries + +Sentence boundaries are often used for triple-click or some other method of selecting or iterating through blocks of text that are larger than single words. They are also used to determine whether words occur within the same sentence in database queries. This package provides tools to determine sentence boundaries within strings. + +### Line Breaking + +Line breaking, also known as word wrapping, is the process of breaking a section of text into lines such that it will fit in the available width of a page, window or other display area. This package provides tools to determine where a string may or may not be broken and where it must be broken (for example after newline characters). + +### Monospace Width + +Most terminals or text displays / text editors using a monospace font (for example source code editors) use a fixed width for each character. Some characters such as emojis or characters found in Asian and other languages may take up more than one character cell. This package provides tools to determine the number of cells a string will take up when displayed in a monospace font. See [here](https://pkg.go.dev/github.com/rivo/uniseg#hdr-Monospace_Width) for more information. + +## Installation + +```bash +go get github.com/rivo/uniseg +``` + +## Examples + +### Counting Characters in a String + +```go +n := uniseg.GraphemeClusterCount("๐Ÿ‡ฉ๐Ÿ‡ช๐Ÿณ๏ธโ€๐ŸŒˆ") +fmt.Println(n) +// 2 +``` + +### Calculating the Monospace String Width + +```go +width := uniseg.StringWidth("๐Ÿ‡ฉ๐Ÿ‡ช๐Ÿณ๏ธโ€๐ŸŒˆ!") +fmt.Println(width) +// 5 +``` + +### Using the [`Graphemes`](https://pkg.go.dev/github.com/rivo/uniseg#Graphemes) Class + +This is the most convenient method of iterating over grapheme clusters: + +```go +gr := uniseg.NewGraphemes("๐Ÿ‘๐Ÿผ!") +for gr.Next() { + fmt.Printf("%x ", gr.Runes()) +} +// [1f44d 1f3fc] [21] +``` + +### Using the [`Step`](https://pkg.go.dev/github.com/rivo/uniseg#Step) or [`StepString`](https://pkg.go.dev/github.com/rivo/uniseg#StepString) Function + +This avoids allocating a new `Graphemes` object but it requires the handling of states and boundaries: + +```go +str := "๐Ÿ‡ฉ๐Ÿ‡ช๐Ÿณ๏ธโ€๐ŸŒˆ" +state := -1 +var c string +for len(str) > 0 { + c, str, _, state = uniseg.StepString(str, state) + fmt.Printf("%x ", []rune(c)) +} +// [1f1e9 1f1ea] [1f3f3 fe0f 200d 1f308] +``` + +### Advanced Examples + +The [`Graphemes`](https://pkg.go.dev/github.com/rivo/uniseg#Graphemes) class offers the most convenient way to access all functionality of this package. But in some cases, it may be better to use the specialized functions directly. For example, if you're only interested in word segmentation, use [`FirstWord`](https://pkg.go.dev/github.com/rivo/uniseg#FirstWord) or [`FirstWordInString`](https://pkg.go.dev/github.com/rivo/uniseg#FirstWordInString): + +```go +str := "Hello, world!" +state := -1 +var c string +for len(str) > 0 { + c, str, state = uniseg.FirstWordInString(str, state) + fmt.Printf("(%s)\n", c) +} +// (Hello) +// (,) +// ( ) +// (world) +// (!) +``` + +Similarly, use + +- [`FirstGraphemeCluster`](https://pkg.go.dev/github.com/rivo/uniseg#FirstGraphemeCluster) or [`FirstGraphemeClusterInString`](https://pkg.go.dev/github.com/rivo/uniseg#FirstGraphemeClusterInString) for grapheme cluster determination only, +- [`FirstSentence`](https://pkg.go.dev/github.com/rivo/uniseg#FirstSentence) or [`FirstSentenceInString`](https://pkg.go.dev/github.com/rivo/uniseg#FirstSentenceInString) for sentence segmentation only, and +- [`FirstLineSegment`](https://pkg.go.dev/github.com/rivo/uniseg#FirstLineSegment) or [`FirstLineSegmentInString`](https://pkg.go.dev/github.com/rivo/uniseg#FirstLineSegmentInString) for line breaking / word wrapping (although using [`Step`](https://pkg.go.dev/github.com/rivo/uniseg#Step) or [`StepString`](https://pkg.go.dev/github.com/rivo/uniseg#StepString) is preferred as it will observe grapheme cluster boundaries). + +If you're only interested in the width of characters, use [`FirstGraphemeCluster`](https://pkg.go.dev/github.com/rivo/uniseg#FirstGraphemeCluster) or [`FirstGraphemeClusterInString`](https://pkg.go.dev/github.com/rivo/uniseg#FirstGraphemeClusterInString). It is much faster than using [`Step`](https://pkg.go.dev/github.com/rivo/uniseg#Step), [`StepString`](https://pkg.go.dev/github.com/rivo/uniseg#StepString), or the [`Graphemes`](https://pkg.go.dev/github.com/rivo/uniseg#Graphemes) class because it does not include the logic for word / sentence / line boundaries. + +Finally, if you need to reverse a string while preserving grapheme clusters, use [`ReverseString`](https://pkg.go.dev/github.com/rivo/uniseg#ReverseString): + +```go +fmt.Println(uniseg.ReverseString("๐Ÿ‡ฉ๐Ÿ‡ช๐Ÿณ๏ธโ€๐ŸŒˆ")) +// ๐Ÿณ๏ธโ€๐ŸŒˆ๐Ÿ‡ฉ๐Ÿ‡ช +``` + +## Documentation + +Refer to https://pkg.go.dev/github.com/rivo/uniseg for the package's documentation. + +## Dependencies + +This package does not depend on any packages outside the standard library. + +## Sponsor this Project + +[Become a Sponsor on GitHub](https://github.com/sponsors/rivo?metadata_source=uniseg_readme) to support this project! + +## Your Feedback + +Add your issue here on GitHub, preferably before submitting any PR's. Feel free to get in touch if you have any questions. \ No newline at end of file diff --git a/vendor/github.com/rivo/uniseg/doc.go b/vendor/github.com/rivo/uniseg/doc.go new file mode 100644 index 0000000000..11224ae22d --- /dev/null +++ b/vendor/github.com/rivo/uniseg/doc.go @@ -0,0 +1,108 @@ +/* +Package uniseg implements Unicode Text Segmentation, Unicode Line Breaking, and +string width calculation for monospace fonts. Unicode Text Segmentation conforms +to Unicode Standard Annex #29 (https://unicode.org/reports/tr29/) and Unicode +Line Breaking conforms to Unicode Standard Annex #14 +(https://unicode.org/reports/tr14/). + +In short, using this package, you can split a string into grapheme clusters +(what people would usually refer to as a "character"), into words, and into +sentences. Or, in its simplest case, this package allows you to count the number +of characters in a string, especially when it contains complex characters such +as emojis, combining characters, or characters from Asian, Arabic, Hebrew, or +other languages. Additionally, you can use it to implement line breaking (or +"word wrapping"), that is, to determine where text can be broken over to the +next line when the width of the line is not big enough to fit the entire text. +Finally, you can use it to calculate the display width of a string for monospace +fonts. + +# Getting Started + +If you just want to count the number of characters in a string, you can use +[GraphemeClusterCount]. If you want to determine the display width of a string, +you can use [StringWidth]. If you want to iterate over a string, you can use +[Step], [StepString], or the [Graphemes] class (more convenient but less +performant). This will provide you with all information: grapheme clusters, +word boundaries, sentence boundaries, line breaks, and monospace character +widths. The specialized functions [FirstGraphemeCluster], +[FirstGraphemeClusterInString], [FirstWord], [FirstWordInString], +[FirstSentence], and [FirstSentenceInString] can be used if only one type of +information is needed. + +# Grapheme Clusters + +Consider the rainbow flag emoji: ๐Ÿณ๏ธโ€๐ŸŒˆ. On most modern systems, it appears as one +character. But its string representation actually has 14 bytes, so counting +bytes (or using len("๐Ÿณ๏ธโ€๐ŸŒˆ")) will not work as expected. Counting runes won't, +either: The flag has 4 Unicode code points, thus 4 runes. The stdlib function +utf8.RuneCountInString("๐Ÿณ๏ธโ€๐ŸŒˆ") and len([]rune("๐Ÿณ๏ธโ€๐ŸŒˆ")) will both return 4. + +The [GraphemeClusterCount] function will return 1 for the rainbow flag emoji. +The Graphemes class and a variety of functions in this package will allow you to +split strings into its grapheme clusters. + +# Word Boundaries + +Word boundaries are used in a number of different contexts. The most familiar +ones are selection (double-click mouse selection), cursor movement ("move to +next word" control-arrow keys), and the dialog option "Whole Word Search" for +search and replace. This package provides methods for determining word +boundaries. + +# Sentence Boundaries + +Sentence boundaries are often used for triple-click or some other method of +selecting or iterating through blocks of text that are larger than single words. +They are also used to determine whether words occur within the same sentence in +database queries. This package provides methods for determining sentence +boundaries. + +# Line Breaking + +Line breaking, also known as word wrapping, is the process of breaking a section +of text into lines such that it will fit in the available width of a page, +window or other display area. This package provides methods to determine the +positions in a string where a line must be broken, may be broken, or must not be +broken. + +# Monospace Width + +Monospace width, as referred to in this package, is the width of a string in a +monospace font. This is commonly used in terminal user interfaces or text +displays or editors that don't support proportional fonts. A width of 1 +corresponds to a single character cell. The C function [wcswidth()] and its +implementation in other programming languages is in widespread use for the same +purpose. However, there is no standard for the calculation of such widths, and +this package differs from wcswidth() in a number of ways, presumably to generate +more visually pleasing results. + +To start, we assume that every code point has a width of 1, with the following +exceptions: + + - Code points with grapheme cluster break properties Control, CR, LF, Extend, + and ZWJ have a width of 0. + - U+2E3A, Two-Em Dash, has a width of 3. + - U+2E3B, Three-Em Dash, has a width of 4. + - Characters with the East-Asian Width properties "Fullwidth" (F) and "Wide" + (W) have a width of 2. (Properties "Ambiguous" (A) and "Neutral" (N) both + have a width of 1.) + - Code points with grapheme cluster break property Regional Indicator have a + width of 2. + - Code points with grapheme cluster break property Extended Pictographic have + a width of 2, unless their Emoji Presentation flag is "No", in which case + the width is 1. + +For Hangul grapheme clusters composed of conjoining Jamo and for Regional +Indicators (flags), all code points except the first one have a width of 0. For +grapheme clusters starting with an Extended Pictographic, any additional code +point will force a total width of 2, except if the Variation Selector-15 +(U+FE0E) is included, in which case the total width is always 1. Grapheme +clusters ending with Variation Selector-16 (U+FE0F) have a width of 2. + +Note that whether these widths appear correct depends on your application's +render engine, to which extent it conforms to the Unicode Standard, and its +choice of font. + +[wcswidth()]: https://man7.org/linux/man-pages/man3/wcswidth.3.html +*/ +package uniseg diff --git a/vendor/github.com/rivo/uniseg/eastasianwidth.go b/vendor/github.com/rivo/uniseg/eastasianwidth.go new file mode 100644 index 0000000000..5fc54d9915 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/eastasianwidth.go @@ -0,0 +1,2588 @@ +// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// eastAsianWidth are taken from +// https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// and +// https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt +// ("Extended_Pictographic" only) +// on September 5, 2023. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var eastAsianWidth = [][3]int{ + {0x0000, 0x001F, prN}, // Cc [32] .. + {0x0020, 0x0020, prNa}, // Zs SPACE + {0x0021, 0x0023, prNa}, // Po [3] EXCLAMATION MARK..NUMBER SIGN + {0x0024, 0x0024, prNa}, // Sc DOLLAR SIGN + {0x0025, 0x0027, prNa}, // Po [3] PERCENT SIGN..APOSTROPHE + {0x0028, 0x0028, prNa}, // Ps LEFT PARENTHESIS + {0x0029, 0x0029, prNa}, // Pe RIGHT PARENTHESIS + {0x002A, 0x002A, prNa}, // Po ASTERISK + {0x002B, 0x002B, prNa}, // Sm PLUS SIGN + {0x002C, 0x002C, prNa}, // Po COMMA + {0x002D, 0x002D, prNa}, // Pd HYPHEN-MINUS + {0x002E, 0x002F, prNa}, // Po [2] FULL STOP..SOLIDUS + {0x0030, 0x0039, prNa}, // Nd [10] DIGIT ZERO..DIGIT NINE + {0x003A, 0x003B, prNa}, // Po [2] COLON..SEMICOLON + {0x003C, 0x003E, prNa}, // Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN + {0x003F, 0x0040, prNa}, // Po [2] QUESTION MARK..COMMERCIAL AT + {0x0041, 0x005A, prNa}, // Lu [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z + {0x005B, 0x005B, prNa}, // Ps LEFT SQUARE BRACKET + {0x005C, 0x005C, prNa}, // Po REVERSE SOLIDUS + {0x005D, 0x005D, prNa}, // Pe RIGHT SQUARE BRACKET + {0x005E, 0x005E, prNa}, // Sk CIRCUMFLEX ACCENT + {0x005F, 0x005F, prNa}, // Pc LOW LINE + {0x0060, 0x0060, prNa}, // Sk GRAVE ACCENT + {0x0061, 0x007A, prNa}, // Ll [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z + {0x007B, 0x007B, prNa}, // Ps LEFT CURLY BRACKET + {0x007C, 0x007C, prNa}, // Sm VERTICAL LINE + {0x007D, 0x007D, prNa}, // Pe RIGHT CURLY BRACKET + {0x007E, 0x007E, prNa}, // Sm TILDE + {0x007F, 0x007F, prN}, // Cc + {0x0080, 0x009F, prN}, // Cc [32] .. + {0x00A0, 0x00A0, prN}, // Zs NO-BREAK SPACE + {0x00A1, 0x00A1, prA}, // Po INVERTED EXCLAMATION MARK + {0x00A2, 0x00A3, prNa}, // Sc [2] CENT SIGN..POUND SIGN + {0x00A4, 0x00A4, prA}, // Sc CURRENCY SIGN + {0x00A5, 0x00A5, prNa}, // Sc YEN SIGN + {0x00A6, 0x00A6, prNa}, // So BROKEN BAR + {0x00A7, 0x00A7, prA}, // Po SECTION SIGN + {0x00A8, 0x00A8, prA}, // Sk DIAERESIS + {0x00A9, 0x00A9, prN}, // So COPYRIGHT SIGN + {0x00AA, 0x00AA, prA}, // Lo FEMININE ORDINAL INDICATOR + {0x00AB, 0x00AB, prN}, // Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00AC, 0x00AC, prNa}, // Sm NOT SIGN + {0x00AD, 0x00AD, prA}, // Cf SOFT HYPHEN + {0x00AE, 0x00AE, prA}, // So REGISTERED SIGN + {0x00AF, 0x00AF, prNa}, // Sk MACRON + {0x00B0, 0x00B0, prA}, // So DEGREE SIGN + {0x00B1, 0x00B1, prA}, // Sm PLUS-MINUS SIGN + {0x00B2, 0x00B3, prA}, // No [2] SUPERSCRIPT TWO..SUPERSCRIPT THREE + {0x00B4, 0x00B4, prA}, // Sk ACUTE ACCENT + {0x00B5, 0x00B5, prN}, // Ll MICRO SIGN + {0x00B6, 0x00B7, prA}, // Po [2] PILCROW SIGN..MIDDLE DOT + {0x00B8, 0x00B8, prA}, // Sk CEDILLA + {0x00B9, 0x00B9, prA}, // No SUPERSCRIPT ONE + {0x00BA, 0x00BA, prA}, // Lo MASCULINE ORDINAL INDICATOR + {0x00BB, 0x00BB, prN}, // Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00BC, 0x00BE, prA}, // No [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS + {0x00BF, 0x00BF, prA}, // Po INVERTED QUESTION MARK + {0x00C0, 0x00C5, prN}, // Lu [6] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER A WITH RING ABOVE + {0x00C6, 0x00C6, prA}, // Lu LATIN CAPITAL LETTER AE + {0x00C7, 0x00CF, prN}, // Lu [9] LATIN CAPITAL LETTER C WITH CEDILLA..LATIN CAPITAL LETTER I WITH DIAERESIS + {0x00D0, 0x00D0, prA}, // Lu LATIN CAPITAL LETTER ETH + {0x00D1, 0x00D6, prN}, // Lu [6] LATIN CAPITAL LETTER N WITH TILDE..LATIN CAPITAL LETTER O WITH DIAERESIS + {0x00D7, 0x00D7, prA}, // Sm MULTIPLICATION SIGN + {0x00D8, 0x00D8, prA}, // Lu LATIN CAPITAL LETTER O WITH STROKE + {0x00D9, 0x00DD, prN}, // Lu [5] LATIN CAPITAL LETTER U WITH GRAVE..LATIN CAPITAL LETTER Y WITH ACUTE + {0x00DE, 0x00E1, prA}, // L& [4] LATIN CAPITAL LETTER THORN..LATIN SMALL LETTER A WITH ACUTE + {0x00E2, 0x00E5, prN}, // Ll [4] LATIN SMALL LETTER A WITH CIRCUMFLEX..LATIN SMALL LETTER A WITH RING ABOVE + {0x00E6, 0x00E6, prA}, // Ll LATIN SMALL LETTER AE + {0x00E7, 0x00E7, prN}, // Ll LATIN SMALL LETTER C WITH CEDILLA + {0x00E8, 0x00EA, prA}, // Ll [3] LATIN SMALL LETTER E WITH GRAVE..LATIN SMALL LETTER E WITH CIRCUMFLEX + {0x00EB, 0x00EB, prN}, // Ll LATIN SMALL LETTER E WITH DIAERESIS + {0x00EC, 0x00ED, prA}, // Ll [2] LATIN SMALL LETTER I WITH GRAVE..LATIN SMALL LETTER I WITH ACUTE + {0x00EE, 0x00EF, prN}, // Ll [2] LATIN SMALL LETTER I WITH CIRCUMFLEX..LATIN SMALL LETTER I WITH DIAERESIS + {0x00F0, 0x00F0, prA}, // Ll LATIN SMALL LETTER ETH + {0x00F1, 0x00F1, prN}, // Ll LATIN SMALL LETTER N WITH TILDE + {0x00F2, 0x00F3, prA}, // Ll [2] LATIN SMALL LETTER O WITH GRAVE..LATIN SMALL LETTER O WITH ACUTE + {0x00F4, 0x00F6, prN}, // Ll [3] LATIN SMALL LETTER O WITH CIRCUMFLEX..LATIN SMALL LETTER O WITH DIAERESIS + {0x00F7, 0x00F7, prA}, // Sm DIVISION SIGN + {0x00F8, 0x00FA, prA}, // Ll [3] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER U WITH ACUTE + {0x00FB, 0x00FB, prN}, // Ll LATIN SMALL LETTER U WITH CIRCUMFLEX + {0x00FC, 0x00FC, prA}, // Ll LATIN SMALL LETTER U WITH DIAERESIS + {0x00FD, 0x00FD, prN}, // Ll LATIN SMALL LETTER Y WITH ACUTE + {0x00FE, 0x00FE, prA}, // Ll LATIN SMALL LETTER THORN + {0x00FF, 0x00FF, prN}, // Ll LATIN SMALL LETTER Y WITH DIAERESIS + {0x0100, 0x0100, prN}, // Lu LATIN CAPITAL LETTER A WITH MACRON + {0x0101, 0x0101, prA}, // Ll LATIN SMALL LETTER A WITH MACRON + {0x0102, 0x0110, prN}, // L& [15] LATIN CAPITAL LETTER A WITH BREVE..LATIN CAPITAL LETTER D WITH STROKE + {0x0111, 0x0111, prA}, // Ll LATIN SMALL LETTER D WITH STROKE + {0x0112, 0x0112, prN}, // Lu LATIN CAPITAL LETTER E WITH MACRON + {0x0113, 0x0113, prA}, // Ll LATIN SMALL LETTER E WITH MACRON + {0x0114, 0x011A, prN}, // L& [7] LATIN CAPITAL LETTER E WITH BREVE..LATIN CAPITAL LETTER E WITH CARON + {0x011B, 0x011B, prA}, // Ll LATIN SMALL LETTER E WITH CARON + {0x011C, 0x0125, prN}, // L& [10] LATIN CAPITAL LETTER G WITH CIRCUMFLEX..LATIN SMALL LETTER H WITH CIRCUMFLEX + {0x0126, 0x0127, prA}, // L& [2] LATIN CAPITAL LETTER H WITH STROKE..LATIN SMALL LETTER H WITH STROKE + {0x0128, 0x012A, prN}, // L& [3] LATIN CAPITAL LETTER I WITH TILDE..LATIN CAPITAL LETTER I WITH MACRON + {0x012B, 0x012B, prA}, // Ll LATIN SMALL LETTER I WITH MACRON + {0x012C, 0x0130, prN}, // L& [5] LATIN CAPITAL LETTER I WITH BREVE..LATIN CAPITAL LETTER I WITH DOT ABOVE + {0x0131, 0x0133, prA}, // L& [3] LATIN SMALL LETTER DOTLESS I..LATIN SMALL LIGATURE IJ + {0x0134, 0x0137, prN}, // L& [4] LATIN CAPITAL LETTER J WITH CIRCUMFLEX..LATIN SMALL LETTER K WITH CEDILLA + {0x0138, 0x0138, prA}, // Ll LATIN SMALL LETTER KRA + {0x0139, 0x013E, prN}, // L& [6] LATIN CAPITAL LETTER L WITH ACUTE..LATIN SMALL LETTER L WITH CARON + {0x013F, 0x0142, prA}, // L& [4] LATIN CAPITAL LETTER L WITH MIDDLE DOT..LATIN SMALL LETTER L WITH STROKE + {0x0143, 0x0143, prN}, // Lu LATIN CAPITAL LETTER N WITH ACUTE + {0x0144, 0x0144, prA}, // Ll LATIN SMALL LETTER N WITH ACUTE + {0x0145, 0x0147, prN}, // L& [3] LATIN CAPITAL LETTER N WITH CEDILLA..LATIN CAPITAL LETTER N WITH CARON + {0x0148, 0x014B, prA}, // L& [4] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER ENG + {0x014C, 0x014C, prN}, // Lu LATIN CAPITAL LETTER O WITH MACRON + {0x014D, 0x014D, prA}, // Ll LATIN SMALL LETTER O WITH MACRON + {0x014E, 0x0151, prN}, // L& [4] LATIN CAPITAL LETTER O WITH BREVE..LATIN SMALL LETTER O WITH DOUBLE ACUTE + {0x0152, 0x0153, prA}, // L& [2] LATIN CAPITAL LIGATURE OE..LATIN SMALL LIGATURE OE + {0x0154, 0x0165, prN}, // L& [18] LATIN CAPITAL LETTER R WITH ACUTE..LATIN SMALL LETTER T WITH CARON + {0x0166, 0x0167, prA}, // L& [2] LATIN CAPITAL LETTER T WITH STROKE..LATIN SMALL LETTER T WITH STROKE + {0x0168, 0x016A, prN}, // L& [3] LATIN CAPITAL LETTER U WITH TILDE..LATIN CAPITAL LETTER U WITH MACRON + {0x016B, 0x016B, prA}, // Ll LATIN SMALL LETTER U WITH MACRON + {0x016C, 0x017F, prN}, // L& [20] LATIN CAPITAL LETTER U WITH BREVE..LATIN SMALL LETTER LONG S + {0x0180, 0x01BA, prN}, // L& [59] LATIN SMALL LETTER B WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL + {0x01BB, 0x01BB, prN}, // Lo LATIN LETTER TWO WITH STROKE + {0x01BC, 0x01BF, prN}, // L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN + {0x01C0, 0x01C3, prN}, // Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK + {0x01C4, 0x01CD, prN}, // L& [10] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER A WITH CARON + {0x01CE, 0x01CE, prA}, // Ll LATIN SMALL LETTER A WITH CARON + {0x01CF, 0x01CF, prN}, // Lu LATIN CAPITAL LETTER I WITH CARON + {0x01D0, 0x01D0, prA}, // Ll LATIN SMALL LETTER I WITH CARON + {0x01D1, 0x01D1, prN}, // Lu LATIN CAPITAL LETTER O WITH CARON + {0x01D2, 0x01D2, prA}, // Ll LATIN SMALL LETTER O WITH CARON + {0x01D3, 0x01D3, prN}, // Lu LATIN CAPITAL LETTER U WITH CARON + {0x01D4, 0x01D4, prA}, // Ll LATIN SMALL LETTER U WITH CARON + {0x01D5, 0x01D5, prN}, // Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + {0x01D6, 0x01D6, prA}, // Ll LATIN SMALL LETTER U WITH DIAERESIS AND MACRON + {0x01D7, 0x01D7, prN}, // Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + {0x01D8, 0x01D8, prA}, // Ll LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE + {0x01D9, 0x01D9, prN}, // Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + {0x01DA, 0x01DA, prA}, // Ll LATIN SMALL LETTER U WITH DIAERESIS AND CARON + {0x01DB, 0x01DB, prN}, // Lu LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + {0x01DC, 0x01DC, prA}, // Ll LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE + {0x01DD, 0x024F, prN}, // L& [115] LATIN SMALL LETTER TURNED E..LATIN SMALL LETTER Y WITH STROKE + {0x0250, 0x0250, prN}, // Ll LATIN SMALL LETTER TURNED A + {0x0251, 0x0251, prA}, // Ll LATIN SMALL LETTER ALPHA + {0x0252, 0x0260, prN}, // Ll [15] LATIN SMALL LETTER TURNED ALPHA..LATIN SMALL LETTER G WITH HOOK + {0x0261, 0x0261, prA}, // Ll LATIN SMALL LETTER SCRIPT G + {0x0262, 0x0293, prN}, // Ll [50] LATIN LETTER SMALL CAPITAL G..LATIN SMALL LETTER EZH WITH CURL + {0x0294, 0x0294, prN}, // Lo LATIN LETTER GLOTTAL STOP + {0x0295, 0x02AF, prN}, // Ll [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL + {0x02B0, 0x02C1, prN}, // Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP + {0x02C2, 0x02C3, prN}, // Sk [2] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER RIGHT ARROWHEAD + {0x02C4, 0x02C4, prA}, // Sk MODIFIER LETTER UP ARROWHEAD + {0x02C5, 0x02C5, prN}, // Sk MODIFIER LETTER DOWN ARROWHEAD + {0x02C6, 0x02C6, prN}, // Lm MODIFIER LETTER CIRCUMFLEX ACCENT + {0x02C7, 0x02C7, prA}, // Lm CARON + {0x02C8, 0x02C8, prN}, // Lm MODIFIER LETTER VERTICAL LINE + {0x02C9, 0x02CB, prA}, // Lm [3] MODIFIER LETTER MACRON..MODIFIER LETTER GRAVE ACCENT + {0x02CC, 0x02CC, prN}, // Lm MODIFIER LETTER LOW VERTICAL LINE + {0x02CD, 0x02CD, prA}, // Lm MODIFIER LETTER LOW MACRON + {0x02CE, 0x02CF, prN}, // Lm [2] MODIFIER LETTER LOW GRAVE ACCENT..MODIFIER LETTER LOW ACUTE ACCENT + {0x02D0, 0x02D0, prA}, // Lm MODIFIER LETTER TRIANGULAR COLON + {0x02D1, 0x02D1, prN}, // Lm MODIFIER LETTER HALF TRIANGULAR COLON + {0x02D2, 0x02D7, prN}, // Sk [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN + {0x02D8, 0x02DB, prA}, // Sk [4] BREVE..OGONEK + {0x02DC, 0x02DC, prN}, // Sk SMALL TILDE + {0x02DD, 0x02DD, prA}, // Sk DOUBLE ACUTE ACCENT + {0x02DE, 0x02DE, prN}, // Sk MODIFIER LETTER RHOTIC HOOK + {0x02DF, 0x02DF, prA}, // Sk MODIFIER LETTER CROSS ACCENT + {0x02E0, 0x02E4, prN}, // Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP + {0x02E5, 0x02EB, prN}, // Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK + {0x02EC, 0x02EC, prN}, // Lm MODIFIER LETTER VOICING + {0x02ED, 0x02ED, prN}, // Sk MODIFIER LETTER UNASPIRATED + {0x02EE, 0x02EE, prN}, // Lm MODIFIER LETTER DOUBLE APOSTROPHE + {0x02EF, 0x02FF, prN}, // Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW + {0x0300, 0x036F, prA}, // Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X + {0x0370, 0x0373, prN}, // L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI + {0x0374, 0x0374, prN}, // Lm GREEK NUMERAL SIGN + {0x0375, 0x0375, prN}, // Sk GREEK LOWER NUMERAL SIGN + {0x0376, 0x0377, prN}, // L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA + {0x037A, 0x037A, prN}, // Lm GREEK YPOGEGRAMMENI + {0x037B, 0x037D, prN}, // Ll [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL + {0x037E, 0x037E, prN}, // Po GREEK QUESTION MARK + {0x037F, 0x037F, prN}, // Lu GREEK CAPITAL LETTER YOT + {0x0384, 0x0385, prN}, // Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS + {0x0386, 0x0386, prN}, // Lu GREEK CAPITAL LETTER ALPHA WITH TONOS + {0x0387, 0x0387, prN}, // Po GREEK ANO TELEIA + {0x0388, 0x038A, prN}, // Lu [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS + {0x038C, 0x038C, prN}, // Lu GREEK CAPITAL LETTER OMICRON WITH TONOS + {0x038E, 0x0390, prN}, // L& [3] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + {0x0391, 0x03A1, prA}, // Lu [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO + {0x03A3, 0x03A9, prA}, // Lu [7] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER OMEGA + {0x03AA, 0x03B0, prN}, // L& [7] GREEK CAPITAL LETTER IOTA WITH DIALYTIKA..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + {0x03B1, 0x03C1, prA}, // Ll [17] GREEK SMALL LETTER ALPHA..GREEK SMALL LETTER RHO + {0x03C2, 0x03C2, prN}, // Ll GREEK SMALL LETTER FINAL SIGMA + {0x03C3, 0x03C9, prA}, // Ll [7] GREEK SMALL LETTER SIGMA..GREEK SMALL LETTER OMEGA + {0x03CA, 0x03F5, prN}, // L& [44] GREEK SMALL LETTER IOTA WITH DIALYTIKA..GREEK LUNATE EPSILON SYMBOL + {0x03F6, 0x03F6, prN}, // Sm GREEK REVERSED LUNATE EPSILON SYMBOL + {0x03F7, 0x03FF, prN}, // L& [9] GREEK CAPITAL LETTER SHO..GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL + {0x0400, 0x0400, prN}, // Lu CYRILLIC CAPITAL LETTER IE WITH GRAVE + {0x0401, 0x0401, prA}, // Lu CYRILLIC CAPITAL LETTER IO + {0x0402, 0x040F, prN}, // Lu [14] CYRILLIC CAPITAL LETTER DJE..CYRILLIC CAPITAL LETTER DZHE + {0x0410, 0x044F, prA}, // L& [64] CYRILLIC CAPITAL LETTER A..CYRILLIC SMALL LETTER YA + {0x0450, 0x0450, prN}, // Ll CYRILLIC SMALL LETTER IE WITH GRAVE + {0x0451, 0x0451, prA}, // Ll CYRILLIC SMALL LETTER IO + {0x0452, 0x0481, prN}, // L& [48] CYRILLIC SMALL LETTER DJE..CYRILLIC SMALL LETTER KOPPA + {0x0482, 0x0482, prN}, // So CYRILLIC THOUSANDS SIGN + {0x0483, 0x0487, prN}, // Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE + {0x0488, 0x0489, prN}, // Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN + {0x048A, 0x04FF, prN}, // L& [118] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER HA WITH STROKE + {0x0500, 0x052F, prN}, // L& [48] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER EL WITH DESCENDER + {0x0531, 0x0556, prN}, // Lu [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH + {0x0559, 0x0559, prN}, // Lm ARMENIAN MODIFIER LETTER LEFT HALF RING + {0x055A, 0x055F, prN}, // Po [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK + {0x0560, 0x0588, prN}, // Ll [41] ARMENIAN SMALL LETTER TURNED AYB..ARMENIAN SMALL LETTER YI WITH STROKE + {0x0589, 0x0589, prN}, // Po ARMENIAN FULL STOP + {0x058A, 0x058A, prN}, // Pd ARMENIAN HYPHEN + {0x058D, 0x058E, prN}, // So [2] RIGHT-FACING ARMENIAN ETERNITY SIGN..LEFT-FACING ARMENIAN ETERNITY SIGN + {0x058F, 0x058F, prN}, // Sc ARMENIAN DRAM SIGN + {0x0591, 0x05BD, prN}, // Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG + {0x05BE, 0x05BE, prN}, // Pd HEBREW PUNCTUATION MAQAF + {0x05BF, 0x05BF, prN}, // Mn HEBREW POINT RAFE + {0x05C0, 0x05C0, prN}, // Po HEBREW PUNCTUATION PASEQ + {0x05C1, 0x05C2, prN}, // Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT + {0x05C3, 0x05C3, prN}, // Po HEBREW PUNCTUATION SOF PASUQ + {0x05C4, 0x05C5, prN}, // Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT + {0x05C6, 0x05C6, prN}, // Po HEBREW PUNCTUATION NUN HAFUKHA + {0x05C7, 0x05C7, prN}, // Mn HEBREW POINT QAMATS QATAN + {0x05D0, 0x05EA, prN}, // Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV + {0x05EF, 0x05F2, prN}, // Lo [4] HEBREW YOD TRIANGLE..HEBREW LIGATURE YIDDISH DOUBLE YOD + {0x05F3, 0x05F4, prN}, // Po [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM + {0x0600, 0x0605, prN}, // Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE + {0x0606, 0x0608, prN}, // Sm [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY + {0x0609, 0x060A, prN}, // Po [2] ARABIC-INDIC PER MILLE SIGN..ARABIC-INDIC PER TEN THOUSAND SIGN + {0x060B, 0x060B, prN}, // Sc AFGHANI SIGN + {0x060C, 0x060D, prN}, // Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR + {0x060E, 0x060F, prN}, // So [2] ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA + {0x0610, 0x061A, prN}, // Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA + {0x061B, 0x061B, prN}, // Po ARABIC SEMICOLON + {0x061C, 0x061C, prN}, // Cf ARABIC LETTER MARK + {0x061D, 0x061F, prN}, // Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK + {0x0620, 0x063F, prN}, // Lo [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE + {0x0640, 0x0640, prN}, // Lm ARABIC TATWEEL + {0x0641, 0x064A, prN}, // Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH + {0x064B, 0x065F, prN}, // Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW + {0x0660, 0x0669, prN}, // Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE + {0x066A, 0x066D, prN}, // Po [4] ARABIC PERCENT SIGN..ARABIC FIVE POINTED STAR + {0x066E, 0x066F, prN}, // Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF + {0x0670, 0x0670, prN}, // Mn ARABIC LETTER SUPERSCRIPT ALEF + {0x0671, 0x06D3, prN}, // Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE + {0x06D4, 0x06D4, prN}, // Po ARABIC FULL STOP + {0x06D5, 0x06D5, prN}, // Lo ARABIC LETTER AE + {0x06D6, 0x06DC, prN}, // Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN + {0x06DD, 0x06DD, prN}, // Cf ARABIC END OF AYAH + {0x06DE, 0x06DE, prN}, // So ARABIC START OF RUB EL HIZB + {0x06DF, 0x06E4, prN}, // Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA + {0x06E5, 0x06E6, prN}, // Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH + {0x06E7, 0x06E8, prN}, // Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON + {0x06E9, 0x06E9, prN}, // So ARABIC PLACE OF SAJDAH + {0x06EA, 0x06ED, prN}, // Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM + {0x06EE, 0x06EF, prN}, // Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V + {0x06F0, 0x06F9, prN}, // Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE + {0x06FA, 0x06FC, prN}, // Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW + {0x06FD, 0x06FE, prN}, // So [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN + {0x06FF, 0x06FF, prN}, // Lo ARABIC LETTER HEH WITH INVERTED V + {0x0700, 0x070D, prN}, // Po [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS + {0x070F, 0x070F, prN}, // Cf SYRIAC ABBREVIATION MARK + {0x0710, 0x0710, prN}, // Lo SYRIAC LETTER ALAPH + {0x0711, 0x0711, prN}, // Mn SYRIAC LETTER SUPERSCRIPT ALAPH + {0x0712, 0x072F, prN}, // Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH + {0x0730, 0x074A, prN}, // Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH + {0x074D, 0x074F, prN}, // Lo [3] SYRIAC LETTER SOGDIAN ZHAIN..SYRIAC LETTER SOGDIAN FE + {0x0750, 0x077F, prN}, // Lo [48] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE + {0x0780, 0x07A5, prN}, // Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU + {0x07A6, 0x07B0, prN}, // Mn [11] THAANA ABAFILI..THAANA SUKUN + {0x07B1, 0x07B1, prN}, // Lo THAANA LETTER NAA + {0x07C0, 0x07C9, prN}, // Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE + {0x07CA, 0x07EA, prN}, // Lo [33] NKO LETTER A..NKO LETTER JONA RA + {0x07EB, 0x07F3, prN}, // Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE + {0x07F4, 0x07F5, prN}, // Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE + {0x07F6, 0x07F6, prN}, // So NKO SYMBOL OO DENNEN + {0x07F7, 0x07F9, prN}, // Po [3] NKO SYMBOL GBAKURUNEN..NKO EXCLAMATION MARK + {0x07FA, 0x07FA, prN}, // Lm NKO LAJANYALAN + {0x07FD, 0x07FD, prN}, // Mn NKO DANTAYALAN + {0x07FE, 0x07FF, prN}, // Sc [2] NKO DOROME SIGN..NKO TAMAN SIGN + {0x0800, 0x0815, prN}, // Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF + {0x0816, 0x0819, prN}, // Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH + {0x081A, 0x081A, prN}, // Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT + {0x081B, 0x0823, prN}, // Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A + {0x0824, 0x0824, prN}, // Lm SAMARITAN MODIFIER LETTER SHORT A + {0x0825, 0x0827, prN}, // Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U + {0x0828, 0x0828, prN}, // Lm SAMARITAN MODIFIER LETTER I + {0x0829, 0x082D, prN}, // Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA + {0x0830, 0x083E, prN}, // Po [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU + {0x0840, 0x0858, prN}, // Lo [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN + {0x0859, 0x085B, prN}, // Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK + {0x085E, 0x085E, prN}, // Po MANDAIC PUNCTUATION + {0x0860, 0x086A, prN}, // Lo [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA + {0x0870, 0x0887, prN}, // Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT + {0x0888, 0x0888, prN}, // Sk ARABIC RAISED ROUND DOT + {0x0889, 0x088E, prN}, // Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL + {0x0890, 0x0891, prN}, // Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE + {0x0898, 0x089F, prN}, // Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA + {0x08A0, 0x08C8, prN}, // Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF + {0x08C9, 0x08C9, prN}, // Lm ARABIC SMALL FARSI YEH + {0x08CA, 0x08E1, prN}, // Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA + {0x08E2, 0x08E2, prN}, // Cf ARABIC DISPUTED END OF AYAH + {0x08E3, 0x08FF, prN}, // Mn [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA + {0x0900, 0x0902, prN}, // Mn [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA + {0x0903, 0x0903, prN}, // Mc DEVANAGARI SIGN VISARGA + {0x0904, 0x0939, prN}, // Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA + {0x093A, 0x093A, prN}, // Mn DEVANAGARI VOWEL SIGN OE + {0x093B, 0x093B, prN}, // Mc DEVANAGARI VOWEL SIGN OOE + {0x093C, 0x093C, prN}, // Mn DEVANAGARI SIGN NUKTA + {0x093D, 0x093D, prN}, // Lo DEVANAGARI SIGN AVAGRAHA + {0x093E, 0x0940, prN}, // Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II + {0x0941, 0x0948, prN}, // Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI + {0x0949, 0x094C, prN}, // Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU + {0x094D, 0x094D, prN}, // Mn DEVANAGARI SIGN VIRAMA + {0x094E, 0x094F, prN}, // Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW + {0x0950, 0x0950, prN}, // Lo DEVANAGARI OM + {0x0951, 0x0957, prN}, // Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE + {0x0958, 0x0961, prN}, // Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL + {0x0962, 0x0963, prN}, // Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL + {0x0964, 0x0965, prN}, // Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA + {0x0966, 0x096F, prN}, // Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE + {0x0970, 0x0970, prN}, // Po DEVANAGARI ABBREVIATION SIGN + {0x0971, 0x0971, prN}, // Lm DEVANAGARI SIGN HIGH SPACING DOT + {0x0972, 0x097F, prN}, // Lo [14] DEVANAGARI LETTER CANDRA A..DEVANAGARI LETTER BBA + {0x0980, 0x0980, prN}, // Lo BENGALI ANJI + {0x0981, 0x0981, prN}, // Mn BENGALI SIGN CANDRABINDU + {0x0982, 0x0983, prN}, // Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA + {0x0985, 0x098C, prN}, // Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L + {0x098F, 0x0990, prN}, // Lo [2] BENGALI LETTER E..BENGALI LETTER AI + {0x0993, 0x09A8, prN}, // Lo [22] BENGALI LETTER O..BENGALI LETTER NA + {0x09AA, 0x09B0, prN}, // Lo [7] BENGALI LETTER PA..BENGALI LETTER RA + {0x09B2, 0x09B2, prN}, // Lo BENGALI LETTER LA + {0x09B6, 0x09B9, prN}, // Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA + {0x09BC, 0x09BC, prN}, // Mn BENGALI SIGN NUKTA + {0x09BD, 0x09BD, prN}, // Lo BENGALI SIGN AVAGRAHA + {0x09BE, 0x09C0, prN}, // Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II + {0x09C1, 0x09C4, prN}, // Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR + {0x09C7, 0x09C8, prN}, // Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI + {0x09CB, 0x09CC, prN}, // Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU + {0x09CD, 0x09CD, prN}, // Mn BENGALI SIGN VIRAMA + {0x09CE, 0x09CE, prN}, // Lo BENGALI LETTER KHANDA TA + {0x09D7, 0x09D7, prN}, // Mc BENGALI AU LENGTH MARK + {0x09DC, 0x09DD, prN}, // Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA + {0x09DF, 0x09E1, prN}, // Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL + {0x09E2, 0x09E3, prN}, // Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL + {0x09E6, 0x09EF, prN}, // Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE + {0x09F0, 0x09F1, prN}, // Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL + {0x09F2, 0x09F3, prN}, // Sc [2] BENGALI RUPEE MARK..BENGALI RUPEE SIGN + {0x09F4, 0x09F9, prN}, // No [6] BENGALI CURRENCY NUMERATOR ONE..BENGALI CURRENCY DENOMINATOR SIXTEEN + {0x09FA, 0x09FA, prN}, // So BENGALI ISSHAR + {0x09FB, 0x09FB, prN}, // Sc BENGALI GANDA MARK + {0x09FC, 0x09FC, prN}, // Lo BENGALI LETTER VEDIC ANUSVARA + {0x09FD, 0x09FD, prN}, // Po BENGALI ABBREVIATION SIGN + {0x09FE, 0x09FE, prN}, // Mn BENGALI SANDHI MARK + {0x0A01, 0x0A02, prN}, // Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI + {0x0A03, 0x0A03, prN}, // Mc GURMUKHI SIGN VISARGA + {0x0A05, 0x0A0A, prN}, // Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU + {0x0A0F, 0x0A10, prN}, // Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI + {0x0A13, 0x0A28, prN}, // Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA + {0x0A2A, 0x0A30, prN}, // Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA + {0x0A32, 0x0A33, prN}, // Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA + {0x0A35, 0x0A36, prN}, // Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA + {0x0A38, 0x0A39, prN}, // Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA + {0x0A3C, 0x0A3C, prN}, // Mn GURMUKHI SIGN NUKTA + {0x0A3E, 0x0A40, prN}, // Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II + {0x0A41, 0x0A42, prN}, // Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU + {0x0A47, 0x0A48, prN}, // Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI + {0x0A4B, 0x0A4D, prN}, // Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA + {0x0A51, 0x0A51, prN}, // Mn GURMUKHI SIGN UDAAT + {0x0A59, 0x0A5C, prN}, // Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA + {0x0A5E, 0x0A5E, prN}, // Lo GURMUKHI LETTER FA + {0x0A66, 0x0A6F, prN}, // Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE + {0x0A70, 0x0A71, prN}, // Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK + {0x0A72, 0x0A74, prN}, // Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR + {0x0A75, 0x0A75, prN}, // Mn GURMUKHI SIGN YAKASH + {0x0A76, 0x0A76, prN}, // Po GURMUKHI ABBREVIATION SIGN + {0x0A81, 0x0A82, prN}, // Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA + {0x0A83, 0x0A83, prN}, // Mc GUJARATI SIGN VISARGA + {0x0A85, 0x0A8D, prN}, // Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E + {0x0A8F, 0x0A91, prN}, // Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O + {0x0A93, 0x0AA8, prN}, // Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA + {0x0AAA, 0x0AB0, prN}, // Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA + {0x0AB2, 0x0AB3, prN}, // Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA + {0x0AB5, 0x0AB9, prN}, // Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA + {0x0ABC, 0x0ABC, prN}, // Mn GUJARATI SIGN NUKTA + {0x0ABD, 0x0ABD, prN}, // Lo GUJARATI SIGN AVAGRAHA + {0x0ABE, 0x0AC0, prN}, // Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II + {0x0AC1, 0x0AC5, prN}, // Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E + {0x0AC7, 0x0AC8, prN}, // Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI + {0x0AC9, 0x0AC9, prN}, // Mc GUJARATI VOWEL SIGN CANDRA O + {0x0ACB, 0x0ACC, prN}, // Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU + {0x0ACD, 0x0ACD, prN}, // Mn GUJARATI SIGN VIRAMA + {0x0AD0, 0x0AD0, prN}, // Lo GUJARATI OM + {0x0AE0, 0x0AE1, prN}, // Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL + {0x0AE2, 0x0AE3, prN}, // Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL + {0x0AE6, 0x0AEF, prN}, // Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE + {0x0AF0, 0x0AF0, prN}, // Po GUJARATI ABBREVIATION SIGN + {0x0AF1, 0x0AF1, prN}, // Sc GUJARATI RUPEE SIGN + {0x0AF9, 0x0AF9, prN}, // Lo GUJARATI LETTER ZHA + {0x0AFA, 0x0AFF, prN}, // Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE + {0x0B01, 0x0B01, prN}, // Mn ORIYA SIGN CANDRABINDU + {0x0B02, 0x0B03, prN}, // Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA + {0x0B05, 0x0B0C, prN}, // Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L + {0x0B0F, 0x0B10, prN}, // Lo [2] ORIYA LETTER E..ORIYA LETTER AI + {0x0B13, 0x0B28, prN}, // Lo [22] ORIYA LETTER O..ORIYA LETTER NA + {0x0B2A, 0x0B30, prN}, // Lo [7] ORIYA LETTER PA..ORIYA LETTER RA + {0x0B32, 0x0B33, prN}, // Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA + {0x0B35, 0x0B39, prN}, // Lo [5] ORIYA LETTER VA..ORIYA LETTER HA + {0x0B3C, 0x0B3C, prN}, // Mn ORIYA SIGN NUKTA + {0x0B3D, 0x0B3D, prN}, // Lo ORIYA SIGN AVAGRAHA + {0x0B3E, 0x0B3E, prN}, // Mc ORIYA VOWEL SIGN AA + {0x0B3F, 0x0B3F, prN}, // Mn ORIYA VOWEL SIGN I + {0x0B40, 0x0B40, prN}, // Mc ORIYA VOWEL SIGN II + {0x0B41, 0x0B44, prN}, // Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR + {0x0B47, 0x0B48, prN}, // Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI + {0x0B4B, 0x0B4C, prN}, // Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU + {0x0B4D, 0x0B4D, prN}, // Mn ORIYA SIGN VIRAMA + {0x0B55, 0x0B56, prN}, // Mn [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK + {0x0B57, 0x0B57, prN}, // Mc ORIYA AU LENGTH MARK + {0x0B5C, 0x0B5D, prN}, // Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA + {0x0B5F, 0x0B61, prN}, // Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL + {0x0B62, 0x0B63, prN}, // Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL + {0x0B66, 0x0B6F, prN}, // Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE + {0x0B70, 0x0B70, prN}, // So ORIYA ISSHAR + {0x0B71, 0x0B71, prN}, // Lo ORIYA LETTER WA + {0x0B72, 0x0B77, prN}, // No [6] ORIYA FRACTION ONE QUARTER..ORIYA FRACTION THREE SIXTEENTHS + {0x0B82, 0x0B82, prN}, // Mn TAMIL SIGN ANUSVARA + {0x0B83, 0x0B83, prN}, // Lo TAMIL SIGN VISARGA + {0x0B85, 0x0B8A, prN}, // Lo [6] TAMIL LETTER A..TAMIL LETTER UU + {0x0B8E, 0x0B90, prN}, // Lo [3] TAMIL LETTER E..TAMIL LETTER AI + {0x0B92, 0x0B95, prN}, // Lo [4] TAMIL LETTER O..TAMIL LETTER KA + {0x0B99, 0x0B9A, prN}, // Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA + {0x0B9C, 0x0B9C, prN}, // Lo TAMIL LETTER JA + {0x0B9E, 0x0B9F, prN}, // Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA + {0x0BA3, 0x0BA4, prN}, // Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA + {0x0BA8, 0x0BAA, prN}, // Lo [3] TAMIL LETTER NA..TAMIL LETTER PA + {0x0BAE, 0x0BB9, prN}, // Lo [12] TAMIL LETTER MA..TAMIL LETTER HA + {0x0BBE, 0x0BBF, prN}, // Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I + {0x0BC0, 0x0BC0, prN}, // Mn TAMIL VOWEL SIGN II + {0x0BC1, 0x0BC2, prN}, // Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU + {0x0BC6, 0x0BC8, prN}, // Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI + {0x0BCA, 0x0BCC, prN}, // Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU + {0x0BCD, 0x0BCD, prN}, // Mn TAMIL SIGN VIRAMA + {0x0BD0, 0x0BD0, prN}, // Lo TAMIL OM + {0x0BD7, 0x0BD7, prN}, // Mc TAMIL AU LENGTH MARK + {0x0BE6, 0x0BEF, prN}, // Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE + {0x0BF0, 0x0BF2, prN}, // No [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND + {0x0BF3, 0x0BF8, prN}, // So [6] TAMIL DAY SIGN..TAMIL AS ABOVE SIGN + {0x0BF9, 0x0BF9, prN}, // Sc TAMIL RUPEE SIGN + {0x0BFA, 0x0BFA, prN}, // So TAMIL NUMBER SIGN + {0x0C00, 0x0C00, prN}, // Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE + {0x0C01, 0x0C03, prN}, // Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA + {0x0C04, 0x0C04, prN}, // Mn TELUGU SIGN COMBINING ANUSVARA ABOVE + {0x0C05, 0x0C0C, prN}, // Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L + {0x0C0E, 0x0C10, prN}, // Lo [3] TELUGU LETTER E..TELUGU LETTER AI + {0x0C12, 0x0C28, prN}, // Lo [23] TELUGU LETTER O..TELUGU LETTER NA + {0x0C2A, 0x0C39, prN}, // Lo [16] TELUGU LETTER PA..TELUGU LETTER HA + {0x0C3C, 0x0C3C, prN}, // Mn TELUGU SIGN NUKTA + {0x0C3D, 0x0C3D, prN}, // Lo TELUGU SIGN AVAGRAHA + {0x0C3E, 0x0C40, prN}, // Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II + {0x0C41, 0x0C44, prN}, // Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR + {0x0C46, 0x0C48, prN}, // Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI + {0x0C4A, 0x0C4D, prN}, // Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA + {0x0C55, 0x0C56, prN}, // Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK + {0x0C58, 0x0C5A, prN}, // Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA + {0x0C5D, 0x0C5D, prN}, // Lo TELUGU LETTER NAKAARA POLLU + {0x0C60, 0x0C61, prN}, // Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL + {0x0C62, 0x0C63, prN}, // Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL + {0x0C66, 0x0C6F, prN}, // Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE + {0x0C77, 0x0C77, prN}, // Po TELUGU SIGN SIDDHAM + {0x0C78, 0x0C7E, prN}, // No [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR + {0x0C7F, 0x0C7F, prN}, // So TELUGU SIGN TUUMU + {0x0C80, 0x0C80, prN}, // Lo KANNADA SIGN SPACING CANDRABINDU + {0x0C81, 0x0C81, prN}, // Mn KANNADA SIGN CANDRABINDU + {0x0C82, 0x0C83, prN}, // Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA + {0x0C84, 0x0C84, prN}, // Po KANNADA SIGN SIDDHAM + {0x0C85, 0x0C8C, prN}, // Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L + {0x0C8E, 0x0C90, prN}, // Lo [3] KANNADA LETTER E..KANNADA LETTER AI + {0x0C92, 0x0CA8, prN}, // Lo [23] KANNADA LETTER O..KANNADA LETTER NA + {0x0CAA, 0x0CB3, prN}, // Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA + {0x0CB5, 0x0CB9, prN}, // Lo [5] KANNADA LETTER VA..KANNADA LETTER HA + {0x0CBC, 0x0CBC, prN}, // Mn KANNADA SIGN NUKTA + {0x0CBD, 0x0CBD, prN}, // Lo KANNADA SIGN AVAGRAHA + {0x0CBE, 0x0CBE, prN}, // Mc KANNADA VOWEL SIGN AA + {0x0CBF, 0x0CBF, prN}, // Mn KANNADA VOWEL SIGN I + {0x0CC0, 0x0CC4, prN}, // Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR + {0x0CC6, 0x0CC6, prN}, // Mn KANNADA VOWEL SIGN E + {0x0CC7, 0x0CC8, prN}, // Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI + {0x0CCA, 0x0CCB, prN}, // Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO + {0x0CCC, 0x0CCD, prN}, // Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA + {0x0CD5, 0x0CD6, prN}, // Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK + {0x0CDD, 0x0CDE, prN}, // Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA + {0x0CE0, 0x0CE1, prN}, // Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL + {0x0CE2, 0x0CE3, prN}, // Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL + {0x0CE6, 0x0CEF, prN}, // Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE + {0x0CF1, 0x0CF2, prN}, // Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA + {0x0CF3, 0x0CF3, prN}, // Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT + {0x0D00, 0x0D01, prN}, // Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU + {0x0D02, 0x0D03, prN}, // Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA + {0x0D04, 0x0D0C, prN}, // Lo [9] MALAYALAM LETTER VEDIC ANUSVARA..MALAYALAM LETTER VOCALIC L + {0x0D0E, 0x0D10, prN}, // Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI + {0x0D12, 0x0D3A, prN}, // Lo [41] MALAYALAM LETTER O..MALAYALAM LETTER TTTA + {0x0D3B, 0x0D3C, prN}, // Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA + {0x0D3D, 0x0D3D, prN}, // Lo MALAYALAM SIGN AVAGRAHA + {0x0D3E, 0x0D40, prN}, // Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II + {0x0D41, 0x0D44, prN}, // Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR + {0x0D46, 0x0D48, prN}, // Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI + {0x0D4A, 0x0D4C, prN}, // Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU + {0x0D4D, 0x0D4D, prN}, // Mn MALAYALAM SIGN VIRAMA + {0x0D4E, 0x0D4E, prN}, // Lo MALAYALAM LETTER DOT REPH + {0x0D4F, 0x0D4F, prN}, // So MALAYALAM SIGN PARA + {0x0D54, 0x0D56, prN}, // Lo [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL + {0x0D57, 0x0D57, prN}, // Mc MALAYALAM AU LENGTH MARK + {0x0D58, 0x0D5E, prN}, // No [7] MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH..MALAYALAM FRACTION ONE FIFTH + {0x0D5F, 0x0D61, prN}, // Lo [3] MALAYALAM LETTER ARCHAIC II..MALAYALAM LETTER VOCALIC LL + {0x0D62, 0x0D63, prN}, // Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL + {0x0D66, 0x0D6F, prN}, // Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE + {0x0D70, 0x0D78, prN}, // No [9] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE SIXTEENTHS + {0x0D79, 0x0D79, prN}, // So MALAYALAM DATE MARK + {0x0D7A, 0x0D7F, prN}, // Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K + {0x0D81, 0x0D81, prN}, // Mn SINHALA SIGN CANDRABINDU + {0x0D82, 0x0D83, prN}, // Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA + {0x0D85, 0x0D96, prN}, // Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA + {0x0D9A, 0x0DB1, prN}, // Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA + {0x0DB3, 0x0DBB, prN}, // Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA + {0x0DBD, 0x0DBD, prN}, // Lo SINHALA LETTER DANTAJA LAYANNA + {0x0DC0, 0x0DC6, prN}, // Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA + {0x0DCA, 0x0DCA, prN}, // Mn SINHALA SIGN AL-LAKUNA + {0x0DCF, 0x0DD1, prN}, // Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA + {0x0DD2, 0x0DD4, prN}, // Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA + {0x0DD6, 0x0DD6, prN}, // Mn SINHALA VOWEL SIGN DIGA PAA-PILLA + {0x0DD8, 0x0DDF, prN}, // Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA + {0x0DE6, 0x0DEF, prN}, // Nd [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE + {0x0DF2, 0x0DF3, prN}, // Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA + {0x0DF4, 0x0DF4, prN}, // Po SINHALA PUNCTUATION KUNDDALIYA + {0x0E01, 0x0E30, prN}, // Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A + {0x0E31, 0x0E31, prN}, // Mn THAI CHARACTER MAI HAN-AKAT + {0x0E32, 0x0E33, prN}, // Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM + {0x0E34, 0x0E3A, prN}, // Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU + {0x0E3F, 0x0E3F, prN}, // Sc THAI CURRENCY SYMBOL BAHT + {0x0E40, 0x0E45, prN}, // Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO + {0x0E46, 0x0E46, prN}, // Lm THAI CHARACTER MAIYAMOK + {0x0E47, 0x0E4E, prN}, // Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN + {0x0E4F, 0x0E4F, prN}, // Po THAI CHARACTER FONGMAN + {0x0E50, 0x0E59, prN}, // Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE + {0x0E5A, 0x0E5B, prN}, // Po [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT + {0x0E81, 0x0E82, prN}, // Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG + {0x0E84, 0x0E84, prN}, // Lo LAO LETTER KHO TAM + {0x0E86, 0x0E8A, prN}, // Lo [5] LAO LETTER PALI GHA..LAO LETTER SO TAM + {0x0E8C, 0x0EA3, prN}, // Lo [24] LAO LETTER PALI JHA..LAO LETTER LO LING + {0x0EA5, 0x0EA5, prN}, // Lo LAO LETTER LO LOOT + {0x0EA7, 0x0EB0, prN}, // Lo [10] LAO LETTER WO..LAO VOWEL SIGN A + {0x0EB1, 0x0EB1, prN}, // Mn LAO VOWEL SIGN MAI KAN + {0x0EB2, 0x0EB3, prN}, // Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM + {0x0EB4, 0x0EBC, prN}, // Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO + {0x0EBD, 0x0EBD, prN}, // Lo LAO SEMIVOWEL SIGN NYO + {0x0EC0, 0x0EC4, prN}, // Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI + {0x0EC6, 0x0EC6, prN}, // Lm LAO KO LA + {0x0EC8, 0x0ECE, prN}, // Mn [7] LAO TONE MAI EK..LAO YAMAKKAN + {0x0ED0, 0x0ED9, prN}, // Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE + {0x0EDC, 0x0EDF, prN}, // Lo [4] LAO HO NO..LAO LETTER KHMU NYO + {0x0F00, 0x0F00, prN}, // Lo TIBETAN SYLLABLE OM + {0x0F01, 0x0F03, prN}, // So [3] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA + {0x0F04, 0x0F12, prN}, // Po [15] TIBETAN MARK INITIAL YIG MGO MDUN MA..TIBETAN MARK RGYA GRAM SHAD + {0x0F13, 0x0F13, prN}, // So TIBETAN MARK CARET -DZUD RTAGS ME LONG CAN + {0x0F14, 0x0F14, prN}, // Po TIBETAN MARK GTER TSHEG + {0x0F15, 0x0F17, prN}, // So [3] TIBETAN LOGOTYPE SIGN CHAD RTAGS..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS + {0x0F18, 0x0F19, prN}, // Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS + {0x0F1A, 0x0F1F, prN}, // So [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG + {0x0F20, 0x0F29, prN}, // Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE + {0x0F2A, 0x0F33, prN}, // No [10] TIBETAN DIGIT HALF ONE..TIBETAN DIGIT HALF ZERO + {0x0F34, 0x0F34, prN}, // So TIBETAN MARK BSDUS RTAGS + {0x0F35, 0x0F35, prN}, // Mn TIBETAN MARK NGAS BZUNG NYI ZLA + {0x0F36, 0x0F36, prN}, // So TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN + {0x0F37, 0x0F37, prN}, // Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS + {0x0F38, 0x0F38, prN}, // So TIBETAN MARK CHE MGO + {0x0F39, 0x0F39, prN}, // Mn TIBETAN MARK TSA -PHRU + {0x0F3A, 0x0F3A, prN}, // Ps TIBETAN MARK GUG RTAGS GYON + {0x0F3B, 0x0F3B, prN}, // Pe TIBETAN MARK GUG RTAGS GYAS + {0x0F3C, 0x0F3C, prN}, // Ps TIBETAN MARK ANG KHANG GYON + {0x0F3D, 0x0F3D, prN}, // Pe TIBETAN MARK ANG KHANG GYAS + {0x0F3E, 0x0F3F, prN}, // Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES + {0x0F40, 0x0F47, prN}, // Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA + {0x0F49, 0x0F6C, prN}, // Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA + {0x0F71, 0x0F7E, prN}, // Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO + {0x0F7F, 0x0F7F, prN}, // Mc TIBETAN SIGN RNAM BCAD + {0x0F80, 0x0F84, prN}, // Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA + {0x0F85, 0x0F85, prN}, // Po TIBETAN MARK PALUTA + {0x0F86, 0x0F87, prN}, // Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS + {0x0F88, 0x0F8C, prN}, // Lo [5] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN INVERTED MCHU CAN + {0x0F8D, 0x0F97, prN}, // Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA + {0x0F99, 0x0FBC, prN}, // Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA + {0x0FBE, 0x0FC5, prN}, // So [8] TIBETAN KU RU KHA..TIBETAN SYMBOL RDO RJE + {0x0FC6, 0x0FC6, prN}, // Mn TIBETAN SYMBOL PADMA GDAN + {0x0FC7, 0x0FCC, prN}, // So [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL + {0x0FCE, 0x0FCF, prN}, // So [2] TIBETAN SIGN RDEL NAG RDEL DKAR..TIBETAN SIGN RDEL NAG GSUM + {0x0FD0, 0x0FD4, prN}, // Po [5] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA + {0x0FD5, 0x0FD8, prN}, // So [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS + {0x0FD9, 0x0FDA, prN}, // Po [2] TIBETAN MARK LEADING MCHAN RTAGS..TIBETAN MARK TRAILING MCHAN RTAGS + {0x1000, 0x102A, prN}, // Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU + {0x102B, 0x102C, prN}, // Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA + {0x102D, 0x1030, prN}, // Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU + {0x1031, 0x1031, prN}, // Mc MYANMAR VOWEL SIGN E + {0x1032, 0x1037, prN}, // Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW + {0x1038, 0x1038, prN}, // Mc MYANMAR SIGN VISARGA + {0x1039, 0x103A, prN}, // Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT + {0x103B, 0x103C, prN}, // Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA + {0x103D, 0x103E, prN}, // Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA + {0x103F, 0x103F, prN}, // Lo MYANMAR LETTER GREAT SA + {0x1040, 0x1049, prN}, // Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE + {0x104A, 0x104F, prN}, // Po [6] MYANMAR SIGN LITTLE SECTION..MYANMAR SYMBOL GENITIVE + {0x1050, 0x1055, prN}, // Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL + {0x1056, 0x1057, prN}, // Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR + {0x1058, 0x1059, prN}, // Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL + {0x105A, 0x105D, prN}, // Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE + {0x105E, 0x1060, prN}, // Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA + {0x1061, 0x1061, prN}, // Lo MYANMAR LETTER SGAW KAREN SHA + {0x1062, 0x1064, prN}, // Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO + {0x1065, 0x1066, prN}, // Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA + {0x1067, 0x106D, prN}, // Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 + {0x106E, 0x1070, prN}, // Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA + {0x1071, 0x1074, prN}, // Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE + {0x1075, 0x1081, prN}, // Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA + {0x1082, 0x1082, prN}, // Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA + {0x1083, 0x1084, prN}, // Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E + {0x1085, 0x1086, prN}, // Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y + {0x1087, 0x108C, prN}, // Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 + {0x108D, 0x108D, prN}, // Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE + {0x108E, 0x108E, prN}, // Lo MYANMAR LETTER RUMAI PALAUNG FA + {0x108F, 0x108F, prN}, // Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 + {0x1090, 0x1099, prN}, // Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE + {0x109A, 0x109C, prN}, // Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A + {0x109D, 0x109D, prN}, // Mn MYANMAR VOWEL SIGN AITON AI + {0x109E, 0x109F, prN}, // So [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION + {0x10A0, 0x10C5, prN}, // Lu [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE + {0x10C7, 0x10C7, prN}, // Lu GEORGIAN CAPITAL LETTER YN + {0x10CD, 0x10CD, prN}, // Lu GEORGIAN CAPITAL LETTER AEN + {0x10D0, 0x10FA, prN}, // Ll [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN + {0x10FB, 0x10FB, prN}, // Po GEORGIAN PARAGRAPH SEPARATOR + {0x10FC, 0x10FC, prN}, // Lm MODIFIER LETTER GEORGIAN NAR + {0x10FD, 0x10FF, prN}, // Ll [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN + {0x1100, 0x115F, prW}, // Lo [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER + {0x1160, 0x11FF, prN}, // Lo [160] HANGUL JUNGSEONG FILLER..HANGUL JONGSEONG SSANGNIEUN + {0x1200, 0x1248, prN}, // Lo [73] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE QWA + {0x124A, 0x124D, prN}, // Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE + {0x1250, 0x1256, prN}, // Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO + {0x1258, 0x1258, prN}, // Lo ETHIOPIC SYLLABLE QHWA + {0x125A, 0x125D, prN}, // Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE + {0x1260, 0x1288, prN}, // Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA + {0x128A, 0x128D, prN}, // Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE + {0x1290, 0x12B0, prN}, // Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA + {0x12B2, 0x12B5, prN}, // Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE + {0x12B8, 0x12BE, prN}, // Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO + {0x12C0, 0x12C0, prN}, // Lo ETHIOPIC SYLLABLE KXWA + {0x12C2, 0x12C5, prN}, // Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE + {0x12C8, 0x12D6, prN}, // Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O + {0x12D8, 0x1310, prN}, // Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA + {0x1312, 0x1315, prN}, // Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE + {0x1318, 0x135A, prN}, // Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA + {0x135D, 0x135F, prN}, // Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK + {0x1360, 0x1368, prN}, // Po [9] ETHIOPIC SECTION MARK..ETHIOPIC PARAGRAPH SEPARATOR + {0x1369, 0x137C, prN}, // No [20] ETHIOPIC DIGIT ONE..ETHIOPIC NUMBER TEN THOUSAND + {0x1380, 0x138F, prN}, // Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE + {0x1390, 0x1399, prN}, // So [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT + {0x13A0, 0x13F5, prN}, // Lu [86] CHEROKEE LETTER A..CHEROKEE LETTER MV + {0x13F8, 0x13FD, prN}, // Ll [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV + {0x1400, 0x1400, prN}, // Pd CANADIAN SYLLABICS HYPHEN + {0x1401, 0x166C, prN}, // Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA + {0x166D, 0x166D, prN}, // So CANADIAN SYLLABICS CHI SIGN + {0x166E, 0x166E, prN}, // Po CANADIAN SYLLABICS FULL STOP + {0x166F, 0x167F, prN}, // Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W + {0x1680, 0x1680, prN}, // Zs OGHAM SPACE MARK + {0x1681, 0x169A, prN}, // Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH + {0x169B, 0x169B, prN}, // Ps OGHAM FEATHER MARK + {0x169C, 0x169C, prN}, // Pe OGHAM REVERSED FEATHER MARK + {0x16A0, 0x16EA, prN}, // Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X + {0x16EB, 0x16ED, prN}, // Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION + {0x16EE, 0x16F0, prN}, // Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL + {0x16F1, 0x16F8, prN}, // Lo [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC + {0x1700, 0x1711, prN}, // Lo [18] TAGALOG LETTER A..TAGALOG LETTER HA + {0x1712, 0x1714, prN}, // Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA + {0x1715, 0x1715, prN}, // Mc TAGALOG SIGN PAMUDPOD + {0x171F, 0x171F, prN}, // Lo TAGALOG LETTER ARCHAIC RA + {0x1720, 0x1731, prN}, // Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA + {0x1732, 0x1733, prN}, // Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U + {0x1734, 0x1734, prN}, // Mc HANUNOO SIGN PAMUDPOD + {0x1735, 0x1736, prN}, // Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION + {0x1740, 0x1751, prN}, // Lo [18] BUHID LETTER A..BUHID LETTER HA + {0x1752, 0x1753, prN}, // Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U + {0x1760, 0x176C, prN}, // Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA + {0x176E, 0x1770, prN}, // Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA + {0x1772, 0x1773, prN}, // Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U + {0x1780, 0x17B3, prN}, // Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU + {0x17B4, 0x17B5, prN}, // Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA + {0x17B6, 0x17B6, prN}, // Mc KHMER VOWEL SIGN AA + {0x17B7, 0x17BD, prN}, // Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA + {0x17BE, 0x17C5, prN}, // Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU + {0x17C6, 0x17C6, prN}, // Mn KHMER SIGN NIKAHIT + {0x17C7, 0x17C8, prN}, // Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU + {0x17C9, 0x17D3, prN}, // Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT + {0x17D4, 0x17D6, prN}, // Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH + {0x17D7, 0x17D7, prN}, // Lm KHMER SIGN LEK TOO + {0x17D8, 0x17DA, prN}, // Po [3] KHMER SIGN BEYYAL..KHMER SIGN KOOMUUT + {0x17DB, 0x17DB, prN}, // Sc KHMER CURRENCY SYMBOL RIEL + {0x17DC, 0x17DC, prN}, // Lo KHMER SIGN AVAKRAHASANYA + {0x17DD, 0x17DD, prN}, // Mn KHMER SIGN ATTHACAN + {0x17E0, 0x17E9, prN}, // Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE + {0x17F0, 0x17F9, prN}, // No [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON + {0x1800, 0x1805, prN}, // Po [6] MONGOLIAN BIRGA..MONGOLIAN FOUR DOTS + {0x1806, 0x1806, prN}, // Pd MONGOLIAN TODO SOFT HYPHEN + {0x1807, 0x180A, prN}, // Po [4] MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER..MONGOLIAN NIRUGU + {0x180B, 0x180D, prN}, // Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE + {0x180E, 0x180E, prN}, // Cf MONGOLIAN VOWEL SEPARATOR + {0x180F, 0x180F, prN}, // Mn MONGOLIAN FREE VARIATION SELECTOR FOUR + {0x1810, 0x1819, prN}, // Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE + {0x1820, 0x1842, prN}, // Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI + {0x1843, 0x1843, prN}, // Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN + {0x1844, 0x1878, prN}, // Lo [53] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER CHA WITH TWO DOTS + {0x1880, 0x1884, prN}, // Lo [5] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER ALI GALI INVERTED UBADAMA + {0x1885, 0x1886, prN}, // Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA + {0x1887, 0x18A8, prN}, // Lo [34] MONGOLIAN LETTER ALI GALI A..MONGOLIAN LETTER MANCHU ALI GALI BHA + {0x18A9, 0x18A9, prN}, // Mn MONGOLIAN LETTER ALI GALI DAGALGA + {0x18AA, 0x18AA, prN}, // Lo MONGOLIAN LETTER MANCHU ALI GALI LHA + {0x18B0, 0x18F5, prN}, // Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S + {0x1900, 0x191E, prN}, // Lo [31] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER TRA + {0x1920, 0x1922, prN}, // Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U + {0x1923, 0x1926, prN}, // Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU + {0x1927, 0x1928, prN}, // Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O + {0x1929, 0x192B, prN}, // Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA + {0x1930, 0x1931, prN}, // Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA + {0x1932, 0x1932, prN}, // Mn LIMBU SMALL LETTER ANUSVARA + {0x1933, 0x1938, prN}, // Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA + {0x1939, 0x193B, prN}, // Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I + {0x1940, 0x1940, prN}, // So LIMBU SIGN LOO + {0x1944, 0x1945, prN}, // Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK + {0x1946, 0x194F, prN}, // Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE + {0x1950, 0x196D, prN}, // Lo [30] TAI LE LETTER KA..TAI LE LETTER AI + {0x1970, 0x1974, prN}, // Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 + {0x1980, 0x19AB, prN}, // Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA + {0x19B0, 0x19C9, prN}, // Lo [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 + {0x19D0, 0x19D9, prN}, // Nd [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE + {0x19DA, 0x19DA, prN}, // No NEW TAI LUE THAM DIGIT ONE + {0x19DE, 0x19DF, prN}, // So [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV + {0x19E0, 0x19FF, prN}, // So [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC + {0x1A00, 0x1A16, prN}, // Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA + {0x1A17, 0x1A18, prN}, // Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U + {0x1A19, 0x1A1A, prN}, // Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O + {0x1A1B, 0x1A1B, prN}, // Mn BUGINESE VOWEL SIGN AE + {0x1A1E, 0x1A1F, prN}, // Po [2] BUGINESE PALLAWA..BUGINESE END OF SECTION + {0x1A20, 0x1A54, prN}, // Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA + {0x1A55, 0x1A55, prN}, // Mc TAI THAM CONSONANT SIGN MEDIAL RA + {0x1A56, 0x1A56, prN}, // Mn TAI THAM CONSONANT SIGN MEDIAL LA + {0x1A57, 0x1A57, prN}, // Mc TAI THAM CONSONANT SIGN LA TANG LAI + {0x1A58, 0x1A5E, prN}, // Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA + {0x1A60, 0x1A60, prN}, // Mn TAI THAM SIGN SAKOT + {0x1A61, 0x1A61, prN}, // Mc TAI THAM VOWEL SIGN A + {0x1A62, 0x1A62, prN}, // Mn TAI THAM VOWEL SIGN MAI SAT + {0x1A63, 0x1A64, prN}, // Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA + {0x1A65, 0x1A6C, prN}, // Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW + {0x1A6D, 0x1A72, prN}, // Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI + {0x1A73, 0x1A7C, prN}, // Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN + {0x1A7F, 0x1A7F, prN}, // Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT + {0x1A80, 0x1A89, prN}, // Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE + {0x1A90, 0x1A99, prN}, // Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE + {0x1AA0, 0x1AA6, prN}, // Po [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA + {0x1AA7, 0x1AA7, prN}, // Lm TAI THAM SIGN MAI YAMOK + {0x1AA8, 0x1AAD, prN}, // Po [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG + {0x1AB0, 0x1ABD, prN}, // Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW + {0x1ABE, 0x1ABE, prN}, // Me COMBINING PARENTHESES OVERLAY + {0x1ABF, 0x1ACE, prN}, // Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T + {0x1B00, 0x1B03, prN}, // Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG + {0x1B04, 0x1B04, prN}, // Mc BALINESE SIGN BISAH + {0x1B05, 0x1B33, prN}, // Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA + {0x1B34, 0x1B34, prN}, // Mn BALINESE SIGN REREKAN + {0x1B35, 0x1B35, prN}, // Mc BALINESE VOWEL SIGN TEDUNG + {0x1B36, 0x1B3A, prN}, // Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA + {0x1B3B, 0x1B3B, prN}, // Mc BALINESE VOWEL SIGN RA REPA TEDUNG + {0x1B3C, 0x1B3C, prN}, // Mn BALINESE VOWEL SIGN LA LENGA + {0x1B3D, 0x1B41, prN}, // Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG + {0x1B42, 0x1B42, prN}, // Mn BALINESE VOWEL SIGN PEPET + {0x1B43, 0x1B44, prN}, // Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG + {0x1B45, 0x1B4C, prN}, // Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA + {0x1B50, 0x1B59, prN}, // Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE + {0x1B5A, 0x1B60, prN}, // Po [7] BALINESE PANTI..BALINESE PAMENENG + {0x1B61, 0x1B6A, prN}, // So [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE + {0x1B6B, 0x1B73, prN}, // Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG + {0x1B74, 0x1B7C, prN}, // So [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING + {0x1B7D, 0x1B7E, prN}, // Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG + {0x1B80, 0x1B81, prN}, // Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR + {0x1B82, 0x1B82, prN}, // Mc SUNDANESE SIGN PANGWISAD + {0x1B83, 0x1BA0, prN}, // Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA + {0x1BA1, 0x1BA1, prN}, // Mc SUNDANESE CONSONANT SIGN PAMINGKAL + {0x1BA2, 0x1BA5, prN}, // Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU + {0x1BA6, 0x1BA7, prN}, // Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG + {0x1BA8, 0x1BA9, prN}, // Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG + {0x1BAA, 0x1BAA, prN}, // Mc SUNDANESE SIGN PAMAAEH + {0x1BAB, 0x1BAD, prN}, // Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA + {0x1BAE, 0x1BAF, prN}, // Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA + {0x1BB0, 0x1BB9, prN}, // Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE + {0x1BBA, 0x1BBF, prN}, // Lo [6] SUNDANESE AVAGRAHA..SUNDANESE LETTER FINAL M + {0x1BC0, 0x1BE5, prN}, // Lo [38] BATAK LETTER A..BATAK LETTER U + {0x1BE6, 0x1BE6, prN}, // Mn BATAK SIGN TOMPI + {0x1BE7, 0x1BE7, prN}, // Mc BATAK VOWEL SIGN E + {0x1BE8, 0x1BE9, prN}, // Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE + {0x1BEA, 0x1BEC, prN}, // Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O + {0x1BED, 0x1BED, prN}, // Mn BATAK VOWEL SIGN KARO O + {0x1BEE, 0x1BEE, prN}, // Mc BATAK VOWEL SIGN U + {0x1BEF, 0x1BF1, prN}, // Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H + {0x1BF2, 0x1BF3, prN}, // Mc [2] BATAK PANGOLAT..BATAK PANONGONAN + {0x1BFC, 0x1BFF, prN}, // Po [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT + {0x1C00, 0x1C23, prN}, // Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A + {0x1C24, 0x1C2B, prN}, // Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU + {0x1C2C, 0x1C33, prN}, // Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T + {0x1C34, 0x1C35, prN}, // Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG + {0x1C36, 0x1C37, prN}, // Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA + {0x1C3B, 0x1C3F, prN}, // Po [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK + {0x1C40, 0x1C49, prN}, // Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE + {0x1C4D, 0x1C4F, prN}, // Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA + {0x1C50, 0x1C59, prN}, // Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE + {0x1C5A, 0x1C77, prN}, // Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH + {0x1C78, 0x1C7D, prN}, // Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD + {0x1C7E, 0x1C7F, prN}, // Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD + {0x1C80, 0x1C88, prN}, // Ll [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK + {0x1C90, 0x1CBA, prN}, // Lu [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN + {0x1CBD, 0x1CBF, prN}, // Lu [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + {0x1CC0, 0x1CC7, prN}, // Po [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA + {0x1CD0, 0x1CD2, prN}, // Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA + {0x1CD3, 0x1CD3, prN}, // Po VEDIC SIGN NIHSHVASA + {0x1CD4, 0x1CE0, prN}, // Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA + {0x1CE1, 0x1CE1, prN}, // Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA + {0x1CE2, 0x1CE8, prN}, // Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL + {0x1CE9, 0x1CEC, prN}, // Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL + {0x1CED, 0x1CED, prN}, // Mn VEDIC SIGN TIRYAK + {0x1CEE, 0x1CF3, prN}, // Lo [6] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ROTATED ARDHAVISARGA + {0x1CF4, 0x1CF4, prN}, // Mn VEDIC TONE CANDRA ABOVE + {0x1CF5, 0x1CF6, prN}, // Lo [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN UPADHMANIYA + {0x1CF7, 0x1CF7, prN}, // Mc VEDIC SIGN ATIKRAMA + {0x1CF8, 0x1CF9, prN}, // Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE + {0x1CFA, 0x1CFA, prN}, // Lo VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA + {0x1D00, 0x1D2B, prN}, // Ll [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL + {0x1D2C, 0x1D6A, prN}, // Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI + {0x1D6B, 0x1D77, prN}, // Ll [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G + {0x1D78, 0x1D78, prN}, // Lm MODIFIER LETTER CYRILLIC EN + {0x1D79, 0x1D7F, prN}, // Ll [7] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER UPSILON WITH STROKE + {0x1D80, 0x1D9A, prN}, // Ll [27] LATIN SMALL LETTER B WITH PALATAL HOOK..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK + {0x1D9B, 0x1DBF, prN}, // Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA + {0x1DC0, 0x1DFF, prN}, // Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW + {0x1E00, 0x1EFF, prN}, // L& [256] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER Y WITH LOOP + {0x1F00, 0x1F15, prN}, // L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA + {0x1F18, 0x1F1D, prN}, // Lu [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + {0x1F20, 0x1F45, prN}, // L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA + {0x1F48, 0x1F4D, prN}, // Lu [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + {0x1F50, 0x1F57, prN}, // Ll [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI + {0x1F59, 0x1F59, prN}, // Lu GREEK CAPITAL LETTER UPSILON WITH DASIA + {0x1F5B, 0x1F5B, prN}, // Lu GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + {0x1F5D, 0x1F5D, prN}, // Lu GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + {0x1F5F, 0x1F7D, prN}, // L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA + {0x1F80, 0x1FB4, prN}, // L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + {0x1FB6, 0x1FBC, prN}, // L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + {0x1FBD, 0x1FBD, prN}, // Sk GREEK KORONIS + {0x1FBE, 0x1FBE, prN}, // Ll GREEK PROSGEGRAMMENI + {0x1FBF, 0x1FC1, prN}, // Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI + {0x1FC2, 0x1FC4, prN}, // Ll [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + {0x1FC6, 0x1FCC, prN}, // L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + {0x1FCD, 0x1FCF, prN}, // Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI + {0x1FD0, 0x1FD3, prN}, // Ll [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + {0x1FD6, 0x1FDB, prN}, // L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA + {0x1FDD, 0x1FDF, prN}, // Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI + {0x1FE0, 0x1FEC, prN}, // L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA + {0x1FED, 0x1FEF, prN}, // Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA + {0x1FF2, 0x1FF4, prN}, // Ll [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + {0x1FF6, 0x1FFC, prN}, // L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + {0x1FFD, 0x1FFE, prN}, // Sk [2] GREEK OXIA..GREEK DASIA + {0x2000, 0x200A, prN}, // Zs [11] EN QUAD..HAIR SPACE + {0x200B, 0x200F, prN}, // Cf [5] ZERO WIDTH SPACE..RIGHT-TO-LEFT MARK + {0x2010, 0x2010, prA}, // Pd HYPHEN + {0x2011, 0x2012, prN}, // Pd [2] NON-BREAKING HYPHEN..FIGURE DASH + {0x2013, 0x2015, prA}, // Pd [3] EN DASH..HORIZONTAL BAR + {0x2016, 0x2016, prA}, // Po DOUBLE VERTICAL LINE + {0x2017, 0x2017, prN}, // Po DOUBLE LOW LINE + {0x2018, 0x2018, prA}, // Pi LEFT SINGLE QUOTATION MARK + {0x2019, 0x2019, prA}, // Pf RIGHT SINGLE QUOTATION MARK + {0x201A, 0x201A, prN}, // Ps SINGLE LOW-9 QUOTATION MARK + {0x201B, 0x201B, prN}, // Pi SINGLE HIGH-REVERSED-9 QUOTATION MARK + {0x201C, 0x201C, prA}, // Pi LEFT DOUBLE QUOTATION MARK + {0x201D, 0x201D, prA}, // Pf RIGHT DOUBLE QUOTATION MARK + {0x201E, 0x201E, prN}, // Ps DOUBLE LOW-9 QUOTATION MARK + {0x201F, 0x201F, prN}, // Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK + {0x2020, 0x2022, prA}, // Po [3] DAGGER..BULLET + {0x2023, 0x2023, prN}, // Po TRIANGULAR BULLET + {0x2024, 0x2027, prA}, // Po [4] ONE DOT LEADER..HYPHENATION POINT + {0x2028, 0x2028, prN}, // Zl LINE SEPARATOR + {0x2029, 0x2029, prN}, // Zp PARAGRAPH SEPARATOR + {0x202A, 0x202E, prN}, // Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE + {0x202F, 0x202F, prN}, // Zs NARROW NO-BREAK SPACE + {0x2030, 0x2030, prA}, // Po PER MILLE SIGN + {0x2031, 0x2031, prN}, // Po PER TEN THOUSAND SIGN + {0x2032, 0x2033, prA}, // Po [2] PRIME..DOUBLE PRIME + {0x2034, 0x2034, prN}, // Po TRIPLE PRIME + {0x2035, 0x2035, prA}, // Po REVERSED PRIME + {0x2036, 0x2038, prN}, // Po [3] REVERSED DOUBLE PRIME..CARET + {0x2039, 0x2039, prN}, // Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK + {0x203A, 0x203A, prN}, // Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + {0x203B, 0x203B, prA}, // Po REFERENCE MARK + {0x203C, 0x203D, prN}, // Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG + {0x203E, 0x203E, prA}, // Po OVERLINE + {0x203F, 0x2040, prN}, // Pc [2] UNDERTIE..CHARACTER TIE + {0x2041, 0x2043, prN}, // Po [3] CARET INSERTION POINT..HYPHEN BULLET + {0x2044, 0x2044, prN}, // Sm FRACTION SLASH + {0x2045, 0x2045, prN}, // Ps LEFT SQUARE BRACKET WITH QUILL + {0x2046, 0x2046, prN}, // Pe RIGHT SQUARE BRACKET WITH QUILL + {0x2047, 0x2051, prN}, // Po [11] DOUBLE QUESTION MARK..TWO ASTERISKS ALIGNED VERTICALLY + {0x2052, 0x2052, prN}, // Sm COMMERCIAL MINUS SIGN + {0x2053, 0x2053, prN}, // Po SWUNG DASH + {0x2054, 0x2054, prN}, // Pc INVERTED UNDERTIE + {0x2055, 0x205E, prN}, // Po [10] FLOWER PUNCTUATION MARK..VERTICAL FOUR DOTS + {0x205F, 0x205F, prN}, // Zs MEDIUM MATHEMATICAL SPACE + {0x2060, 0x2064, prN}, // Cf [5] WORD JOINER..INVISIBLE PLUS + {0x2066, 0x206F, prN}, // Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES + {0x2070, 0x2070, prN}, // No SUPERSCRIPT ZERO + {0x2071, 0x2071, prN}, // Lm SUPERSCRIPT LATIN SMALL LETTER I + {0x2074, 0x2074, prA}, // No SUPERSCRIPT FOUR + {0x2075, 0x2079, prN}, // No [5] SUPERSCRIPT FIVE..SUPERSCRIPT NINE + {0x207A, 0x207C, prN}, // Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN + {0x207D, 0x207D, prN}, // Ps SUPERSCRIPT LEFT PARENTHESIS + {0x207E, 0x207E, prN}, // Pe SUPERSCRIPT RIGHT PARENTHESIS + {0x207F, 0x207F, prA}, // Lm SUPERSCRIPT LATIN SMALL LETTER N + {0x2080, 0x2080, prN}, // No SUBSCRIPT ZERO + {0x2081, 0x2084, prA}, // No [4] SUBSCRIPT ONE..SUBSCRIPT FOUR + {0x2085, 0x2089, prN}, // No [5] SUBSCRIPT FIVE..SUBSCRIPT NINE + {0x208A, 0x208C, prN}, // Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN + {0x208D, 0x208D, prN}, // Ps SUBSCRIPT LEFT PARENTHESIS + {0x208E, 0x208E, prN}, // Pe SUBSCRIPT RIGHT PARENTHESIS + {0x2090, 0x209C, prN}, // Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T + {0x20A0, 0x20A8, prN}, // Sc [9] EURO-CURRENCY SIGN..RUPEE SIGN + {0x20A9, 0x20A9, prH}, // Sc WON SIGN + {0x20AA, 0x20AB, prN}, // Sc [2] NEW SHEQEL SIGN..DONG SIGN + {0x20AC, 0x20AC, prA}, // Sc EURO SIGN + {0x20AD, 0x20C0, prN}, // Sc [20] KIP SIGN..SOM SIGN + {0x20D0, 0x20DC, prN}, // Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE + {0x20DD, 0x20E0, prN}, // Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH + {0x20E1, 0x20E1, prN}, // Mn COMBINING LEFT RIGHT ARROW ABOVE + {0x20E2, 0x20E4, prN}, // Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE + {0x20E5, 0x20F0, prN}, // Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE + {0x2100, 0x2101, prN}, // So [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT + {0x2102, 0x2102, prN}, // Lu DOUBLE-STRUCK CAPITAL C + {0x2103, 0x2103, prA}, // So DEGREE CELSIUS + {0x2104, 0x2104, prN}, // So CENTRE LINE SYMBOL + {0x2105, 0x2105, prA}, // So CARE OF + {0x2106, 0x2106, prN}, // So CADA UNA + {0x2107, 0x2107, prN}, // Lu EULER CONSTANT + {0x2108, 0x2108, prN}, // So SCRUPLE + {0x2109, 0x2109, prA}, // So DEGREE FAHRENHEIT + {0x210A, 0x2112, prN}, // L& [9] SCRIPT SMALL G..SCRIPT CAPITAL L + {0x2113, 0x2113, prA}, // Ll SCRIPT SMALL L + {0x2114, 0x2114, prN}, // So L B BAR SYMBOL + {0x2115, 0x2115, prN}, // Lu DOUBLE-STRUCK CAPITAL N + {0x2116, 0x2116, prA}, // So NUMERO SIGN + {0x2117, 0x2117, prN}, // So SOUND RECORDING COPYRIGHT + {0x2118, 0x2118, prN}, // Sm SCRIPT CAPITAL P + {0x2119, 0x211D, prN}, // Lu [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R + {0x211E, 0x2120, prN}, // So [3] PRESCRIPTION TAKE..SERVICE MARK + {0x2121, 0x2122, prA}, // So [2] TELEPHONE SIGN..TRADE MARK SIGN + {0x2123, 0x2123, prN}, // So VERSICLE + {0x2124, 0x2124, prN}, // Lu DOUBLE-STRUCK CAPITAL Z + {0x2125, 0x2125, prN}, // So OUNCE SIGN + {0x2126, 0x2126, prA}, // Lu OHM SIGN + {0x2127, 0x2127, prN}, // So INVERTED OHM SIGN + {0x2128, 0x2128, prN}, // Lu BLACK-LETTER CAPITAL Z + {0x2129, 0x2129, prN}, // So TURNED GREEK SMALL LETTER IOTA + {0x212A, 0x212A, prN}, // Lu KELVIN SIGN + {0x212B, 0x212B, prA}, // Lu ANGSTROM SIGN + {0x212C, 0x212D, prN}, // Lu [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C + {0x212E, 0x212E, prN}, // So ESTIMATED SYMBOL + {0x212F, 0x2134, prN}, // L& [6] SCRIPT SMALL E..SCRIPT SMALL O + {0x2135, 0x2138, prN}, // Lo [4] ALEF SYMBOL..DALET SYMBOL + {0x2139, 0x2139, prN}, // Ll INFORMATION SOURCE + {0x213A, 0x213B, prN}, // So [2] ROTATED CAPITAL Q..FACSIMILE SIGN + {0x213C, 0x213F, prN}, // L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI + {0x2140, 0x2144, prN}, // Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y + {0x2145, 0x2149, prN}, // L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J + {0x214A, 0x214A, prN}, // So PROPERTY LINE + {0x214B, 0x214B, prN}, // Sm TURNED AMPERSAND + {0x214C, 0x214D, prN}, // So [2] PER SIGN..AKTIESELSKAB + {0x214E, 0x214E, prN}, // Ll TURNED SMALL F + {0x214F, 0x214F, prN}, // So SYMBOL FOR SAMARITAN SOURCE + {0x2150, 0x2152, prN}, // No [3] VULGAR FRACTION ONE SEVENTH..VULGAR FRACTION ONE TENTH + {0x2153, 0x2154, prA}, // No [2] VULGAR FRACTION ONE THIRD..VULGAR FRACTION TWO THIRDS + {0x2155, 0x215A, prN}, // No [6] VULGAR FRACTION ONE FIFTH..VULGAR FRACTION FIVE SIXTHS + {0x215B, 0x215E, prA}, // No [4] VULGAR FRACTION ONE EIGHTH..VULGAR FRACTION SEVEN EIGHTHS + {0x215F, 0x215F, prN}, // No FRACTION NUMERATOR ONE + {0x2160, 0x216B, prA}, // Nl [12] ROMAN NUMERAL ONE..ROMAN NUMERAL TWELVE + {0x216C, 0x216F, prN}, // Nl [4] ROMAN NUMERAL FIFTY..ROMAN NUMERAL ONE THOUSAND + {0x2170, 0x2179, prA}, // Nl [10] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL TEN + {0x217A, 0x2182, prN}, // Nl [9] SMALL ROMAN NUMERAL ELEVEN..ROMAN NUMERAL TEN THOUSAND + {0x2183, 0x2184, prN}, // L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C + {0x2185, 0x2188, prN}, // Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND + {0x2189, 0x2189, prA}, // No VULGAR FRACTION ZERO THIRDS + {0x218A, 0x218B, prN}, // So [2] TURNED DIGIT TWO..TURNED DIGIT THREE + {0x2190, 0x2194, prA}, // Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW + {0x2195, 0x2199, prA}, // So [5] UP DOWN ARROW..SOUTH WEST ARROW + {0x219A, 0x219B, prN}, // Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE + {0x219C, 0x219F, prN}, // So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW + {0x21A0, 0x21A0, prN}, // Sm RIGHTWARDS TWO HEADED ARROW + {0x21A1, 0x21A2, prN}, // So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL + {0x21A3, 0x21A3, prN}, // Sm RIGHTWARDS ARROW WITH TAIL + {0x21A4, 0x21A5, prN}, // So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR + {0x21A6, 0x21A6, prN}, // Sm RIGHTWARDS ARROW FROM BAR + {0x21A7, 0x21AD, prN}, // So [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW + {0x21AE, 0x21AE, prN}, // Sm LEFT RIGHT ARROW WITH STROKE + {0x21AF, 0x21B7, prN}, // So [9] DOWNWARDS ZIGZAG ARROW..CLOCKWISE TOP SEMICIRCLE ARROW + {0x21B8, 0x21B9, prA}, // So [2] NORTH WEST ARROW TO LONG BAR..LEFTWARDS ARROW TO BAR OVER RIGHTWARDS ARROW TO BAR + {0x21BA, 0x21CD, prN}, // So [20] ANTICLOCKWISE OPEN CIRCLE ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE + {0x21CE, 0x21CF, prN}, // Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE + {0x21D0, 0x21D1, prN}, // So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW + {0x21D2, 0x21D2, prA}, // Sm RIGHTWARDS DOUBLE ARROW + {0x21D3, 0x21D3, prN}, // So DOWNWARDS DOUBLE ARROW + {0x21D4, 0x21D4, prA}, // Sm LEFT RIGHT DOUBLE ARROW + {0x21D5, 0x21E6, prN}, // So [18] UP DOWN DOUBLE ARROW..LEFTWARDS WHITE ARROW + {0x21E7, 0x21E7, prA}, // So UPWARDS WHITE ARROW + {0x21E8, 0x21F3, prN}, // So [12] RIGHTWARDS WHITE ARROW..UP DOWN WHITE ARROW + {0x21F4, 0x21FF, prN}, // Sm [12] RIGHT ARROW WITH SMALL CIRCLE..LEFT RIGHT OPEN-HEADED ARROW + {0x2200, 0x2200, prA}, // Sm FOR ALL + {0x2201, 0x2201, prN}, // Sm COMPLEMENT + {0x2202, 0x2203, prA}, // Sm [2] PARTIAL DIFFERENTIAL..THERE EXISTS + {0x2204, 0x2206, prN}, // Sm [3] THERE DOES NOT EXIST..INCREMENT + {0x2207, 0x2208, prA}, // Sm [2] NABLA..ELEMENT OF + {0x2209, 0x220A, prN}, // Sm [2] NOT AN ELEMENT OF..SMALL ELEMENT OF + {0x220B, 0x220B, prA}, // Sm CONTAINS AS MEMBER + {0x220C, 0x220E, prN}, // Sm [3] DOES NOT CONTAIN AS MEMBER..END OF PROOF + {0x220F, 0x220F, prA}, // Sm N-ARY PRODUCT + {0x2210, 0x2210, prN}, // Sm N-ARY COPRODUCT + {0x2211, 0x2211, prA}, // Sm N-ARY SUMMATION + {0x2212, 0x2214, prN}, // Sm [3] MINUS SIGN..DOT PLUS + {0x2215, 0x2215, prA}, // Sm DIVISION SLASH + {0x2216, 0x2219, prN}, // Sm [4] SET MINUS..BULLET OPERATOR + {0x221A, 0x221A, prA}, // Sm SQUARE ROOT + {0x221B, 0x221C, prN}, // Sm [2] CUBE ROOT..FOURTH ROOT + {0x221D, 0x2220, prA}, // Sm [4] PROPORTIONAL TO..ANGLE + {0x2221, 0x2222, prN}, // Sm [2] MEASURED ANGLE..SPHERICAL ANGLE + {0x2223, 0x2223, prA}, // Sm DIVIDES + {0x2224, 0x2224, prN}, // Sm DOES NOT DIVIDE + {0x2225, 0x2225, prA}, // Sm PARALLEL TO + {0x2226, 0x2226, prN}, // Sm NOT PARALLEL TO + {0x2227, 0x222C, prA}, // Sm [6] LOGICAL AND..DOUBLE INTEGRAL + {0x222D, 0x222D, prN}, // Sm TRIPLE INTEGRAL + {0x222E, 0x222E, prA}, // Sm CONTOUR INTEGRAL + {0x222F, 0x2233, prN}, // Sm [5] SURFACE INTEGRAL..ANTICLOCKWISE CONTOUR INTEGRAL + {0x2234, 0x2237, prA}, // Sm [4] THEREFORE..PROPORTION + {0x2238, 0x223B, prN}, // Sm [4] DOT MINUS..HOMOTHETIC + {0x223C, 0x223D, prA}, // Sm [2] TILDE OPERATOR..REVERSED TILDE + {0x223E, 0x2247, prN}, // Sm [10] INVERTED LAZY S..NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO + {0x2248, 0x2248, prA}, // Sm ALMOST EQUAL TO + {0x2249, 0x224B, prN}, // Sm [3] NOT ALMOST EQUAL TO..TRIPLE TILDE + {0x224C, 0x224C, prA}, // Sm ALL EQUAL TO + {0x224D, 0x2251, prN}, // Sm [5] EQUIVALENT TO..GEOMETRICALLY EQUAL TO + {0x2252, 0x2252, prA}, // Sm APPROXIMATELY EQUAL TO OR THE IMAGE OF + {0x2253, 0x225F, prN}, // Sm [13] IMAGE OF OR APPROXIMATELY EQUAL TO..QUESTIONED EQUAL TO + {0x2260, 0x2261, prA}, // Sm [2] NOT EQUAL TO..IDENTICAL TO + {0x2262, 0x2263, prN}, // Sm [2] NOT IDENTICAL TO..STRICTLY EQUIVALENT TO + {0x2264, 0x2267, prA}, // Sm [4] LESS-THAN OR EQUAL TO..GREATER-THAN OVER EQUAL TO + {0x2268, 0x2269, prN}, // Sm [2] LESS-THAN BUT NOT EQUAL TO..GREATER-THAN BUT NOT EQUAL TO + {0x226A, 0x226B, prA}, // Sm [2] MUCH LESS-THAN..MUCH GREATER-THAN + {0x226C, 0x226D, prN}, // Sm [2] BETWEEN..NOT EQUIVALENT TO + {0x226E, 0x226F, prA}, // Sm [2] NOT LESS-THAN..NOT GREATER-THAN + {0x2270, 0x2281, prN}, // Sm [18] NEITHER LESS-THAN NOR EQUAL TO..DOES NOT SUCCEED + {0x2282, 0x2283, prA}, // Sm [2] SUBSET OF..SUPERSET OF + {0x2284, 0x2285, prN}, // Sm [2] NOT A SUBSET OF..NOT A SUPERSET OF + {0x2286, 0x2287, prA}, // Sm [2] SUBSET OF OR EQUAL TO..SUPERSET OF OR EQUAL TO + {0x2288, 0x2294, prN}, // Sm [13] NEITHER A SUBSET OF NOR EQUAL TO..SQUARE CUP + {0x2295, 0x2295, prA}, // Sm CIRCLED PLUS + {0x2296, 0x2298, prN}, // Sm [3] CIRCLED MINUS..CIRCLED DIVISION SLASH + {0x2299, 0x2299, prA}, // Sm CIRCLED DOT OPERATOR + {0x229A, 0x22A4, prN}, // Sm [11] CIRCLED RING OPERATOR..DOWN TACK + {0x22A5, 0x22A5, prA}, // Sm UP TACK + {0x22A6, 0x22BE, prN}, // Sm [25] ASSERTION..RIGHT ANGLE WITH ARC + {0x22BF, 0x22BF, prA}, // Sm RIGHT TRIANGLE + {0x22C0, 0x22FF, prN}, // Sm [64] N-ARY LOGICAL AND..Z NOTATION BAG MEMBERSHIP + {0x2300, 0x2307, prN}, // So [8] DIAMETER SIGN..WAVY LINE + {0x2308, 0x2308, prN}, // Ps LEFT CEILING + {0x2309, 0x2309, prN}, // Pe RIGHT CEILING + {0x230A, 0x230A, prN}, // Ps LEFT FLOOR + {0x230B, 0x230B, prN}, // Pe RIGHT FLOOR + {0x230C, 0x2311, prN}, // So [6] BOTTOM RIGHT CROP..SQUARE LOZENGE + {0x2312, 0x2312, prA}, // So ARC + {0x2313, 0x2319, prN}, // So [7] SEGMENT..TURNED NOT SIGN + {0x231A, 0x231B, prW}, // So [2] WATCH..HOURGLASS + {0x231C, 0x231F, prN}, // So [4] TOP LEFT CORNER..BOTTOM RIGHT CORNER + {0x2320, 0x2321, prN}, // Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL + {0x2322, 0x2328, prN}, // So [7] FROWN..KEYBOARD + {0x2329, 0x2329, prW}, // Ps LEFT-POINTING ANGLE BRACKET + {0x232A, 0x232A, prW}, // Pe RIGHT-POINTING ANGLE BRACKET + {0x232B, 0x237B, prN}, // So [81] ERASE TO THE LEFT..NOT CHECK MARK + {0x237C, 0x237C, prN}, // Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW + {0x237D, 0x239A, prN}, // So [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL + {0x239B, 0x23B3, prN}, // Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM + {0x23B4, 0x23DB, prN}, // So [40] TOP SQUARE BRACKET..FUSE + {0x23DC, 0x23E1, prN}, // Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET + {0x23E2, 0x23E8, prN}, // So [7] WHITE TRAPEZIUM..DECIMAL EXPONENT SYMBOL + {0x23E9, 0x23EC, prW}, // So [4] BLACK RIGHT-POINTING DOUBLE TRIANGLE..BLACK DOWN-POINTING DOUBLE TRIANGLE + {0x23ED, 0x23EF, prN}, // So [3] BLACK RIGHT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR..BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR + {0x23F0, 0x23F0, prW}, // So ALARM CLOCK + {0x23F1, 0x23F2, prN}, // So [2] STOPWATCH..TIMER CLOCK + {0x23F3, 0x23F3, prW}, // So HOURGLASS WITH FLOWING SAND + {0x23F4, 0x23FF, prN}, // So [12] BLACK MEDIUM LEFT-POINTING TRIANGLE..OBSERVER EYE SYMBOL + {0x2400, 0x2426, prN}, // So [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO + {0x2440, 0x244A, prN}, // So [11] OCR HOOK..OCR DOUBLE BACKSLASH + {0x2460, 0x249B, prA}, // No [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP + {0x249C, 0x24E9, prA}, // So [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z + {0x24EA, 0x24EA, prN}, // No CIRCLED DIGIT ZERO + {0x24EB, 0x24FF, prA}, // No [21] NEGATIVE CIRCLED NUMBER ELEVEN..NEGATIVE CIRCLED DIGIT ZERO + {0x2500, 0x254B, prA}, // So [76] BOX DRAWINGS LIGHT HORIZONTAL..BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL + {0x254C, 0x254F, prN}, // So [4] BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL..BOX DRAWINGS HEAVY DOUBLE DASH VERTICAL + {0x2550, 0x2573, prA}, // So [36] BOX DRAWINGS DOUBLE HORIZONTAL..BOX DRAWINGS LIGHT DIAGONAL CROSS + {0x2574, 0x257F, prN}, // So [12] BOX DRAWINGS LIGHT LEFT..BOX DRAWINGS HEAVY UP AND LIGHT DOWN + {0x2580, 0x258F, prA}, // So [16] UPPER HALF BLOCK..LEFT ONE EIGHTH BLOCK + {0x2590, 0x2591, prN}, // So [2] RIGHT HALF BLOCK..LIGHT SHADE + {0x2592, 0x2595, prA}, // So [4] MEDIUM SHADE..RIGHT ONE EIGHTH BLOCK + {0x2596, 0x259F, prN}, // So [10] QUADRANT LOWER LEFT..QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT + {0x25A0, 0x25A1, prA}, // So [2] BLACK SQUARE..WHITE SQUARE + {0x25A2, 0x25A2, prN}, // So WHITE SQUARE WITH ROUNDED CORNERS + {0x25A3, 0x25A9, prA}, // So [7] WHITE SQUARE CONTAINING BLACK SMALL SQUARE..SQUARE WITH DIAGONAL CROSSHATCH FILL + {0x25AA, 0x25B1, prN}, // So [8] BLACK SMALL SQUARE..WHITE PARALLELOGRAM + {0x25B2, 0x25B3, prA}, // So [2] BLACK UP-POINTING TRIANGLE..WHITE UP-POINTING TRIANGLE + {0x25B4, 0x25B5, prN}, // So [2] BLACK UP-POINTING SMALL TRIANGLE..WHITE UP-POINTING SMALL TRIANGLE + {0x25B6, 0x25B6, prA}, // So BLACK RIGHT-POINTING TRIANGLE + {0x25B7, 0x25B7, prA}, // Sm WHITE RIGHT-POINTING TRIANGLE + {0x25B8, 0x25BB, prN}, // So [4] BLACK RIGHT-POINTING SMALL TRIANGLE..WHITE RIGHT-POINTING POINTER + {0x25BC, 0x25BD, prA}, // So [2] BLACK DOWN-POINTING TRIANGLE..WHITE DOWN-POINTING TRIANGLE + {0x25BE, 0x25BF, prN}, // So [2] BLACK DOWN-POINTING SMALL TRIANGLE..WHITE DOWN-POINTING SMALL TRIANGLE + {0x25C0, 0x25C0, prA}, // So BLACK LEFT-POINTING TRIANGLE + {0x25C1, 0x25C1, prA}, // Sm WHITE LEFT-POINTING TRIANGLE + {0x25C2, 0x25C5, prN}, // So [4] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE LEFT-POINTING POINTER + {0x25C6, 0x25C8, prA}, // So [3] BLACK DIAMOND..WHITE DIAMOND CONTAINING BLACK SMALL DIAMOND + {0x25C9, 0x25CA, prN}, // So [2] FISHEYE..LOZENGE + {0x25CB, 0x25CB, prA}, // So WHITE CIRCLE + {0x25CC, 0x25CD, prN}, // So [2] DOTTED CIRCLE..CIRCLE WITH VERTICAL FILL + {0x25CE, 0x25D1, prA}, // So [4] BULLSEYE..CIRCLE WITH RIGHT HALF BLACK + {0x25D2, 0x25E1, prN}, // So [16] CIRCLE WITH LOWER HALF BLACK..LOWER HALF CIRCLE + {0x25E2, 0x25E5, prA}, // So [4] BLACK LOWER RIGHT TRIANGLE..BLACK UPPER RIGHT TRIANGLE + {0x25E6, 0x25EE, prN}, // So [9] WHITE BULLET..UP-POINTING TRIANGLE WITH RIGHT HALF BLACK + {0x25EF, 0x25EF, prA}, // So LARGE CIRCLE + {0x25F0, 0x25F7, prN}, // So [8] WHITE SQUARE WITH UPPER LEFT QUADRANT..WHITE CIRCLE WITH UPPER RIGHT QUADRANT + {0x25F8, 0x25FC, prN}, // Sm [5] UPPER LEFT TRIANGLE..BLACK MEDIUM SQUARE + {0x25FD, 0x25FE, prW}, // Sm [2] WHITE MEDIUM SMALL SQUARE..BLACK MEDIUM SMALL SQUARE + {0x25FF, 0x25FF, prN}, // Sm LOWER RIGHT TRIANGLE + {0x2600, 0x2604, prN}, // So [5] BLACK SUN WITH RAYS..COMET + {0x2605, 0x2606, prA}, // So [2] BLACK STAR..WHITE STAR + {0x2607, 0x2608, prN}, // So [2] LIGHTNING..THUNDERSTORM + {0x2609, 0x2609, prA}, // So SUN + {0x260A, 0x260D, prN}, // So [4] ASCENDING NODE..OPPOSITION + {0x260E, 0x260F, prA}, // So [2] BLACK TELEPHONE..WHITE TELEPHONE + {0x2610, 0x2613, prN}, // So [4] BALLOT BOX..SALTIRE + {0x2614, 0x2615, prW}, // So [2] UMBRELLA WITH RAIN DROPS..HOT BEVERAGE + {0x2616, 0x261B, prN}, // So [6] WHITE SHOGI PIECE..BLACK RIGHT POINTING INDEX + {0x261C, 0x261C, prA}, // So WHITE LEFT POINTING INDEX + {0x261D, 0x261D, prN}, // So WHITE UP POINTING INDEX + {0x261E, 0x261E, prA}, // So WHITE RIGHT POINTING INDEX + {0x261F, 0x263F, prN}, // So [33] WHITE DOWN POINTING INDEX..MERCURY + {0x2640, 0x2640, prA}, // So FEMALE SIGN + {0x2641, 0x2641, prN}, // So EARTH + {0x2642, 0x2642, prA}, // So MALE SIGN + {0x2643, 0x2647, prN}, // So [5] JUPITER..PLUTO + {0x2648, 0x2653, prW}, // So [12] ARIES..PISCES + {0x2654, 0x265F, prN}, // So [12] WHITE CHESS KING..BLACK CHESS PAWN + {0x2660, 0x2661, prA}, // So [2] BLACK SPADE SUIT..WHITE HEART SUIT + {0x2662, 0x2662, prN}, // So WHITE DIAMOND SUIT + {0x2663, 0x2665, prA}, // So [3] BLACK CLUB SUIT..BLACK HEART SUIT + {0x2666, 0x2666, prN}, // So BLACK DIAMOND SUIT + {0x2667, 0x266A, prA}, // So [4] WHITE CLUB SUIT..EIGHTH NOTE + {0x266B, 0x266B, prN}, // So BEAMED EIGHTH NOTES + {0x266C, 0x266D, prA}, // So [2] BEAMED SIXTEENTH NOTES..MUSIC FLAT SIGN + {0x266E, 0x266E, prN}, // So MUSIC NATURAL SIGN + {0x266F, 0x266F, prA}, // Sm MUSIC SHARP SIGN + {0x2670, 0x267E, prN}, // So [15] WEST SYRIAC CROSS..PERMANENT PAPER SIGN + {0x267F, 0x267F, prW}, // So WHEELCHAIR SYMBOL + {0x2680, 0x2692, prN}, // So [19] DIE FACE-1..HAMMER AND PICK + {0x2693, 0x2693, prW}, // So ANCHOR + {0x2694, 0x269D, prN}, // So [10] CROSSED SWORDS..OUTLINED WHITE STAR + {0x269E, 0x269F, prA}, // So [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT + {0x26A0, 0x26A0, prN}, // So WARNING SIGN + {0x26A1, 0x26A1, prW}, // So HIGH VOLTAGE SIGN + {0x26A2, 0x26A9, prN}, // So [8] DOUBLED FEMALE SIGN..HORIZONTAL MALE WITH STROKE SIGN + {0x26AA, 0x26AB, prW}, // So [2] MEDIUM WHITE CIRCLE..MEDIUM BLACK CIRCLE + {0x26AC, 0x26BC, prN}, // So [17] MEDIUM SMALL WHITE CIRCLE..SESQUIQUADRATE + {0x26BD, 0x26BE, prW}, // So [2] SOCCER BALL..BASEBALL + {0x26BF, 0x26BF, prA}, // So SQUARED KEY + {0x26C0, 0x26C3, prN}, // So [4] WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING + {0x26C4, 0x26C5, prW}, // So [2] SNOWMAN WITHOUT SNOW..SUN BEHIND CLOUD + {0x26C6, 0x26CD, prA}, // So [8] RAIN..DISABLED CAR + {0x26CE, 0x26CE, prW}, // So OPHIUCHUS + {0x26CF, 0x26D3, prA}, // So [5] PICK..CHAINS + {0x26D4, 0x26D4, prW}, // So NO ENTRY + {0x26D5, 0x26E1, prA}, // So [13] ALTERNATE ONE-WAY LEFT WAY TRAFFIC..RESTRICTED LEFT ENTRY-2 + {0x26E2, 0x26E2, prN}, // So ASTRONOMICAL SYMBOL FOR URANUS + {0x26E3, 0x26E3, prA}, // So HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE + {0x26E4, 0x26E7, prN}, // So [4] PENTAGRAM..INVERTED PENTAGRAM + {0x26E8, 0x26E9, prA}, // So [2] BLACK CROSS ON SHIELD..SHINTO SHRINE + {0x26EA, 0x26EA, prW}, // So CHURCH + {0x26EB, 0x26F1, prA}, // So [7] CASTLE..UMBRELLA ON GROUND + {0x26F2, 0x26F3, prW}, // So [2] FOUNTAIN..FLAG IN HOLE + {0x26F4, 0x26F4, prA}, // So FERRY + {0x26F5, 0x26F5, prW}, // So SAILBOAT + {0x26F6, 0x26F9, prA}, // So [4] SQUARE FOUR CORNERS..PERSON WITH BALL + {0x26FA, 0x26FA, prW}, // So TENT + {0x26FB, 0x26FC, prA}, // So [2] JAPANESE BANK SYMBOL..HEADSTONE GRAVEYARD SYMBOL + {0x26FD, 0x26FD, prW}, // So FUEL PUMP + {0x26FE, 0x26FF, prA}, // So [2] CUP ON BLACK SQUARE..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE + {0x2700, 0x2704, prN}, // So [5] BLACK SAFETY SCISSORS..WHITE SCISSORS + {0x2705, 0x2705, prW}, // So WHITE HEAVY CHECK MARK + {0x2706, 0x2709, prN}, // So [4] TELEPHONE LOCATION SIGN..ENVELOPE + {0x270A, 0x270B, prW}, // So [2] RAISED FIST..RAISED HAND + {0x270C, 0x2727, prN}, // So [28] VICTORY HAND..WHITE FOUR POINTED STAR + {0x2728, 0x2728, prW}, // So SPARKLES + {0x2729, 0x273C, prN}, // So [20] STRESS OUTLINED WHITE STAR..OPEN CENTRE TEARDROP-SPOKED ASTERISK + {0x273D, 0x273D, prA}, // So HEAVY TEARDROP-SPOKED ASTERISK + {0x273E, 0x274B, prN}, // So [14] SIX PETALLED BLACK AND WHITE FLORETTE..HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK + {0x274C, 0x274C, prW}, // So CROSS MARK + {0x274D, 0x274D, prN}, // So SHADOWED WHITE CIRCLE + {0x274E, 0x274E, prW}, // So NEGATIVE SQUARED CROSS MARK + {0x274F, 0x2752, prN}, // So [4] LOWER RIGHT DROP-SHADOWED WHITE SQUARE..UPPER RIGHT SHADOWED WHITE SQUARE + {0x2753, 0x2755, prW}, // So [3] BLACK QUESTION MARK ORNAMENT..WHITE EXCLAMATION MARK ORNAMENT + {0x2756, 0x2756, prN}, // So BLACK DIAMOND MINUS WHITE X + {0x2757, 0x2757, prW}, // So HEAVY EXCLAMATION MARK SYMBOL + {0x2758, 0x2767, prN}, // So [16] LIGHT VERTICAL BAR..ROTATED FLORAL HEART BULLET + {0x2768, 0x2768, prN}, // Ps MEDIUM LEFT PARENTHESIS ORNAMENT + {0x2769, 0x2769, prN}, // Pe MEDIUM RIGHT PARENTHESIS ORNAMENT + {0x276A, 0x276A, prN}, // Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT + {0x276B, 0x276B, prN}, // Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT + {0x276C, 0x276C, prN}, // Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT + {0x276D, 0x276D, prN}, // Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT + {0x276E, 0x276E, prN}, // Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT + {0x276F, 0x276F, prN}, // Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT + {0x2770, 0x2770, prN}, // Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT + {0x2771, 0x2771, prN}, // Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT + {0x2772, 0x2772, prN}, // Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT + {0x2773, 0x2773, prN}, // Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT + {0x2774, 0x2774, prN}, // Ps MEDIUM LEFT CURLY BRACKET ORNAMENT + {0x2775, 0x2775, prN}, // Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT + {0x2776, 0x277F, prA}, // No [10] DINGBAT NEGATIVE CIRCLED DIGIT ONE..DINGBAT NEGATIVE CIRCLED NUMBER TEN + {0x2780, 0x2793, prN}, // No [20] DINGBAT CIRCLED SANS-SERIF DIGIT ONE..DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN + {0x2794, 0x2794, prN}, // So HEAVY WIDE-HEADED RIGHTWARDS ARROW + {0x2795, 0x2797, prW}, // So [3] HEAVY PLUS SIGN..HEAVY DIVISION SIGN + {0x2798, 0x27AF, prN}, // So [24] HEAVY SOUTH EAST ARROW..NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW + {0x27B0, 0x27B0, prW}, // So CURLY LOOP + {0x27B1, 0x27BE, prN}, // So [14] NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW..OPEN-OUTLINED RIGHTWARDS ARROW + {0x27BF, 0x27BF, prW}, // So DOUBLE CURLY LOOP + {0x27C0, 0x27C4, prN}, // Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET + {0x27C5, 0x27C5, prN}, // Ps LEFT S-SHAPED BAG DELIMITER + {0x27C6, 0x27C6, prN}, // Pe RIGHT S-SHAPED BAG DELIMITER + {0x27C7, 0x27E5, prN}, // Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK + {0x27E6, 0x27E6, prNa}, // Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET + {0x27E7, 0x27E7, prNa}, // Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET + {0x27E8, 0x27E8, prNa}, // Ps MATHEMATICAL LEFT ANGLE BRACKET + {0x27E9, 0x27E9, prNa}, // Pe MATHEMATICAL RIGHT ANGLE BRACKET + {0x27EA, 0x27EA, prNa}, // Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET + {0x27EB, 0x27EB, prNa}, // Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET + {0x27EC, 0x27EC, prNa}, // Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET + {0x27ED, 0x27ED, prNa}, // Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET + {0x27EE, 0x27EE, prN}, // Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS + {0x27EF, 0x27EF, prN}, // Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS + {0x27F0, 0x27FF, prN}, // Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW + {0x2800, 0x28FF, prN}, // So [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 + {0x2900, 0x297F, prN}, // Sm [128] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..DOWN FISH TAIL + {0x2980, 0x2982, prN}, // Sm [3] TRIPLE VERTICAL BAR DELIMITER..Z NOTATION TYPE COLON + {0x2983, 0x2983, prN}, // Ps LEFT WHITE CURLY BRACKET + {0x2984, 0x2984, prN}, // Pe RIGHT WHITE CURLY BRACKET + {0x2985, 0x2985, prNa}, // Ps LEFT WHITE PARENTHESIS + {0x2986, 0x2986, prNa}, // Pe RIGHT WHITE PARENTHESIS + {0x2987, 0x2987, prN}, // Ps Z NOTATION LEFT IMAGE BRACKET + {0x2988, 0x2988, prN}, // Pe Z NOTATION RIGHT IMAGE BRACKET + {0x2989, 0x2989, prN}, // Ps Z NOTATION LEFT BINDING BRACKET + {0x298A, 0x298A, prN}, // Pe Z NOTATION RIGHT BINDING BRACKET + {0x298B, 0x298B, prN}, // Ps LEFT SQUARE BRACKET WITH UNDERBAR + {0x298C, 0x298C, prN}, // Pe RIGHT SQUARE BRACKET WITH UNDERBAR + {0x298D, 0x298D, prN}, // Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER + {0x298E, 0x298E, prN}, // Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + {0x298F, 0x298F, prN}, // Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + {0x2990, 0x2990, prN}, // Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER + {0x2991, 0x2991, prN}, // Ps LEFT ANGLE BRACKET WITH DOT + {0x2992, 0x2992, prN}, // Pe RIGHT ANGLE BRACKET WITH DOT + {0x2993, 0x2993, prN}, // Ps LEFT ARC LESS-THAN BRACKET + {0x2994, 0x2994, prN}, // Pe RIGHT ARC GREATER-THAN BRACKET + {0x2995, 0x2995, prN}, // Ps DOUBLE LEFT ARC GREATER-THAN BRACKET + {0x2996, 0x2996, prN}, // Pe DOUBLE RIGHT ARC LESS-THAN BRACKET + {0x2997, 0x2997, prN}, // Ps LEFT BLACK TORTOISE SHELL BRACKET + {0x2998, 0x2998, prN}, // Pe RIGHT BLACK TORTOISE SHELL BRACKET + {0x2999, 0x29D7, prN}, // Sm [63] DOTTED FENCE..BLACK HOURGLASS + {0x29D8, 0x29D8, prN}, // Ps LEFT WIGGLY FENCE + {0x29D9, 0x29D9, prN}, // Pe RIGHT WIGGLY FENCE + {0x29DA, 0x29DA, prN}, // Ps LEFT DOUBLE WIGGLY FENCE + {0x29DB, 0x29DB, prN}, // Pe RIGHT DOUBLE WIGGLY FENCE + {0x29DC, 0x29FB, prN}, // Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS + {0x29FC, 0x29FC, prN}, // Ps LEFT-POINTING CURVED ANGLE BRACKET + {0x29FD, 0x29FD, prN}, // Pe RIGHT-POINTING CURVED ANGLE BRACKET + {0x29FE, 0x29FF, prN}, // Sm [2] TINY..MINY + {0x2A00, 0x2AFF, prN}, // Sm [256] N-ARY CIRCLED DOT OPERATOR..N-ARY WHITE VERTICAL BAR + {0x2B00, 0x2B1A, prN}, // So [27] NORTH EAST WHITE ARROW..DOTTED SQUARE + {0x2B1B, 0x2B1C, prW}, // So [2] BLACK LARGE SQUARE..WHITE LARGE SQUARE + {0x2B1D, 0x2B2F, prN}, // So [19] BLACK VERY SMALL SQUARE..WHITE VERTICAL ELLIPSE + {0x2B30, 0x2B44, prN}, // Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET + {0x2B45, 0x2B46, prN}, // So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW + {0x2B47, 0x2B4C, prN}, // Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR + {0x2B4D, 0x2B4F, prN}, // So [3] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..SHORT BACKSLANTED SOUTH ARROW + {0x2B50, 0x2B50, prW}, // So WHITE MEDIUM STAR + {0x2B51, 0x2B54, prN}, // So [4] BLACK SMALL STAR..WHITE RIGHT-POINTING PENTAGON + {0x2B55, 0x2B55, prW}, // So HEAVY LARGE CIRCLE + {0x2B56, 0x2B59, prA}, // So [4] HEAVY OVAL WITH OVAL INSIDE..HEAVY CIRCLED SALTIRE + {0x2B5A, 0x2B73, prN}, // So [26] SLANTED NORTH ARROW WITH HOOKED HEAD..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR + {0x2B76, 0x2B95, prN}, // So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW + {0x2B97, 0x2BFF, prN}, // So [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL + {0x2C00, 0x2C5F, prN}, // L& [96] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC SMALL LETTER CAUDATE CHRIVI + {0x2C60, 0x2C7B, prN}, // L& [28] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN LETTER SMALL CAPITAL TURNED E + {0x2C7C, 0x2C7D, prN}, // Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V + {0x2C7E, 0x2C7F, prN}, // Lu [2] LATIN CAPITAL LETTER S WITH SWASH TAIL..LATIN CAPITAL LETTER Z WITH SWASH TAIL + {0x2C80, 0x2CE4, prN}, // L& [101] COPTIC CAPITAL LETTER ALFA..COPTIC SYMBOL KAI + {0x2CE5, 0x2CEA, prN}, // So [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA + {0x2CEB, 0x2CEE, prN}, // L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA + {0x2CEF, 0x2CF1, prN}, // Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS + {0x2CF2, 0x2CF3, prN}, // L& [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI + {0x2CF9, 0x2CFC, prN}, // Po [4] COPTIC OLD NUBIAN FULL STOP..COPTIC OLD NUBIAN VERSE DIVIDER + {0x2CFD, 0x2CFD, prN}, // No COPTIC FRACTION ONE HALF + {0x2CFE, 0x2CFF, prN}, // Po [2] COPTIC FULL STOP..COPTIC MORPHOLOGICAL DIVIDER + {0x2D00, 0x2D25, prN}, // Ll [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE + {0x2D27, 0x2D27, prN}, // Ll GEORGIAN SMALL LETTER YN + {0x2D2D, 0x2D2D, prN}, // Ll GEORGIAN SMALL LETTER AEN + {0x2D30, 0x2D67, prN}, // Lo [56] TIFINAGH LETTER YA..TIFINAGH LETTER YO + {0x2D6F, 0x2D6F, prN}, // Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK + {0x2D70, 0x2D70, prN}, // Po TIFINAGH SEPARATOR MARK + {0x2D7F, 0x2D7F, prN}, // Mn TIFINAGH CONSONANT JOINER + {0x2D80, 0x2D96, prN}, // Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE + {0x2DA0, 0x2DA6, prN}, // Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO + {0x2DA8, 0x2DAE, prN}, // Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO + {0x2DB0, 0x2DB6, prN}, // Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO + {0x2DB8, 0x2DBE, prN}, // Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO + {0x2DC0, 0x2DC6, prN}, // Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO + {0x2DC8, 0x2DCE, prN}, // Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO + {0x2DD0, 0x2DD6, prN}, // Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO + {0x2DD8, 0x2DDE, prN}, // Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO + {0x2DE0, 0x2DFF, prN}, // Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS + {0x2E00, 0x2E01, prN}, // Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER + {0x2E02, 0x2E02, prN}, // Pi LEFT SUBSTITUTION BRACKET + {0x2E03, 0x2E03, prN}, // Pf RIGHT SUBSTITUTION BRACKET + {0x2E04, 0x2E04, prN}, // Pi LEFT DOTTED SUBSTITUTION BRACKET + {0x2E05, 0x2E05, prN}, // Pf RIGHT DOTTED SUBSTITUTION BRACKET + {0x2E06, 0x2E08, prN}, // Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER + {0x2E09, 0x2E09, prN}, // Pi LEFT TRANSPOSITION BRACKET + {0x2E0A, 0x2E0A, prN}, // Pf RIGHT TRANSPOSITION BRACKET + {0x2E0B, 0x2E0B, prN}, // Po RAISED SQUARE + {0x2E0C, 0x2E0C, prN}, // Pi LEFT RAISED OMISSION BRACKET + {0x2E0D, 0x2E0D, prN}, // Pf RIGHT RAISED OMISSION BRACKET + {0x2E0E, 0x2E16, prN}, // Po [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE + {0x2E17, 0x2E17, prN}, // Pd DOUBLE OBLIQUE HYPHEN + {0x2E18, 0x2E19, prN}, // Po [2] INVERTED INTERROBANG..PALM BRANCH + {0x2E1A, 0x2E1A, prN}, // Pd HYPHEN WITH DIAERESIS + {0x2E1B, 0x2E1B, prN}, // Po TILDE WITH RING ABOVE + {0x2E1C, 0x2E1C, prN}, // Pi LEFT LOW PARAPHRASE BRACKET + {0x2E1D, 0x2E1D, prN}, // Pf RIGHT LOW PARAPHRASE BRACKET + {0x2E1E, 0x2E1F, prN}, // Po [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW + {0x2E20, 0x2E20, prN}, // Pi LEFT VERTICAL BAR WITH QUILL + {0x2E21, 0x2E21, prN}, // Pf RIGHT VERTICAL BAR WITH QUILL + {0x2E22, 0x2E22, prN}, // Ps TOP LEFT HALF BRACKET + {0x2E23, 0x2E23, prN}, // Pe TOP RIGHT HALF BRACKET + {0x2E24, 0x2E24, prN}, // Ps BOTTOM LEFT HALF BRACKET + {0x2E25, 0x2E25, prN}, // Pe BOTTOM RIGHT HALF BRACKET + {0x2E26, 0x2E26, prN}, // Ps LEFT SIDEWAYS U BRACKET + {0x2E27, 0x2E27, prN}, // Pe RIGHT SIDEWAYS U BRACKET + {0x2E28, 0x2E28, prN}, // Ps LEFT DOUBLE PARENTHESIS + {0x2E29, 0x2E29, prN}, // Pe RIGHT DOUBLE PARENTHESIS + {0x2E2A, 0x2E2E, prN}, // Po [5] TWO DOTS OVER ONE DOT PUNCTUATION..REVERSED QUESTION MARK + {0x2E2F, 0x2E2F, prN}, // Lm VERTICAL TILDE + {0x2E30, 0x2E39, prN}, // Po [10] RING POINT..TOP HALF SECTION SIGN + {0x2E3A, 0x2E3B, prN}, // Pd [2] TWO-EM DASH..THREE-EM DASH + {0x2E3C, 0x2E3F, prN}, // Po [4] STENOGRAPHIC FULL STOP..CAPITULUM + {0x2E40, 0x2E40, prN}, // Pd DOUBLE HYPHEN + {0x2E41, 0x2E41, prN}, // Po REVERSED COMMA + {0x2E42, 0x2E42, prN}, // Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK + {0x2E43, 0x2E4F, prN}, // Po [13] DASH WITH LEFT UPTURN..CORNISH VERSE DIVIDER + {0x2E50, 0x2E51, prN}, // So [2] CROSS PATTY WITH RIGHT CROSSBAR..CROSS PATTY WITH LEFT CROSSBAR + {0x2E52, 0x2E54, prN}, // Po [3] TIRONIAN SIGN CAPITAL ET..MEDIEVAL QUESTION MARK + {0x2E55, 0x2E55, prN}, // Ps LEFT SQUARE BRACKET WITH STROKE + {0x2E56, 0x2E56, prN}, // Pe RIGHT SQUARE BRACKET WITH STROKE + {0x2E57, 0x2E57, prN}, // Ps LEFT SQUARE BRACKET WITH DOUBLE STROKE + {0x2E58, 0x2E58, prN}, // Pe RIGHT SQUARE BRACKET WITH DOUBLE STROKE + {0x2E59, 0x2E59, prN}, // Ps TOP HALF LEFT PARENTHESIS + {0x2E5A, 0x2E5A, prN}, // Pe TOP HALF RIGHT PARENTHESIS + {0x2E5B, 0x2E5B, prN}, // Ps BOTTOM HALF LEFT PARENTHESIS + {0x2E5C, 0x2E5C, prN}, // Pe BOTTOM HALF RIGHT PARENTHESIS + {0x2E5D, 0x2E5D, prN}, // Pd OBLIQUE HYPHEN + {0x2E80, 0x2E99, prW}, // So [26] CJK RADICAL REPEAT..CJK RADICAL RAP + {0x2E9B, 0x2EF3, prW}, // So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE + {0x2F00, 0x2FD5, prW}, // So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE + {0x2FF0, 0x2FFB, prW}, // So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID + {0x3000, 0x3000, prF}, // Zs IDEOGRAPHIC SPACE + {0x3001, 0x3003, prW}, // Po [3] IDEOGRAPHIC COMMA..DITTO MARK + {0x3004, 0x3004, prW}, // So JAPANESE INDUSTRIAL STANDARD SYMBOL + {0x3005, 0x3005, prW}, // Lm IDEOGRAPHIC ITERATION MARK + {0x3006, 0x3006, prW}, // Lo IDEOGRAPHIC CLOSING MARK + {0x3007, 0x3007, prW}, // Nl IDEOGRAPHIC NUMBER ZERO + {0x3008, 0x3008, prW}, // Ps LEFT ANGLE BRACKET + {0x3009, 0x3009, prW}, // Pe RIGHT ANGLE BRACKET + {0x300A, 0x300A, prW}, // Ps LEFT DOUBLE ANGLE BRACKET + {0x300B, 0x300B, prW}, // Pe RIGHT DOUBLE ANGLE BRACKET + {0x300C, 0x300C, prW}, // Ps LEFT CORNER BRACKET + {0x300D, 0x300D, prW}, // Pe RIGHT CORNER BRACKET + {0x300E, 0x300E, prW}, // Ps LEFT WHITE CORNER BRACKET + {0x300F, 0x300F, prW}, // Pe RIGHT WHITE CORNER BRACKET + {0x3010, 0x3010, prW}, // Ps LEFT BLACK LENTICULAR BRACKET + {0x3011, 0x3011, prW}, // Pe RIGHT BLACK LENTICULAR BRACKET + {0x3012, 0x3013, prW}, // So [2] POSTAL MARK..GETA MARK + {0x3014, 0x3014, prW}, // Ps LEFT TORTOISE SHELL BRACKET + {0x3015, 0x3015, prW}, // Pe RIGHT TORTOISE SHELL BRACKET + {0x3016, 0x3016, prW}, // Ps LEFT WHITE LENTICULAR BRACKET + {0x3017, 0x3017, prW}, // Pe RIGHT WHITE LENTICULAR BRACKET + {0x3018, 0x3018, prW}, // Ps LEFT WHITE TORTOISE SHELL BRACKET + {0x3019, 0x3019, prW}, // Pe RIGHT WHITE TORTOISE SHELL BRACKET + {0x301A, 0x301A, prW}, // Ps LEFT WHITE SQUARE BRACKET + {0x301B, 0x301B, prW}, // Pe RIGHT WHITE SQUARE BRACKET + {0x301C, 0x301C, prW}, // Pd WAVE DASH + {0x301D, 0x301D, prW}, // Ps REVERSED DOUBLE PRIME QUOTATION MARK + {0x301E, 0x301F, prW}, // Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK + {0x3020, 0x3020, prW}, // So POSTAL MARK FACE + {0x3021, 0x3029, prW}, // Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE + {0x302A, 0x302D, prW}, // Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK + {0x302E, 0x302F, prW}, // Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK + {0x3030, 0x3030, prW}, // Pd WAVY DASH + {0x3031, 0x3035, prW}, // Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF + {0x3036, 0x3037, prW}, // So [2] CIRCLED POSTAL MARK..IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL + {0x3038, 0x303A, prW}, // Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY + {0x303B, 0x303B, prW}, // Lm VERTICAL IDEOGRAPHIC ITERATION MARK + {0x303C, 0x303C, prW}, // Lo MASU MARK + {0x303D, 0x303D, prW}, // Po PART ALTERNATION MARK + {0x303E, 0x303E, prW}, // So IDEOGRAPHIC VARIATION INDICATOR + {0x303F, 0x303F, prN}, // So IDEOGRAPHIC HALF FILL SPACE + {0x3041, 0x3096, prW}, // Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE + {0x3099, 0x309A, prW}, // Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x309B, 0x309C, prW}, // Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x309D, 0x309E, prW}, // Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK + {0x309F, 0x309F, prW}, // Lo HIRAGANA DIGRAPH YORI + {0x30A0, 0x30A0, prW}, // Pd KATAKANA-HIRAGANA DOUBLE HYPHEN + {0x30A1, 0x30FA, prW}, // Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO + {0x30FB, 0x30FB, prW}, // Po KATAKANA MIDDLE DOT + {0x30FC, 0x30FE, prW}, // Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK + {0x30FF, 0x30FF, prW}, // Lo KATAKANA DIGRAPH KOTO + {0x3105, 0x312F, prW}, // Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN + {0x3131, 0x318E, prW}, // Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE + {0x3190, 0x3191, prW}, // So [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK + {0x3192, 0x3195, prW}, // No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK + {0x3196, 0x319F, prW}, // So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK + {0x31A0, 0x31BF, prW}, // Lo [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH + {0x31C0, 0x31E3, prW}, // So [36] CJK STROKE T..CJK STROKE Q + {0x31F0, 0x31FF, prW}, // Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO + {0x3200, 0x321E, prW}, // So [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU + {0x3220, 0x3229, prW}, // No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN + {0x322A, 0x3247, prW}, // So [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO + {0x3248, 0x324F, prA}, // No [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE + {0x3250, 0x3250, prW}, // So PARTNERSHIP SIGN + {0x3251, 0x325F, prW}, // No [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE + {0x3260, 0x327F, prW}, // So [32] CIRCLED HANGUL KIYEOK..KOREAN STANDARD SYMBOL + {0x3280, 0x3289, prW}, // No [10] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN + {0x328A, 0x32B0, prW}, // So [39] CIRCLED IDEOGRAPH MOON..CIRCLED IDEOGRAPH NIGHT + {0x32B1, 0x32BF, prW}, // No [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY + {0x32C0, 0x32FF, prW}, // So [64] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..SQUARE ERA NAME REIWA + {0x3300, 0x33FF, prW}, // So [256] SQUARE APAATO..SQUARE GAL + {0x3400, 0x4DBF, prW}, // Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF + {0x4DC0, 0x4DFF, prN}, // So [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION + {0x4E00, 0x9FFF, prW}, // Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF + {0xA000, 0xA014, prW}, // Lo [21] YI SYLLABLE IT..YI SYLLABLE E + {0xA015, 0xA015, prW}, // Lm YI SYLLABLE WU + {0xA016, 0xA48C, prW}, // Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR + {0xA490, 0xA4C6, prW}, // So [55] YI RADICAL QOT..YI RADICAL KE + {0xA4D0, 0xA4F7, prN}, // Lo [40] LISU LETTER BA..LISU LETTER OE + {0xA4F8, 0xA4FD, prN}, // Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU + {0xA4FE, 0xA4FF, prN}, // Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP + {0xA500, 0xA60B, prN}, // Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG + {0xA60C, 0xA60C, prN}, // Lm VAI SYLLABLE LENGTHENER + {0xA60D, 0xA60F, prN}, // Po [3] VAI COMMA..VAI QUESTION MARK + {0xA610, 0xA61F, prN}, // Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG + {0xA620, 0xA629, prN}, // Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE + {0xA62A, 0xA62B, prN}, // Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO + {0xA640, 0xA66D, prN}, // L& [46] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O + {0xA66E, 0xA66E, prN}, // Lo CYRILLIC LETTER MULTIOCULAR O + {0xA66F, 0xA66F, prN}, // Mn COMBINING CYRILLIC VZMET + {0xA670, 0xA672, prN}, // Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN + {0xA673, 0xA673, prN}, // Po SLAVONIC ASTERISK + {0xA674, 0xA67D, prN}, // Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK + {0xA67E, 0xA67E, prN}, // Po CYRILLIC KAVYKA + {0xA67F, 0xA67F, prN}, // Lm CYRILLIC PAYEROK + {0xA680, 0xA69B, prN}, // L& [28] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER CROSSED O + {0xA69C, 0xA69D, prN}, // Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN + {0xA69E, 0xA69F, prN}, // Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E + {0xA6A0, 0xA6E5, prN}, // Lo [70] BAMUM LETTER A..BAMUM LETTER KI + {0xA6E6, 0xA6EF, prN}, // Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM + {0xA6F0, 0xA6F1, prN}, // Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS + {0xA6F2, 0xA6F7, prN}, // Po [6] BAMUM NJAEMLI..BAMUM QUESTION MARK + {0xA700, 0xA716, prN}, // Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR + {0xA717, 0xA71F, prN}, // Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK + {0xA720, 0xA721, prN}, // Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE + {0xA722, 0xA76F, prN}, // L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON + {0xA770, 0xA770, prN}, // Lm MODIFIER LETTER US + {0xA771, 0xA787, prN}, // L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T + {0xA788, 0xA788, prN}, // Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT + {0xA789, 0xA78A, prN}, // Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN + {0xA78B, 0xA78E, prN}, // L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT + {0xA78F, 0xA78F, prN}, // Lo LATIN LETTER SINOLOGICAL DOT + {0xA790, 0xA7CA, prN}, // L& [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY + {0xA7D0, 0xA7D1, prN}, // L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G + {0xA7D3, 0xA7D3, prN}, // Ll LATIN SMALL LETTER DOUBLE THORN + {0xA7D5, 0xA7D9, prN}, // L& [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S + {0xA7F2, 0xA7F4, prN}, // Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q + {0xA7F5, 0xA7F6, prN}, // L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H + {0xA7F7, 0xA7F7, prN}, // Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I + {0xA7F8, 0xA7F9, prN}, // Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE + {0xA7FA, 0xA7FA, prN}, // Ll LATIN LETTER SMALL CAPITAL TURNED M + {0xA7FB, 0xA7FF, prN}, // Lo [5] LATIN EPIGRAPHIC LETTER REVERSED F..LATIN EPIGRAPHIC LETTER ARCHAIC M + {0xA800, 0xA801, prN}, // Lo [2] SYLOTI NAGRI LETTER A..SYLOTI NAGRI LETTER I + {0xA802, 0xA802, prN}, // Mn SYLOTI NAGRI SIGN DVISVARA + {0xA803, 0xA805, prN}, // Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O + {0xA806, 0xA806, prN}, // Mn SYLOTI NAGRI SIGN HASANTA + {0xA807, 0xA80A, prN}, // Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO + {0xA80B, 0xA80B, prN}, // Mn SYLOTI NAGRI SIGN ANUSVARA + {0xA80C, 0xA822, prN}, // Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO + {0xA823, 0xA824, prN}, // Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I + {0xA825, 0xA826, prN}, // Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E + {0xA827, 0xA827, prN}, // Mc SYLOTI NAGRI VOWEL SIGN OO + {0xA828, 0xA82B, prN}, // So [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 + {0xA82C, 0xA82C, prN}, // Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA + {0xA830, 0xA835, prN}, // No [6] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS + {0xA836, 0xA837, prN}, // So [2] NORTH INDIC QUARTER MARK..NORTH INDIC PLACEHOLDER MARK + {0xA838, 0xA838, prN}, // Sc NORTH INDIC RUPEE MARK + {0xA839, 0xA839, prN}, // So NORTH INDIC QUANTITY MARK + {0xA840, 0xA873, prN}, // Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU + {0xA874, 0xA877, prN}, // Po [4] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA MARK DOUBLE SHAD + {0xA880, 0xA881, prN}, // Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA + {0xA882, 0xA8B3, prN}, // Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA + {0xA8B4, 0xA8C3, prN}, // Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU + {0xA8C4, 0xA8C5, prN}, // Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU + {0xA8CE, 0xA8CF, prN}, // Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA + {0xA8D0, 0xA8D9, prN}, // Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE + {0xA8E0, 0xA8F1, prN}, // Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA + {0xA8F2, 0xA8F7, prN}, // Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA + {0xA8F8, 0xA8FA, prN}, // Po [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET + {0xA8FB, 0xA8FB, prN}, // Lo DEVANAGARI HEADSTROKE + {0xA8FC, 0xA8FC, prN}, // Po DEVANAGARI SIGN SIDDHAM + {0xA8FD, 0xA8FE, prN}, // Lo [2] DEVANAGARI JAIN OM..DEVANAGARI LETTER AY + {0xA8FF, 0xA8FF, prN}, // Mn DEVANAGARI VOWEL SIGN AY + {0xA900, 0xA909, prN}, // Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE + {0xA90A, 0xA925, prN}, // Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO + {0xA926, 0xA92D, prN}, // Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU + {0xA92E, 0xA92F, prN}, // Po [2] KAYAH LI SIGN CWI..KAYAH LI SIGN SHYA + {0xA930, 0xA946, prN}, // Lo [23] REJANG LETTER KA..REJANG LETTER A + {0xA947, 0xA951, prN}, // Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R + {0xA952, 0xA953, prN}, // Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA + {0xA95F, 0xA95F, prN}, // Po REJANG SECTION MARK + {0xA960, 0xA97C, prW}, // Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH + {0xA980, 0xA982, prN}, // Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR + {0xA983, 0xA983, prN}, // Mc JAVANESE SIGN WIGNYAN + {0xA984, 0xA9B2, prN}, // Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA + {0xA9B3, 0xA9B3, prN}, // Mn JAVANESE SIGN CECAK TELU + {0xA9B4, 0xA9B5, prN}, // Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG + {0xA9B6, 0xA9B9, prN}, // Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT + {0xA9BA, 0xA9BB, prN}, // Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE + {0xA9BC, 0xA9BD, prN}, // Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET + {0xA9BE, 0xA9C0, prN}, // Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON + {0xA9C1, 0xA9CD, prN}, // Po [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH + {0xA9CF, 0xA9CF, prN}, // Lm JAVANESE PANGRANGKEP + {0xA9D0, 0xA9D9, prN}, // Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE + {0xA9DE, 0xA9DF, prN}, // Po [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN + {0xA9E0, 0xA9E4, prN}, // Lo [5] MYANMAR LETTER SHAN GHA..MYANMAR LETTER SHAN BHA + {0xA9E5, 0xA9E5, prN}, // Mn MYANMAR SIGN SHAN SAW + {0xA9E6, 0xA9E6, prN}, // Lm MYANMAR MODIFIER LETTER SHAN REDUPLICATION + {0xA9E7, 0xA9EF, prN}, // Lo [9] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING NNA + {0xA9F0, 0xA9F9, prN}, // Nd [10] MYANMAR TAI LAING DIGIT ZERO..MYANMAR TAI LAING DIGIT NINE + {0xA9FA, 0xA9FE, prN}, // Lo [5] MYANMAR LETTER TAI LAING LLA..MYANMAR LETTER TAI LAING BHA + {0xAA00, 0xAA28, prN}, // Lo [41] CHAM LETTER A..CHAM LETTER HA + {0xAA29, 0xAA2E, prN}, // Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE + {0xAA2F, 0xAA30, prN}, // Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI + {0xAA31, 0xAA32, prN}, // Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE + {0xAA33, 0xAA34, prN}, // Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA + {0xAA35, 0xAA36, prN}, // Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA + {0xAA40, 0xAA42, prN}, // Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG + {0xAA43, 0xAA43, prN}, // Mn CHAM CONSONANT SIGN FINAL NG + {0xAA44, 0xAA4B, prN}, // Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS + {0xAA4C, 0xAA4C, prN}, // Mn CHAM CONSONANT SIGN FINAL M + {0xAA4D, 0xAA4D, prN}, // Mc CHAM CONSONANT SIGN FINAL H + {0xAA50, 0xAA59, prN}, // Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE + {0xAA5C, 0xAA5F, prN}, // Po [4] CHAM PUNCTUATION SPIRAL..CHAM PUNCTUATION TRIPLE DANDA + {0xAA60, 0xAA6F, prN}, // Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA + {0xAA70, 0xAA70, prN}, // Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION + {0xAA71, 0xAA76, prN}, // Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM + {0xAA77, 0xAA79, prN}, // So [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO + {0xAA7A, 0xAA7A, prN}, // Lo MYANMAR LETTER AITON RA + {0xAA7B, 0xAA7B, prN}, // Mc MYANMAR SIGN PAO KAREN TONE + {0xAA7C, 0xAA7C, prN}, // Mn MYANMAR SIGN TAI LAING TONE-2 + {0xAA7D, 0xAA7D, prN}, // Mc MYANMAR SIGN TAI LAING TONE-5 + {0xAA7E, 0xAA7F, prN}, // Lo [2] MYANMAR LETTER SHWE PALAUNG CHA..MYANMAR LETTER SHWE PALAUNG SHA + {0xAA80, 0xAAAF, prN}, // Lo [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O + {0xAAB0, 0xAAB0, prN}, // Mn TAI VIET MAI KANG + {0xAAB1, 0xAAB1, prN}, // Lo TAI VIET VOWEL AA + {0xAAB2, 0xAAB4, prN}, // Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U + {0xAAB5, 0xAAB6, prN}, // Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O + {0xAAB7, 0xAAB8, prN}, // Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA + {0xAAB9, 0xAABD, prN}, // Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN + {0xAABE, 0xAABF, prN}, // Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK + {0xAAC0, 0xAAC0, prN}, // Lo TAI VIET TONE MAI NUENG + {0xAAC1, 0xAAC1, prN}, // Mn TAI VIET TONE MAI THO + {0xAAC2, 0xAAC2, prN}, // Lo TAI VIET TONE MAI SONG + {0xAADB, 0xAADC, prN}, // Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG + {0xAADD, 0xAADD, prN}, // Lm TAI VIET SYMBOL SAM + {0xAADE, 0xAADF, prN}, // Po [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI + {0xAAE0, 0xAAEA, prN}, // Lo [11] MEETEI MAYEK LETTER E..MEETEI MAYEK LETTER SSA + {0xAAEB, 0xAAEB, prN}, // Mc MEETEI MAYEK VOWEL SIGN II + {0xAAEC, 0xAAED, prN}, // Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI + {0xAAEE, 0xAAEF, prN}, // Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU + {0xAAF0, 0xAAF1, prN}, // Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM + {0xAAF2, 0xAAF2, prN}, // Lo MEETEI MAYEK ANJI + {0xAAF3, 0xAAF4, prN}, // Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK + {0xAAF5, 0xAAF5, prN}, // Mc MEETEI MAYEK VOWEL SIGN VISARGA + {0xAAF6, 0xAAF6, prN}, // Mn MEETEI MAYEK VIRAMA + {0xAB01, 0xAB06, prN}, // Lo [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO + {0xAB09, 0xAB0E, prN}, // Lo [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO + {0xAB11, 0xAB16, prN}, // Lo [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO + {0xAB20, 0xAB26, prN}, // Lo [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO + {0xAB28, 0xAB2E, prN}, // Lo [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO + {0xAB30, 0xAB5A, prN}, // Ll [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG + {0xAB5B, 0xAB5B, prN}, // Sk MODIFIER BREVE WITH INVERTED BREVE + {0xAB5C, 0xAB5F, prN}, // Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK + {0xAB60, 0xAB68, prN}, // Ll [9] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE + {0xAB69, 0xAB69, prN}, // Lm MODIFIER LETTER SMALL TURNED W + {0xAB6A, 0xAB6B, prN}, // Sk [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK + {0xAB70, 0xABBF, prN}, // Ll [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA + {0xABC0, 0xABE2, prN}, // Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM + {0xABE3, 0xABE4, prN}, // Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP + {0xABE5, 0xABE5, prN}, // Mn MEETEI MAYEK VOWEL SIGN ANAP + {0xABE6, 0xABE7, prN}, // Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP + {0xABE8, 0xABE8, prN}, // Mn MEETEI MAYEK VOWEL SIGN UNAP + {0xABE9, 0xABEA, prN}, // Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG + {0xABEB, 0xABEB, prN}, // Po MEETEI MAYEK CHEIKHEI + {0xABEC, 0xABEC, prN}, // Mc MEETEI MAYEK LUM IYEK + {0xABED, 0xABED, prN}, // Mn MEETEI MAYEK APUN IYEK + {0xABF0, 0xABF9, prN}, // Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE + {0xAC00, 0xD7A3, prW}, // Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH + {0xD7B0, 0xD7C6, prN}, // Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E + {0xD7CB, 0xD7FB, prN}, // Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH + {0xD800, 0xDB7F, prN}, // Cs [896] .. + {0xDB80, 0xDBFF, prN}, // Cs [128] .. + {0xDC00, 0xDFFF, prN}, // Cs [1024] .. + {0xE000, 0xF8FF, prA}, // Co [6400] .. + {0xF900, 0xFA6D, prW}, // Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D + {0xFA6E, 0xFA6F, prW}, // Cn [2] .. + {0xFA70, 0xFAD9, prW}, // Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 + {0xFADA, 0xFAFF, prW}, // Cn [38] .. + {0xFB00, 0xFB06, prN}, // Ll [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST + {0xFB13, 0xFB17, prN}, // Ll [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH + {0xFB1D, 0xFB1D, prN}, // Lo HEBREW LETTER YOD WITH HIRIQ + {0xFB1E, 0xFB1E, prN}, // Mn HEBREW POINT JUDEO-SPANISH VARIKA + {0xFB1F, 0xFB28, prN}, // Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV + {0xFB29, 0xFB29, prN}, // Sm HEBREW LETTER ALTERNATIVE PLUS SIGN + {0xFB2A, 0xFB36, prN}, // Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH + {0xFB38, 0xFB3C, prN}, // Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH + {0xFB3E, 0xFB3E, prN}, // Lo HEBREW LETTER MEM WITH DAGESH + {0xFB40, 0xFB41, prN}, // Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH + {0xFB43, 0xFB44, prN}, // Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH + {0xFB46, 0xFB4F, prN}, // Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATURE ALEF LAMED + {0xFB50, 0xFBB1, prN}, // Lo [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM + {0xFBB2, 0xFBC2, prN}, // Sk [17] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL WASLA ABOVE + {0xFBD3, 0xFD3D, prN}, // Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM + {0xFD3E, 0xFD3E, prN}, // Pe ORNATE LEFT PARENTHESIS + {0xFD3F, 0xFD3F, prN}, // Ps ORNATE RIGHT PARENTHESIS + {0xFD40, 0xFD4F, prN}, // So [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH + {0xFD50, 0xFD8F, prN}, // Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM + {0xFD92, 0xFDC7, prN}, // Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM + {0xFDCF, 0xFDCF, prN}, // So ARABIC LIGATURE SALAAMUHU ALAYNAA + {0xFDF0, 0xFDFB, prN}, // Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU + {0xFDFC, 0xFDFC, prN}, // Sc RIAL SIGN + {0xFDFD, 0xFDFF, prN}, // So [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL + {0xFE00, 0xFE0F, prA}, // Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 + {0xFE10, 0xFE16, prW}, // Po [7] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL QUESTION MARK + {0xFE17, 0xFE17, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET + {0xFE18, 0xFE18, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET + {0xFE19, 0xFE19, prW}, // Po PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS + {0xFE20, 0xFE2F, prN}, // Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + {0xFE30, 0xFE30, prW}, // Po PRESENTATION FORM FOR VERTICAL TWO DOT LEADER + {0xFE31, 0xFE32, prW}, // Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH + {0xFE33, 0xFE34, prW}, // Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE + {0xFE35, 0xFE35, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS + {0xFE36, 0xFE36, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS + {0xFE37, 0xFE37, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET + {0xFE38, 0xFE38, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET + {0xFE39, 0xFE39, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET + {0xFE3A, 0xFE3A, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET + {0xFE3B, 0xFE3B, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET + {0xFE3C, 0xFE3C, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET + {0xFE3D, 0xFE3D, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET + {0xFE3E, 0xFE3E, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET + {0xFE3F, 0xFE3F, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET + {0xFE40, 0xFE40, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET + {0xFE41, 0xFE41, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET + {0xFE42, 0xFE42, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET + {0xFE43, 0xFE43, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET + {0xFE44, 0xFE44, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET + {0xFE45, 0xFE46, prW}, // Po [2] SESAME DOT..WHITE SESAME DOT + {0xFE47, 0xFE47, prW}, // Ps PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET + {0xFE48, 0xFE48, prW}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET + {0xFE49, 0xFE4C, prW}, // Po [4] DASHED OVERLINE..DOUBLE WAVY OVERLINE + {0xFE4D, 0xFE4F, prW}, // Pc [3] DASHED LOW LINE..WAVY LOW LINE + {0xFE50, 0xFE52, prW}, // Po [3] SMALL COMMA..SMALL FULL STOP + {0xFE54, 0xFE57, prW}, // Po [4] SMALL SEMICOLON..SMALL EXCLAMATION MARK + {0xFE58, 0xFE58, prW}, // Pd SMALL EM DASH + {0xFE59, 0xFE59, prW}, // Ps SMALL LEFT PARENTHESIS + {0xFE5A, 0xFE5A, prW}, // Pe SMALL RIGHT PARENTHESIS + {0xFE5B, 0xFE5B, prW}, // Ps SMALL LEFT CURLY BRACKET + {0xFE5C, 0xFE5C, prW}, // Pe SMALL RIGHT CURLY BRACKET + {0xFE5D, 0xFE5D, prW}, // Ps SMALL LEFT TORTOISE SHELL BRACKET + {0xFE5E, 0xFE5E, prW}, // Pe SMALL RIGHT TORTOISE SHELL BRACKET + {0xFE5F, 0xFE61, prW}, // Po [3] SMALL NUMBER SIGN..SMALL ASTERISK + {0xFE62, 0xFE62, prW}, // Sm SMALL PLUS SIGN + {0xFE63, 0xFE63, prW}, // Pd SMALL HYPHEN-MINUS + {0xFE64, 0xFE66, prW}, // Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN + {0xFE68, 0xFE68, prW}, // Po SMALL REVERSE SOLIDUS + {0xFE69, 0xFE69, prW}, // Sc SMALL DOLLAR SIGN + {0xFE6A, 0xFE6B, prW}, // Po [2] SMALL PERCENT SIGN..SMALL COMMERCIAL AT + {0xFE70, 0xFE74, prN}, // Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM + {0xFE76, 0xFEFC, prN}, // Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM + {0xFEFF, 0xFEFF, prN}, // Cf ZERO WIDTH NO-BREAK SPACE + {0xFF01, 0xFF03, prF}, // Po [3] FULLWIDTH EXCLAMATION MARK..FULLWIDTH NUMBER SIGN + {0xFF04, 0xFF04, prF}, // Sc FULLWIDTH DOLLAR SIGN + {0xFF05, 0xFF07, prF}, // Po [3] FULLWIDTH PERCENT SIGN..FULLWIDTH APOSTROPHE + {0xFF08, 0xFF08, prF}, // Ps FULLWIDTH LEFT PARENTHESIS + {0xFF09, 0xFF09, prF}, // Pe FULLWIDTH RIGHT PARENTHESIS + {0xFF0A, 0xFF0A, prF}, // Po FULLWIDTH ASTERISK + {0xFF0B, 0xFF0B, prF}, // Sm FULLWIDTH PLUS SIGN + {0xFF0C, 0xFF0C, prF}, // Po FULLWIDTH COMMA + {0xFF0D, 0xFF0D, prF}, // Pd FULLWIDTH HYPHEN-MINUS + {0xFF0E, 0xFF0F, prF}, // Po [2] FULLWIDTH FULL STOP..FULLWIDTH SOLIDUS + {0xFF10, 0xFF19, prF}, // Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE + {0xFF1A, 0xFF1B, prF}, // Po [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON + {0xFF1C, 0xFF1E, prF}, // Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN + {0xFF1F, 0xFF20, prF}, // Po [2] FULLWIDTH QUESTION MARK..FULLWIDTH COMMERCIAL AT + {0xFF21, 0xFF3A, prF}, // Lu [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z + {0xFF3B, 0xFF3B, prF}, // Ps FULLWIDTH LEFT SQUARE BRACKET + {0xFF3C, 0xFF3C, prF}, // Po FULLWIDTH REVERSE SOLIDUS + {0xFF3D, 0xFF3D, prF}, // Pe FULLWIDTH RIGHT SQUARE BRACKET + {0xFF3E, 0xFF3E, prF}, // Sk FULLWIDTH CIRCUMFLEX ACCENT + {0xFF3F, 0xFF3F, prF}, // Pc FULLWIDTH LOW LINE + {0xFF40, 0xFF40, prF}, // Sk FULLWIDTH GRAVE ACCENT + {0xFF41, 0xFF5A, prF}, // Ll [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z + {0xFF5B, 0xFF5B, prF}, // Ps FULLWIDTH LEFT CURLY BRACKET + {0xFF5C, 0xFF5C, prF}, // Sm FULLWIDTH VERTICAL LINE + {0xFF5D, 0xFF5D, prF}, // Pe FULLWIDTH RIGHT CURLY BRACKET + {0xFF5E, 0xFF5E, prF}, // Sm FULLWIDTH TILDE + {0xFF5F, 0xFF5F, prF}, // Ps FULLWIDTH LEFT WHITE PARENTHESIS + {0xFF60, 0xFF60, prF}, // Pe FULLWIDTH RIGHT WHITE PARENTHESIS + {0xFF61, 0xFF61, prH}, // Po HALFWIDTH IDEOGRAPHIC FULL STOP + {0xFF62, 0xFF62, prH}, // Ps HALFWIDTH LEFT CORNER BRACKET + {0xFF63, 0xFF63, prH}, // Pe HALFWIDTH RIGHT CORNER BRACKET + {0xFF64, 0xFF65, prH}, // Po [2] HALFWIDTH IDEOGRAPHIC COMMA..HALFWIDTH KATAKANA MIDDLE DOT + {0xFF66, 0xFF6F, prH}, // Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU + {0xFF70, 0xFF70, prH}, // Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK + {0xFF71, 0xFF9D, prH}, // Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N + {0xFF9E, 0xFF9F, prH}, // Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK + {0xFFA0, 0xFFBE, prH}, // Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH + {0xFFC2, 0xFFC7, prH}, // Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E + {0xFFCA, 0xFFCF, prH}, // Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE + {0xFFD2, 0xFFD7, prH}, // Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU + {0xFFDA, 0xFFDC, prH}, // Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I + {0xFFE0, 0xFFE1, prF}, // Sc [2] FULLWIDTH CENT SIGN..FULLWIDTH POUND SIGN + {0xFFE2, 0xFFE2, prF}, // Sm FULLWIDTH NOT SIGN + {0xFFE3, 0xFFE3, prF}, // Sk FULLWIDTH MACRON + {0xFFE4, 0xFFE4, prF}, // So FULLWIDTH BROKEN BAR + {0xFFE5, 0xFFE6, prF}, // Sc [2] FULLWIDTH YEN SIGN..FULLWIDTH WON SIGN + {0xFFE8, 0xFFE8, prH}, // So HALFWIDTH FORMS LIGHT VERTICAL + {0xFFE9, 0xFFEC, prH}, // Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW + {0xFFED, 0xFFEE, prH}, // So [2] HALFWIDTH BLACK SQUARE..HALFWIDTH WHITE CIRCLE + {0xFFF9, 0xFFFB, prN}, // Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR + {0xFFFC, 0xFFFC, prN}, // So OBJECT REPLACEMENT CHARACTER + {0xFFFD, 0xFFFD, prA}, // So REPLACEMENT CHARACTER + {0x10000, 0x1000B, prN}, // Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE + {0x1000D, 0x10026, prN}, // Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO + {0x10028, 0x1003A, prN}, // Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO + {0x1003C, 0x1003D, prN}, // Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE + {0x1003F, 0x1004D, prN}, // Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO + {0x10050, 0x1005D, prN}, // Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 + {0x10080, 0x100FA, prN}, // Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 + {0x10100, 0x10102, prN}, // Po [3] AEGEAN WORD SEPARATOR LINE..AEGEAN CHECK MARK + {0x10107, 0x10133, prN}, // No [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND + {0x10137, 0x1013F, prN}, // So [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT + {0x10140, 0x10174, prN}, // Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS + {0x10175, 0x10178, prN}, // No [4] GREEK ONE HALF SIGN..GREEK THREE QUARTERS SIGN + {0x10179, 0x10189, prN}, // So [17] GREEK YEAR SIGN..GREEK TRYBLION BASE SIGN + {0x1018A, 0x1018B, prN}, // No [2] GREEK ZERO SIGN..GREEK ONE QUARTER SIGN + {0x1018C, 0x1018E, prN}, // So [3] GREEK SINUSOID SIGN..NOMISMA SIGN + {0x10190, 0x1019C, prN}, // So [13] ROMAN SEXTANS SIGN..ASCIA SYMBOL + {0x101A0, 0x101A0, prN}, // So GREEK SYMBOL TAU RHO + {0x101D0, 0x101FC, prN}, // So [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND + {0x101FD, 0x101FD, prN}, // Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE + {0x10280, 0x1029C, prN}, // Lo [29] LYCIAN LETTER A..LYCIAN LETTER X + {0x102A0, 0x102D0, prN}, // Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 + {0x102E0, 0x102E0, prN}, // Mn COPTIC EPACT THOUSANDS MARK + {0x102E1, 0x102FB, prN}, // No [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED + {0x10300, 0x1031F, prN}, // Lo [32] OLD ITALIC LETTER A..OLD ITALIC LETTER ESS + {0x10320, 0x10323, prN}, // No [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY + {0x1032D, 0x1032F, prN}, // Lo [3] OLD ITALIC LETTER YE..OLD ITALIC LETTER SOUTHERN TSE + {0x10330, 0x10340, prN}, // Lo [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA + {0x10341, 0x10341, prN}, // Nl GOTHIC LETTER NINETY + {0x10342, 0x10349, prN}, // Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL + {0x1034A, 0x1034A, prN}, // Nl GOTHIC LETTER NINE HUNDRED + {0x10350, 0x10375, prN}, // Lo [38] OLD PERMIC LETTER AN..OLD PERMIC LETTER IA + {0x10376, 0x1037A, prN}, // Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII + {0x10380, 0x1039D, prN}, // Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU + {0x1039F, 0x1039F, prN}, // Po UGARITIC WORD DIVIDER + {0x103A0, 0x103C3, prN}, // Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA + {0x103C8, 0x103CF, prN}, // Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH + {0x103D0, 0x103D0, prN}, // Po OLD PERSIAN WORD DIVIDER + {0x103D1, 0x103D5, prN}, // Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED + {0x10400, 0x1044F, prN}, // L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW + {0x10450, 0x1047F, prN}, // Lo [48] SHAVIAN LETTER PEEP..SHAVIAN LETTER YEW + {0x10480, 0x1049D, prN}, // Lo [30] OSMANYA LETTER ALEF..OSMANYA LETTER OO + {0x104A0, 0x104A9, prN}, // Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE + {0x104B0, 0x104D3, prN}, // Lu [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA + {0x104D8, 0x104FB, prN}, // Ll [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA + {0x10500, 0x10527, prN}, // Lo [40] ELBASAN LETTER A..ELBASAN LETTER KHE + {0x10530, 0x10563, prN}, // Lo [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW + {0x1056F, 0x1056F, prN}, // Po CAUCASIAN ALBANIAN CITATION MARK + {0x10570, 0x1057A, prN}, // Lu [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA + {0x1057C, 0x1058A, prN}, // Lu [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE + {0x1058C, 0x10592, prN}, // Lu [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE + {0x10594, 0x10595, prN}, // Lu [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE + {0x10597, 0x105A1, prN}, // Ll [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA + {0x105A3, 0x105B1, prN}, // Ll [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE + {0x105B3, 0x105B9, prN}, // Ll [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE + {0x105BB, 0x105BC, prN}, // Ll [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE + {0x10600, 0x10736, prN}, // Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 + {0x10740, 0x10755, prN}, // Lo [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE + {0x10760, 0x10767, prN}, // Lo [8] LINEAR A SIGN A800..LINEAR A SIGN A807 + {0x10780, 0x10785, prN}, // Lm [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK + {0x10787, 0x107B0, prN}, // Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK + {0x107B2, 0x107BA, prN}, // Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL + {0x10800, 0x10805, prN}, // Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA + {0x10808, 0x10808, prN}, // Lo CYPRIOT SYLLABLE JO + {0x1080A, 0x10835, prN}, // Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO + {0x10837, 0x10838, prN}, // Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE + {0x1083C, 0x1083C, prN}, // Lo CYPRIOT SYLLABLE ZA + {0x1083F, 0x1083F, prN}, // Lo CYPRIOT SYLLABLE ZO + {0x10840, 0x10855, prN}, // Lo [22] IMPERIAL ARAMAIC LETTER ALEPH..IMPERIAL ARAMAIC LETTER TAW + {0x10857, 0x10857, prN}, // Po IMPERIAL ARAMAIC SECTION SIGN + {0x10858, 0x1085F, prN}, // No [8] IMPERIAL ARAMAIC NUMBER ONE..IMPERIAL ARAMAIC NUMBER TEN THOUSAND + {0x10860, 0x10876, prN}, // Lo [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW + {0x10877, 0x10878, prN}, // So [2] PALMYRENE LEFT-POINTING FLEURON..PALMYRENE RIGHT-POINTING FLEURON + {0x10879, 0x1087F, prN}, // No [7] PALMYRENE NUMBER ONE..PALMYRENE NUMBER TWENTY + {0x10880, 0x1089E, prN}, // Lo [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW + {0x108A7, 0x108AF, prN}, // No [9] NABATAEAN NUMBER ONE..NABATAEAN NUMBER ONE HUNDRED + {0x108E0, 0x108F2, prN}, // Lo [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH + {0x108F4, 0x108F5, prN}, // Lo [2] HATRAN LETTER SHIN..HATRAN LETTER TAW + {0x108FB, 0x108FF, prN}, // No [5] HATRAN NUMBER ONE..HATRAN NUMBER ONE HUNDRED + {0x10900, 0x10915, prN}, // Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU + {0x10916, 0x1091B, prN}, // No [6] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER THREE + {0x1091F, 0x1091F, prN}, // Po PHOENICIAN WORD SEPARATOR + {0x10920, 0x10939, prN}, // Lo [26] LYDIAN LETTER A..LYDIAN LETTER C + {0x1093F, 0x1093F, prN}, // Po LYDIAN TRIANGULAR MARK + {0x10980, 0x1099F, prN}, // Lo [32] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC HIEROGLYPHIC SYMBOL VIDJ-2 + {0x109A0, 0x109B7, prN}, // Lo [24] MEROITIC CURSIVE LETTER A..MEROITIC CURSIVE LETTER DA + {0x109BC, 0x109BD, prN}, // No [2] MEROITIC CURSIVE FRACTION ELEVEN TWELFTHS..MEROITIC CURSIVE FRACTION ONE HALF + {0x109BE, 0x109BF, prN}, // Lo [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN + {0x109C0, 0x109CF, prN}, // No [16] MEROITIC CURSIVE NUMBER ONE..MEROITIC CURSIVE NUMBER SEVENTY + {0x109D2, 0x109FF, prN}, // No [46] MEROITIC CURSIVE NUMBER ONE HUNDRED..MEROITIC CURSIVE FRACTION TEN TWELFTHS + {0x10A00, 0x10A00, prN}, // Lo KHAROSHTHI LETTER A + {0x10A01, 0x10A03, prN}, // Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R + {0x10A05, 0x10A06, prN}, // Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O + {0x10A0C, 0x10A0F, prN}, // Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA + {0x10A10, 0x10A13, prN}, // Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA + {0x10A15, 0x10A17, prN}, // Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA + {0x10A19, 0x10A35, prN}, // Lo [29] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER VHA + {0x10A38, 0x10A3A, prN}, // Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW + {0x10A3F, 0x10A3F, prN}, // Mn KHAROSHTHI VIRAMA + {0x10A40, 0x10A48, prN}, // No [9] KHAROSHTHI DIGIT ONE..KHAROSHTHI FRACTION ONE HALF + {0x10A50, 0x10A58, prN}, // Po [9] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION LINES + {0x10A60, 0x10A7C, prN}, // Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH + {0x10A7D, 0x10A7E, prN}, // No [2] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMBER FIFTY + {0x10A7F, 0x10A7F, prN}, // Po OLD SOUTH ARABIAN NUMERIC INDICATOR + {0x10A80, 0x10A9C, prN}, // Lo [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH + {0x10A9D, 0x10A9F, prN}, // No [3] OLD NORTH ARABIAN NUMBER ONE..OLD NORTH ARABIAN NUMBER TWENTY + {0x10AC0, 0x10AC7, prN}, // Lo [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW + {0x10AC8, 0x10AC8, prN}, // So MANICHAEAN SIGN UD + {0x10AC9, 0x10AE4, prN}, // Lo [28] MANICHAEAN LETTER ZAYIN..MANICHAEAN LETTER TAW + {0x10AE5, 0x10AE6, prN}, // Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW + {0x10AEB, 0x10AEF, prN}, // No [5] MANICHAEAN NUMBER ONE..MANICHAEAN NUMBER ONE HUNDRED + {0x10AF0, 0x10AF6, prN}, // Po [7] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION LINE FILLER + {0x10B00, 0x10B35, prN}, // Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE + {0x10B39, 0x10B3F, prN}, // Po [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION + {0x10B40, 0x10B55, prN}, // Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW + {0x10B58, 0x10B5F, prN}, // No [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND + {0x10B60, 0x10B72, prN}, // Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW + {0x10B78, 0x10B7F, prN}, // No [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND + {0x10B80, 0x10B91, prN}, // Lo [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW + {0x10B99, 0x10B9C, prN}, // Po [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT + {0x10BA9, 0x10BAF, prN}, // No [7] PSALTER PAHLAVI NUMBER ONE..PSALTER PAHLAVI NUMBER ONE HUNDRED + {0x10C00, 0x10C48, prN}, // Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH + {0x10C80, 0x10CB2, prN}, // Lu [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US + {0x10CC0, 0x10CF2, prN}, // Ll [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US + {0x10CFA, 0x10CFF, prN}, // No [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND + {0x10D00, 0x10D23, prN}, // Lo [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA + {0x10D24, 0x10D27, prN}, // Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI + {0x10D30, 0x10D39, prN}, // Nd [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE + {0x10E60, 0x10E7E, prN}, // No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS + {0x10E80, 0x10EA9, prN}, // Lo [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET + {0x10EAB, 0x10EAC, prN}, // Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK + {0x10EAD, 0x10EAD, prN}, // Pd YEZIDI HYPHENATION MARK + {0x10EB0, 0x10EB1, prN}, // Lo [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE + {0x10EFD, 0x10EFF, prN}, // Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA + {0x10F00, 0x10F1C, prN}, // Lo [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL + {0x10F1D, 0x10F26, prN}, // No [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF + {0x10F27, 0x10F27, prN}, // Lo OLD SOGDIAN LIGATURE AYIN-DALETH + {0x10F30, 0x10F45, prN}, // Lo [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN + {0x10F46, 0x10F50, prN}, // Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW + {0x10F51, 0x10F54, prN}, // No [4] SOGDIAN NUMBER ONE..SOGDIAN NUMBER ONE HUNDRED + {0x10F55, 0x10F59, prN}, // Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT + {0x10F70, 0x10F81, prN}, // Lo [18] OLD UYGHUR LETTER ALEPH..OLD UYGHUR LETTER LESH + {0x10F82, 0x10F85, prN}, // Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW + {0x10F86, 0x10F89, prN}, // Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS + {0x10FB0, 0x10FC4, prN}, // Lo [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW + {0x10FC5, 0x10FCB, prN}, // No [7] CHORASMIAN NUMBER ONE..CHORASMIAN NUMBER ONE HUNDRED + {0x10FE0, 0x10FF6, prN}, // Lo [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH + {0x11000, 0x11000, prN}, // Mc BRAHMI SIGN CANDRABINDU + {0x11001, 0x11001, prN}, // Mn BRAHMI SIGN ANUSVARA + {0x11002, 0x11002, prN}, // Mc BRAHMI SIGN VISARGA + {0x11003, 0x11037, prN}, // Lo [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI LETTER OLD TAMIL NNNA + {0x11038, 0x11046, prN}, // Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA + {0x11047, 0x1104D, prN}, // Po [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS + {0x11052, 0x11065, prN}, // No [20] BRAHMI NUMBER ONE..BRAHMI NUMBER ONE THOUSAND + {0x11066, 0x1106F, prN}, // Nd [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE + {0x11070, 0x11070, prN}, // Mn BRAHMI SIGN OLD TAMIL VIRAMA + {0x11071, 0x11072, prN}, // Lo [2] BRAHMI LETTER OLD TAMIL SHORT E..BRAHMI LETTER OLD TAMIL SHORT O + {0x11073, 0x11074, prN}, // Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O + {0x11075, 0x11075, prN}, // Lo BRAHMI LETTER OLD TAMIL LLA + {0x1107F, 0x1107F, prN}, // Mn BRAHMI NUMBER JOINER + {0x11080, 0x11081, prN}, // Mn [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA + {0x11082, 0x11082, prN}, // Mc KAITHI SIGN VISARGA + {0x11083, 0x110AF, prN}, // Lo [45] KAITHI LETTER A..KAITHI LETTER HA + {0x110B0, 0x110B2, prN}, // Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II + {0x110B3, 0x110B6, prN}, // Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI + {0x110B7, 0x110B8, prN}, // Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU + {0x110B9, 0x110BA, prN}, // Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA + {0x110BB, 0x110BC, prN}, // Po [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN + {0x110BD, 0x110BD, prN}, // Cf KAITHI NUMBER SIGN + {0x110BE, 0x110C1, prN}, // Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA + {0x110C2, 0x110C2, prN}, // Mn KAITHI VOWEL SIGN VOCALIC R + {0x110CD, 0x110CD, prN}, // Cf KAITHI NUMBER SIGN ABOVE + {0x110D0, 0x110E8, prN}, // Lo [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE + {0x110F0, 0x110F9, prN}, // Nd [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE + {0x11100, 0x11102, prN}, // Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA + {0x11103, 0x11126, prN}, // Lo [36] CHAKMA LETTER AA..CHAKMA LETTER HAA + {0x11127, 0x1112B, prN}, // Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU + {0x1112C, 0x1112C, prN}, // Mc CHAKMA VOWEL SIGN E + {0x1112D, 0x11134, prN}, // Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA + {0x11136, 0x1113F, prN}, // Nd [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE + {0x11140, 0x11143, prN}, // Po [4] CHAKMA SECTION MARK..CHAKMA QUESTION MARK + {0x11144, 0x11144, prN}, // Lo CHAKMA LETTER LHAA + {0x11145, 0x11146, prN}, // Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI + {0x11147, 0x11147, prN}, // Lo CHAKMA LETTER VAA + {0x11150, 0x11172, prN}, // Lo [35] MAHAJANI LETTER A..MAHAJANI LETTER RRA + {0x11173, 0x11173, prN}, // Mn MAHAJANI SIGN NUKTA + {0x11174, 0x11175, prN}, // Po [2] MAHAJANI ABBREVIATION SIGN..MAHAJANI SECTION MARK + {0x11176, 0x11176, prN}, // Lo MAHAJANI LIGATURE SHRI + {0x11180, 0x11181, prN}, // Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA + {0x11182, 0x11182, prN}, // Mc SHARADA SIGN VISARGA + {0x11183, 0x111B2, prN}, // Lo [48] SHARADA LETTER A..SHARADA LETTER HA + {0x111B3, 0x111B5, prN}, // Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II + {0x111B6, 0x111BE, prN}, // Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O + {0x111BF, 0x111C0, prN}, // Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA + {0x111C1, 0x111C4, prN}, // Lo [4] SHARADA SIGN AVAGRAHA..SHARADA OM + {0x111C5, 0x111C8, prN}, // Po [4] SHARADA DANDA..SHARADA SEPARATOR + {0x111C9, 0x111CC, prN}, // Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK + {0x111CD, 0x111CD, prN}, // Po SHARADA SUTRA MARK + {0x111CE, 0x111CE, prN}, // Mc SHARADA VOWEL SIGN PRISHTHAMATRA E + {0x111CF, 0x111CF, prN}, // Mn SHARADA SIGN INVERTED CANDRABINDU + {0x111D0, 0x111D9, prN}, // Nd [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE + {0x111DA, 0x111DA, prN}, // Lo SHARADA EKAM + {0x111DB, 0x111DB, prN}, // Po SHARADA SIGN SIDDHAM + {0x111DC, 0x111DC, prN}, // Lo SHARADA HEADSTROKE + {0x111DD, 0x111DF, prN}, // Po [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 + {0x111E1, 0x111F4, prN}, // No [20] SINHALA ARCHAIC DIGIT ONE..SINHALA ARCHAIC NUMBER ONE THOUSAND + {0x11200, 0x11211, prN}, // Lo [18] KHOJKI LETTER A..KHOJKI LETTER JJA + {0x11213, 0x1122B, prN}, // Lo [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA + {0x1122C, 0x1122E, prN}, // Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II + {0x1122F, 0x11231, prN}, // Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI + {0x11232, 0x11233, prN}, // Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU + {0x11234, 0x11234, prN}, // Mn KHOJKI SIGN ANUSVARA + {0x11235, 0x11235, prN}, // Mc KHOJKI SIGN VIRAMA + {0x11236, 0x11237, prN}, // Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA + {0x11238, 0x1123D, prN}, // Po [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN + {0x1123E, 0x1123E, prN}, // Mn KHOJKI SIGN SUKUN + {0x1123F, 0x11240, prN}, // Lo [2] KHOJKI LETTER QA..KHOJKI LETTER SHORT I + {0x11241, 0x11241, prN}, // Mn KHOJKI VOWEL SIGN VOCALIC R + {0x11280, 0x11286, prN}, // Lo [7] MULTANI LETTER A..MULTANI LETTER GA + {0x11288, 0x11288, prN}, // Lo MULTANI LETTER GHA + {0x1128A, 0x1128D, prN}, // Lo [4] MULTANI LETTER CA..MULTANI LETTER JJA + {0x1128F, 0x1129D, prN}, // Lo [15] MULTANI LETTER NYA..MULTANI LETTER BA + {0x1129F, 0x112A8, prN}, // Lo [10] MULTANI LETTER BHA..MULTANI LETTER RHA + {0x112A9, 0x112A9, prN}, // Po MULTANI SECTION MARK + {0x112B0, 0x112DE, prN}, // Lo [47] KHUDAWADI LETTER A..KHUDAWADI LETTER HA + {0x112DF, 0x112DF, prN}, // Mn KHUDAWADI SIGN ANUSVARA + {0x112E0, 0x112E2, prN}, // Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II + {0x112E3, 0x112EA, prN}, // Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA + {0x112F0, 0x112F9, prN}, // Nd [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE + {0x11300, 0x11301, prN}, // Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU + {0x11302, 0x11303, prN}, // Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA + {0x11305, 0x1130C, prN}, // Lo [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L + {0x1130F, 0x11310, prN}, // Lo [2] GRANTHA LETTER EE..GRANTHA LETTER AI + {0x11313, 0x11328, prN}, // Lo [22] GRANTHA LETTER OO..GRANTHA LETTER NA + {0x1132A, 0x11330, prN}, // Lo [7] GRANTHA LETTER PA..GRANTHA LETTER RA + {0x11332, 0x11333, prN}, // Lo [2] GRANTHA LETTER LA..GRANTHA LETTER LLA + {0x11335, 0x11339, prN}, // Lo [5] GRANTHA LETTER VA..GRANTHA LETTER HA + {0x1133B, 0x1133C, prN}, // Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA + {0x1133D, 0x1133D, prN}, // Lo GRANTHA SIGN AVAGRAHA + {0x1133E, 0x1133F, prN}, // Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I + {0x11340, 0x11340, prN}, // Mn GRANTHA VOWEL SIGN II + {0x11341, 0x11344, prN}, // Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR + {0x11347, 0x11348, prN}, // Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI + {0x1134B, 0x1134D, prN}, // Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA + {0x11350, 0x11350, prN}, // Lo GRANTHA OM + {0x11357, 0x11357, prN}, // Mc GRANTHA AU LENGTH MARK + {0x1135D, 0x11361, prN}, // Lo [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL + {0x11362, 0x11363, prN}, // Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL + {0x11366, 0x1136C, prN}, // Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX + {0x11370, 0x11374, prN}, // Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA + {0x11400, 0x11434, prN}, // Lo [53] NEWA LETTER A..NEWA LETTER HA + {0x11435, 0x11437, prN}, // Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II + {0x11438, 0x1143F, prN}, // Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI + {0x11440, 0x11441, prN}, // Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU + {0x11442, 0x11444, prN}, // Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA + {0x11445, 0x11445, prN}, // Mc NEWA SIGN VISARGA + {0x11446, 0x11446, prN}, // Mn NEWA SIGN NUKTA + {0x11447, 0x1144A, prN}, // Lo [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI + {0x1144B, 0x1144F, prN}, // Po [5] NEWA DANDA..NEWA ABBREVIATION SIGN + {0x11450, 0x11459, prN}, // Nd [10] NEWA DIGIT ZERO..NEWA DIGIT NINE + {0x1145A, 0x1145B, prN}, // Po [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK + {0x1145D, 0x1145D, prN}, // Po NEWA INSERTION SIGN + {0x1145E, 0x1145E, prN}, // Mn NEWA SANDHI MARK + {0x1145F, 0x11461, prN}, // Lo [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA + {0x11480, 0x114AF, prN}, // Lo [48] TIRHUTA ANJI..TIRHUTA LETTER HA + {0x114B0, 0x114B2, prN}, // Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II + {0x114B3, 0x114B8, prN}, // Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL + {0x114B9, 0x114B9, prN}, // Mc TIRHUTA VOWEL SIGN E + {0x114BA, 0x114BA, prN}, // Mn TIRHUTA VOWEL SIGN SHORT E + {0x114BB, 0x114BE, prN}, // Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU + {0x114BF, 0x114C0, prN}, // Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA + {0x114C1, 0x114C1, prN}, // Mc TIRHUTA SIGN VISARGA + {0x114C2, 0x114C3, prN}, // Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA + {0x114C4, 0x114C5, prN}, // Lo [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA GVANG + {0x114C6, 0x114C6, prN}, // Po TIRHUTA ABBREVIATION SIGN + {0x114C7, 0x114C7, prN}, // Lo TIRHUTA OM + {0x114D0, 0x114D9, prN}, // Nd [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE + {0x11580, 0x115AE, prN}, // Lo [47] SIDDHAM LETTER A..SIDDHAM LETTER HA + {0x115AF, 0x115B1, prN}, // Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II + {0x115B2, 0x115B5, prN}, // Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR + {0x115B8, 0x115BB, prN}, // Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU + {0x115BC, 0x115BD, prN}, // Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA + {0x115BE, 0x115BE, prN}, // Mc SIDDHAM SIGN VISARGA + {0x115BF, 0x115C0, prN}, // Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA + {0x115C1, 0x115D7, prN}, // Po [23] SIDDHAM SIGN SIDDHAM..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES + {0x115D8, 0x115DB, prN}, // Lo [4] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM LETTER ALTERNATE U + {0x115DC, 0x115DD, prN}, // Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU + {0x11600, 0x1162F, prN}, // Lo [48] MODI LETTER A..MODI LETTER LLA + {0x11630, 0x11632, prN}, // Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II + {0x11633, 0x1163A, prN}, // Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI + {0x1163B, 0x1163C, prN}, // Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU + {0x1163D, 0x1163D, prN}, // Mn MODI SIGN ANUSVARA + {0x1163E, 0x1163E, prN}, // Mc MODI SIGN VISARGA + {0x1163F, 0x11640, prN}, // Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA + {0x11641, 0x11643, prN}, // Po [3] MODI DANDA..MODI ABBREVIATION SIGN + {0x11644, 0x11644, prN}, // Lo MODI SIGN HUVA + {0x11650, 0x11659, prN}, // Nd [10] MODI DIGIT ZERO..MODI DIGIT NINE + {0x11660, 0x1166C, prN}, // Po [13] MONGOLIAN BIRGA WITH ORNAMENT..MONGOLIAN TURNED SWIRL BIRGA WITH DOUBLE ORNAMENT + {0x11680, 0x116AA, prN}, // Lo [43] TAKRI LETTER A..TAKRI LETTER RRA + {0x116AB, 0x116AB, prN}, // Mn TAKRI SIGN ANUSVARA + {0x116AC, 0x116AC, prN}, // Mc TAKRI SIGN VISARGA + {0x116AD, 0x116AD, prN}, // Mn TAKRI VOWEL SIGN AA + {0x116AE, 0x116AF, prN}, // Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II + {0x116B0, 0x116B5, prN}, // Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU + {0x116B6, 0x116B6, prN}, // Mc TAKRI SIGN VIRAMA + {0x116B7, 0x116B7, prN}, // Mn TAKRI SIGN NUKTA + {0x116B8, 0x116B8, prN}, // Lo TAKRI LETTER ARCHAIC KHA + {0x116B9, 0x116B9, prN}, // Po TAKRI ABBREVIATION SIGN + {0x116C0, 0x116C9, prN}, // Nd [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE + {0x11700, 0x1171A, prN}, // Lo [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA + {0x1171D, 0x1171F, prN}, // Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA + {0x11720, 0x11721, prN}, // Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA + {0x11722, 0x11725, prN}, // Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU + {0x11726, 0x11726, prN}, // Mc AHOM VOWEL SIGN E + {0x11727, 0x1172B, prN}, // Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER + {0x11730, 0x11739, prN}, // Nd [10] AHOM DIGIT ZERO..AHOM DIGIT NINE + {0x1173A, 0x1173B, prN}, // No [2] AHOM NUMBER TEN..AHOM NUMBER TWENTY + {0x1173C, 0x1173E, prN}, // Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI + {0x1173F, 0x1173F, prN}, // So AHOM SYMBOL VI + {0x11740, 0x11746, prN}, // Lo [7] AHOM LETTER CA..AHOM LETTER LLA + {0x11800, 0x1182B, prN}, // Lo [44] DOGRA LETTER A..DOGRA LETTER RRA + {0x1182C, 0x1182E, prN}, // Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II + {0x1182F, 0x11837, prN}, // Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA + {0x11838, 0x11838, prN}, // Mc DOGRA SIGN VISARGA + {0x11839, 0x1183A, prN}, // Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA + {0x1183B, 0x1183B, prN}, // Po DOGRA ABBREVIATION SIGN + {0x118A0, 0x118DF, prN}, // L& [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO + {0x118E0, 0x118E9, prN}, // Nd [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE + {0x118EA, 0x118F2, prN}, // No [9] WARANG CITI NUMBER TEN..WARANG CITI NUMBER NINETY + {0x118FF, 0x118FF, prN}, // Lo WARANG CITI OM + {0x11900, 0x11906, prN}, // Lo [7] DIVES AKURU LETTER A..DIVES AKURU LETTER E + {0x11909, 0x11909, prN}, // Lo DIVES AKURU LETTER O + {0x1190C, 0x11913, prN}, // Lo [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA + {0x11915, 0x11916, prN}, // Lo [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA + {0x11918, 0x1192F, prN}, // Lo [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA + {0x11930, 0x11935, prN}, // Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E + {0x11937, 0x11938, prN}, // Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O + {0x1193B, 0x1193C, prN}, // Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU + {0x1193D, 0x1193D, prN}, // Mc DIVES AKURU SIGN HALANTA + {0x1193E, 0x1193E, prN}, // Mn DIVES AKURU VIRAMA + {0x1193F, 0x1193F, prN}, // Lo DIVES AKURU PREFIXED NASAL SIGN + {0x11940, 0x11940, prN}, // Mc DIVES AKURU MEDIAL YA + {0x11941, 0x11941, prN}, // Lo DIVES AKURU INITIAL RA + {0x11942, 0x11942, prN}, // Mc DIVES AKURU MEDIAL RA + {0x11943, 0x11943, prN}, // Mn DIVES AKURU SIGN NUKTA + {0x11944, 0x11946, prN}, // Po [3] DIVES AKURU DOUBLE DANDA..DIVES AKURU END OF TEXT MARK + {0x11950, 0x11959, prN}, // Nd [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE + {0x119A0, 0x119A7, prN}, // Lo [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR + {0x119AA, 0x119D0, prN}, // Lo [39] NANDINAGARI LETTER E..NANDINAGARI LETTER RRA + {0x119D1, 0x119D3, prN}, // Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II + {0x119D4, 0x119D7, prN}, // Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR + {0x119DA, 0x119DB, prN}, // Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI + {0x119DC, 0x119DF, prN}, // Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA + {0x119E0, 0x119E0, prN}, // Mn NANDINAGARI SIGN VIRAMA + {0x119E1, 0x119E1, prN}, // Lo NANDINAGARI SIGN AVAGRAHA + {0x119E2, 0x119E2, prN}, // Po NANDINAGARI SIGN SIDDHAM + {0x119E3, 0x119E3, prN}, // Lo NANDINAGARI HEADSTROKE + {0x119E4, 0x119E4, prN}, // Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E + {0x11A00, 0x11A00, prN}, // Lo ZANABAZAR SQUARE LETTER A + {0x11A01, 0x11A0A, prN}, // Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK + {0x11A0B, 0x11A32, prN}, // Lo [40] ZANABAZAR SQUARE LETTER KA..ZANABAZAR SQUARE LETTER KSSA + {0x11A33, 0x11A38, prN}, // Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA + {0x11A39, 0x11A39, prN}, // Mc ZANABAZAR SQUARE SIGN VISARGA + {0x11A3A, 0x11A3A, prN}, // Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA + {0x11A3B, 0x11A3E, prN}, // Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA + {0x11A3F, 0x11A46, prN}, // Po [8] ZANABAZAR SQUARE INITIAL HEAD MARK..ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK + {0x11A47, 0x11A47, prN}, // Mn ZANABAZAR SQUARE SUBJOINER + {0x11A50, 0x11A50, prN}, // Lo SOYOMBO LETTER A + {0x11A51, 0x11A56, prN}, // Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE + {0x11A57, 0x11A58, prN}, // Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU + {0x11A59, 0x11A5B, prN}, // Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK + {0x11A5C, 0x11A89, prN}, // Lo [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA + {0x11A8A, 0x11A96, prN}, // Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA + {0x11A97, 0x11A97, prN}, // Mc SOYOMBO SIGN VISARGA + {0x11A98, 0x11A99, prN}, // Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER + {0x11A9A, 0x11A9C, prN}, // Po [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD + {0x11A9D, 0x11A9D, prN}, // Lo SOYOMBO MARK PLUTA + {0x11A9E, 0x11AA2, prN}, // Po [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2 + {0x11AB0, 0x11ABF, prN}, // Lo [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA + {0x11AC0, 0x11AF8, prN}, // Lo [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL + {0x11B00, 0x11B09, prN}, // Po [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU + {0x11C00, 0x11C08, prN}, // Lo [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L + {0x11C0A, 0x11C2E, prN}, // Lo [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA + {0x11C2F, 0x11C2F, prN}, // Mc BHAIKSUKI VOWEL SIGN AA + {0x11C30, 0x11C36, prN}, // Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L + {0x11C38, 0x11C3D, prN}, // Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA + {0x11C3E, 0x11C3E, prN}, // Mc BHAIKSUKI SIGN VISARGA + {0x11C3F, 0x11C3F, prN}, // Mn BHAIKSUKI SIGN VIRAMA + {0x11C40, 0x11C40, prN}, // Lo BHAIKSUKI SIGN AVAGRAHA + {0x11C41, 0x11C45, prN}, // Po [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2 + {0x11C50, 0x11C59, prN}, // Nd [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE + {0x11C5A, 0x11C6C, prN}, // No [19] BHAIKSUKI NUMBER ONE..BHAIKSUKI HUNDREDS UNIT MARK + {0x11C70, 0x11C71, prN}, // Po [2] MARCHEN HEAD MARK..MARCHEN MARK SHAD + {0x11C72, 0x11C8F, prN}, // Lo [30] MARCHEN LETTER KA..MARCHEN LETTER A + {0x11C92, 0x11CA7, prN}, // Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA + {0x11CA9, 0x11CA9, prN}, // Mc MARCHEN SUBJOINED LETTER YA + {0x11CAA, 0x11CB0, prN}, // Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA + {0x11CB1, 0x11CB1, prN}, // Mc MARCHEN VOWEL SIGN I + {0x11CB2, 0x11CB3, prN}, // Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E + {0x11CB4, 0x11CB4, prN}, // Mc MARCHEN VOWEL SIGN O + {0x11CB5, 0x11CB6, prN}, // Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU + {0x11D00, 0x11D06, prN}, // Lo [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E + {0x11D08, 0x11D09, prN}, // Lo [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O + {0x11D0B, 0x11D30, prN}, // Lo [38] MASARAM GONDI LETTER AU..MASARAM GONDI LETTER TRA + {0x11D31, 0x11D36, prN}, // Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R + {0x11D3A, 0x11D3A, prN}, // Mn MASARAM GONDI VOWEL SIGN E + {0x11D3C, 0x11D3D, prN}, // Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O + {0x11D3F, 0x11D45, prN}, // Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA + {0x11D46, 0x11D46, prN}, // Lo MASARAM GONDI REPHA + {0x11D47, 0x11D47, prN}, // Mn MASARAM GONDI RA-KARA + {0x11D50, 0x11D59, prN}, // Nd [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE + {0x11D60, 0x11D65, prN}, // Lo [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU + {0x11D67, 0x11D68, prN}, // Lo [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI + {0x11D6A, 0x11D89, prN}, // Lo [32] GUNJALA GONDI LETTER OO..GUNJALA GONDI LETTER SA + {0x11D8A, 0x11D8E, prN}, // Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU + {0x11D90, 0x11D91, prN}, // Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI + {0x11D93, 0x11D94, prN}, // Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU + {0x11D95, 0x11D95, prN}, // Mn GUNJALA GONDI SIGN ANUSVARA + {0x11D96, 0x11D96, prN}, // Mc GUNJALA GONDI SIGN VISARGA + {0x11D97, 0x11D97, prN}, // Mn GUNJALA GONDI VIRAMA + {0x11D98, 0x11D98, prN}, // Lo GUNJALA GONDI OM + {0x11DA0, 0x11DA9, prN}, // Nd [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE + {0x11EE0, 0x11EF2, prN}, // Lo [19] MAKASAR LETTER KA..MAKASAR ANGKA + {0x11EF3, 0x11EF4, prN}, // Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U + {0x11EF5, 0x11EF6, prN}, // Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O + {0x11EF7, 0x11EF8, prN}, // Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION + {0x11F00, 0x11F01, prN}, // Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA + {0x11F02, 0x11F02, prN}, // Lo KAWI SIGN REPHA + {0x11F03, 0x11F03, prN}, // Mc KAWI SIGN VISARGA + {0x11F04, 0x11F10, prN}, // Lo [13] KAWI LETTER A..KAWI LETTER O + {0x11F12, 0x11F33, prN}, // Lo [34] KAWI LETTER KA..KAWI LETTER JNYA + {0x11F34, 0x11F35, prN}, // Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA + {0x11F36, 0x11F3A, prN}, // Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R + {0x11F3E, 0x11F3F, prN}, // Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI + {0x11F40, 0x11F40, prN}, // Mn KAWI VOWEL SIGN EU + {0x11F41, 0x11F41, prN}, // Mc KAWI SIGN KILLER + {0x11F42, 0x11F42, prN}, // Mn KAWI CONJOINER + {0x11F43, 0x11F4F, prN}, // Po [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL + {0x11F50, 0x11F59, prN}, // Nd [10] KAWI DIGIT ZERO..KAWI DIGIT NINE + {0x11FB0, 0x11FB0, prN}, // Lo LISU LETTER YHA + {0x11FC0, 0x11FD4, prN}, // No [21] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL FRACTION DOWNSCALING FACTOR KIIZH + {0x11FD5, 0x11FDC, prN}, // So [8] TAMIL SIGN NEL..TAMIL SIGN MUKKURUNI + {0x11FDD, 0x11FE0, prN}, // Sc [4] TAMIL SIGN KAACU..TAMIL SIGN VARAAKAN + {0x11FE1, 0x11FF1, prN}, // So [17] TAMIL SIGN PAARAM..TAMIL SIGN VAKAIYARAA + {0x11FFF, 0x11FFF, prN}, // Po TAMIL PUNCTUATION END OF TEXT + {0x12000, 0x12399, prN}, // Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U + {0x12400, 0x1246E, prN}, // Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM + {0x12470, 0x12474, prN}, // Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON + {0x12480, 0x12543, prN}, // Lo [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU + {0x12F90, 0x12FF0, prN}, // Lo [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 + {0x12FF1, 0x12FF2, prN}, // Po [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302 + {0x13000, 0x1342F, prN}, // Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D + {0x13430, 0x1343F, prN}, // Cf [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE + {0x13440, 0x13440, prN}, // Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY + {0x13441, 0x13446, prN}, // Lo [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN + {0x13447, 0x13455, prN}, // Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED + {0x14400, 0x14646, prN}, // Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 + {0x16800, 0x16A38, prN}, // Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ + {0x16A40, 0x16A5E, prN}, // Lo [31] MRO LETTER TA..MRO LETTER TEK + {0x16A60, 0x16A69, prN}, // Nd [10] MRO DIGIT ZERO..MRO DIGIT NINE + {0x16A6E, 0x16A6F, prN}, // Po [2] MRO DANDA..MRO DOUBLE DANDA + {0x16A70, 0x16ABE, prN}, // Lo [79] TANGSA LETTER OZ..TANGSA LETTER ZA + {0x16AC0, 0x16AC9, prN}, // Nd [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE + {0x16AD0, 0x16AED, prN}, // Lo [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I + {0x16AF0, 0x16AF4, prN}, // Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE + {0x16AF5, 0x16AF5, prN}, // Po BASSA VAH FULL STOP + {0x16B00, 0x16B2F, prN}, // Lo [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU + {0x16B30, 0x16B36, prN}, // Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM + {0x16B37, 0x16B3B, prN}, // Po [5] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS FEEM + {0x16B3C, 0x16B3F, prN}, // So [4] PAHAWH HMONG SIGN XYEEM NTXIV..PAHAWH HMONG SIGN XYEEM FAIB + {0x16B40, 0x16B43, prN}, // Lm [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM + {0x16B44, 0x16B44, prN}, // Po PAHAWH HMONG SIGN XAUS + {0x16B45, 0x16B45, prN}, // So PAHAWH HMONG SIGN CIM TSOV ROG + {0x16B50, 0x16B59, prN}, // Nd [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE + {0x16B5B, 0x16B61, prN}, // No [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS + {0x16B63, 0x16B77, prN}, // Lo [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS + {0x16B7D, 0x16B8F, prN}, // Lo [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ + {0x16E40, 0x16E7F, prN}, // L& [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y + {0x16E80, 0x16E96, prN}, // No [23] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN DIGIT THREE ALTERNATE FORM + {0x16E97, 0x16E9A, prN}, // Po [4] MEDEFAIDRIN COMMA..MEDEFAIDRIN EXCLAMATION OH + {0x16F00, 0x16F4A, prN}, // Lo [75] MIAO LETTER PA..MIAO LETTER RTE + {0x16F4F, 0x16F4F, prN}, // Mn MIAO SIGN CONSONANT MODIFIER BAR + {0x16F50, 0x16F50, prN}, // Lo MIAO LETTER NASALIZATION + {0x16F51, 0x16F87, prN}, // Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI + {0x16F8F, 0x16F92, prN}, // Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW + {0x16F93, 0x16F9F, prN}, // Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 + {0x16FE0, 0x16FE1, prW}, // Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK + {0x16FE2, 0x16FE2, prW}, // Po OLD CHINESE HOOK MARK + {0x16FE3, 0x16FE3, prW}, // Lm OLD CHINESE ITERATION MARK + {0x16FE4, 0x16FE4, prW}, // Mn KHITAN SMALL SCRIPT FILLER + {0x16FF0, 0x16FF1, prW}, // Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY + {0x17000, 0x187F7, prW}, // Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 + {0x18800, 0x18AFF, prW}, // Lo [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768 + {0x18B00, 0x18CD5, prW}, // Lo [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5 + {0x18D00, 0x18D08, prW}, // Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 + {0x1AFF0, 0x1AFF3, prW}, // Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 + {0x1AFF5, 0x1AFFB, prW}, // Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 + {0x1AFFD, 0x1AFFE, prW}, // Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 + {0x1B000, 0x1B0FF, prW}, // Lo [256] KATAKANA LETTER ARCHAIC E..HENTAIGANA LETTER RE-2 + {0x1B100, 0x1B122, prW}, // Lo [35] HENTAIGANA LETTER RE-3..KATAKANA LETTER ARCHAIC WU + {0x1B132, 0x1B132, prW}, // Lo HIRAGANA LETTER SMALL KO + {0x1B150, 0x1B152, prW}, // Lo [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO + {0x1B155, 0x1B155, prW}, // Lo KATAKANA LETTER SMALL KO + {0x1B164, 0x1B167, prW}, // Lo [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N + {0x1B170, 0x1B2FB, prW}, // Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB + {0x1BC00, 0x1BC6A, prN}, // Lo [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M + {0x1BC70, 0x1BC7C, prN}, // Lo [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK + {0x1BC80, 0x1BC88, prN}, // Lo [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL + {0x1BC90, 0x1BC99, prN}, // Lo [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW + {0x1BC9C, 0x1BC9C, prN}, // So DUPLOYAN SIGN O WITH CROSS + {0x1BC9D, 0x1BC9E, prN}, // Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK + {0x1BC9F, 0x1BC9F, prN}, // Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP + {0x1BCA0, 0x1BCA3, prN}, // Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP + {0x1CF00, 0x1CF2D, prN}, // Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT + {0x1CF30, 0x1CF46, prN}, // Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG + {0x1CF50, 0x1CFC3, prN}, // So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK + {0x1D000, 0x1D0F5, prN}, // So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO + {0x1D100, 0x1D126, prN}, // So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 + {0x1D129, 0x1D164, prN}, // So [60] MUSICAL SYMBOL MULTIPLE MEASURE REST..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE + {0x1D165, 0x1D166, prN}, // Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM + {0x1D167, 0x1D169, prN}, // Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 + {0x1D16A, 0x1D16C, prN}, // So [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 + {0x1D16D, 0x1D172, prN}, // Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 + {0x1D173, 0x1D17A, prN}, // Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE + {0x1D17B, 0x1D182, prN}, // Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE + {0x1D183, 0x1D184, prN}, // So [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN + {0x1D185, 0x1D18B, prN}, // Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE + {0x1D18C, 0x1D1A9, prN}, // So [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH + {0x1D1AA, 0x1D1AD, prN}, // Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO + {0x1D1AE, 0x1D1EA, prN}, // So [61] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL KORON + {0x1D200, 0x1D241, prN}, // So [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 + {0x1D242, 0x1D244, prN}, // Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + {0x1D245, 0x1D245, prN}, // So GREEK MUSICAL LEIMMA + {0x1D2C0, 0x1D2D3, prN}, // No [20] KAKTOVIK NUMERAL ZERO..KAKTOVIK NUMERAL NINETEEN + {0x1D2E0, 0x1D2F3, prN}, // No [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN + {0x1D300, 0x1D356, prN}, // So [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING + {0x1D360, 0x1D378, prN}, // No [25] COUNTING ROD UNIT DIGIT ONE..TALLY MARK FIVE + {0x1D400, 0x1D454, prN}, // L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G + {0x1D456, 0x1D49C, prN}, // L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A + {0x1D49E, 0x1D49F, prN}, // Lu [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D + {0x1D4A2, 0x1D4A2, prN}, // Lu MATHEMATICAL SCRIPT CAPITAL G + {0x1D4A5, 0x1D4A6, prN}, // Lu [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K + {0x1D4A9, 0x1D4AC, prN}, // Lu [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q + {0x1D4AE, 0x1D4B9, prN}, // L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D + {0x1D4BB, 0x1D4BB, prN}, // Ll MATHEMATICAL SCRIPT SMALL F + {0x1D4BD, 0x1D4C3, prN}, // Ll [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N + {0x1D4C5, 0x1D505, prN}, // L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B + {0x1D507, 0x1D50A, prN}, // Lu [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G + {0x1D50D, 0x1D514, prN}, // Lu [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q + {0x1D516, 0x1D51C, prN}, // Lu [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y + {0x1D51E, 0x1D539, prN}, // L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B + {0x1D53B, 0x1D53E, prN}, // Lu [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G + {0x1D540, 0x1D544, prN}, // Lu [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M + {0x1D546, 0x1D546, prN}, // Lu MATHEMATICAL DOUBLE-STRUCK CAPITAL O + {0x1D54A, 0x1D550, prN}, // Lu [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y + {0x1D552, 0x1D6A5, prN}, // L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J + {0x1D6A8, 0x1D6C0, prN}, // Lu [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA + {0x1D6C1, 0x1D6C1, prN}, // Sm MATHEMATICAL BOLD NABLA + {0x1D6C2, 0x1D6DA, prN}, // Ll [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA + {0x1D6DB, 0x1D6DB, prN}, // Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL + {0x1D6DC, 0x1D6FA, prN}, // L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA + {0x1D6FB, 0x1D6FB, prN}, // Sm MATHEMATICAL ITALIC NABLA + {0x1D6FC, 0x1D714, prN}, // Ll [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA + {0x1D715, 0x1D715, prN}, // Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL + {0x1D716, 0x1D734, prN}, // L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA + {0x1D735, 0x1D735, prN}, // Sm MATHEMATICAL BOLD ITALIC NABLA + {0x1D736, 0x1D74E, prN}, // Ll [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA + {0x1D74F, 0x1D74F, prN}, // Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL + {0x1D750, 0x1D76E, prN}, // L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA + {0x1D76F, 0x1D76F, prN}, // Sm MATHEMATICAL SANS-SERIF BOLD NABLA + {0x1D770, 0x1D788, prN}, // Ll [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA + {0x1D789, 0x1D789, prN}, // Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL + {0x1D78A, 0x1D7A8, prN}, // L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA + {0x1D7A9, 0x1D7A9, prN}, // Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA + {0x1D7AA, 0x1D7C2, prN}, // Ll [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA + {0x1D7C3, 0x1D7C3, prN}, // Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL + {0x1D7C4, 0x1D7CB, prN}, // L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA + {0x1D7CE, 0x1D7FF, prN}, // Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + {0x1D800, 0x1D9FF, prN}, // So [512] SIGNWRITING HAND-FIST INDEX..SIGNWRITING HEAD + {0x1DA00, 0x1DA36, prN}, // Mn [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN + {0x1DA37, 0x1DA3A, prN}, // So [4] SIGNWRITING AIR BLOW SMALL ROTATIONS..SIGNWRITING BREATH EXHALE + {0x1DA3B, 0x1DA6C, prN}, // Mn [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT + {0x1DA6D, 0x1DA74, prN}, // So [8] SIGNWRITING SHOULDER HIP SPINE..SIGNWRITING TORSO-FLOORPLANE TWISTING + {0x1DA75, 0x1DA75, prN}, // Mn SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS + {0x1DA76, 0x1DA83, prN}, // So [14] SIGNWRITING LIMB COMBINATION..SIGNWRITING LOCATION DEPTH + {0x1DA84, 0x1DA84, prN}, // Mn SIGNWRITING LOCATION HEAD NECK + {0x1DA85, 0x1DA86, prN}, // So [2] SIGNWRITING LOCATION TORSO..SIGNWRITING LOCATION LIMBS DIGITS + {0x1DA87, 0x1DA8B, prN}, // Po [5] SIGNWRITING COMMA..SIGNWRITING PARENTHESIS + {0x1DA9B, 0x1DA9F, prN}, // Mn [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 + {0x1DAA1, 0x1DAAF, prN}, // Mn [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 + {0x1DF00, 0x1DF09, prN}, // Ll [10] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK + {0x1DF0A, 0x1DF0A, prN}, // Lo LATIN LETTER RETROFLEX CLICK WITH RETROFLEX HOOK + {0x1DF0B, 0x1DF1E, prN}, // Ll [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL + {0x1DF25, 0x1DF2A, prN}, // Ll [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK + {0x1E000, 0x1E006, prN}, // Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE + {0x1E008, 0x1E018, prN}, // Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU + {0x1E01B, 0x1E021, prN}, // Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI + {0x1E023, 0x1E024, prN}, // Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS + {0x1E026, 0x1E02A, prN}, // Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA + {0x1E030, 0x1E06D, prN}, // Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE + {0x1E08F, 0x1E08F, prN}, // Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + {0x1E100, 0x1E12C, prN}, // Lo [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W + {0x1E130, 0x1E136, prN}, // Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D + {0x1E137, 0x1E13D, prN}, // Lm [7] NYIAKENG PUACHUE HMONG SIGN FOR PERSON..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER + {0x1E140, 0x1E149, prN}, // Nd [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE + {0x1E14E, 0x1E14E, prN}, // Lo NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ + {0x1E14F, 0x1E14F, prN}, // So NYIAKENG PUACHUE HMONG CIRCLED CA + {0x1E290, 0x1E2AD, prN}, // Lo [30] TOTO LETTER PA..TOTO LETTER A + {0x1E2AE, 0x1E2AE, prN}, // Mn TOTO SIGN RISING TONE + {0x1E2C0, 0x1E2EB, prN}, // Lo [44] WANCHO LETTER AA..WANCHO LETTER YIH + {0x1E2EC, 0x1E2EF, prN}, // Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI + {0x1E2F0, 0x1E2F9, prN}, // Nd [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE + {0x1E2FF, 0x1E2FF, prN}, // Sc WANCHO NGUN SIGN + {0x1E4D0, 0x1E4EA, prN}, // Lo [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL + {0x1E4EB, 0x1E4EB, prN}, // Lm NAG MUNDARI SIGN OJOD + {0x1E4EC, 0x1E4EF, prN}, // Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH + {0x1E4F0, 0x1E4F9, prN}, // Nd [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE + {0x1E7E0, 0x1E7E6, prN}, // Lo [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO + {0x1E7E8, 0x1E7EB, prN}, // Lo [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE + {0x1E7ED, 0x1E7EE, prN}, // Lo [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE + {0x1E7F0, 0x1E7FE, prN}, // Lo [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE + {0x1E800, 0x1E8C4, prN}, // Lo [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON + {0x1E8C7, 0x1E8CF, prN}, // No [9] MENDE KIKAKUI DIGIT ONE..MENDE KIKAKUI DIGIT NINE + {0x1E8D0, 0x1E8D6, prN}, // Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + {0x1E900, 0x1E943, prN}, // L& [68] ADLAM CAPITAL LETTER ALIF..ADLAM SMALL LETTER SHA + {0x1E944, 0x1E94A, prN}, // Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA + {0x1E94B, 0x1E94B, prN}, // Lm ADLAM NASALIZATION MARK + {0x1E950, 0x1E959, prN}, // Nd [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE + {0x1E95E, 0x1E95F, prN}, // Po [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK + {0x1EC71, 0x1ECAB, prN}, // No [59] INDIC SIYAQ NUMBER ONE..INDIC SIYAQ NUMBER PREFIXED NINE + {0x1ECAC, 0x1ECAC, prN}, // So INDIC SIYAQ PLACEHOLDER + {0x1ECAD, 0x1ECAF, prN}, // No [3] INDIC SIYAQ FRACTION ONE QUARTER..INDIC SIYAQ FRACTION THREE QUARTERS + {0x1ECB0, 0x1ECB0, prN}, // Sc INDIC SIYAQ RUPEE MARK + {0x1ECB1, 0x1ECB4, prN}, // No [4] INDIC SIYAQ NUMBER ALTERNATE ONE..INDIC SIYAQ ALTERNATE LAKH MARK + {0x1ED01, 0x1ED2D, prN}, // No [45] OTTOMAN SIYAQ NUMBER ONE..OTTOMAN SIYAQ NUMBER NINETY THOUSAND + {0x1ED2E, 0x1ED2E, prN}, // So OTTOMAN SIYAQ MARRATAN + {0x1ED2F, 0x1ED3D, prN}, // No [15] OTTOMAN SIYAQ ALTERNATE NUMBER TWO..OTTOMAN SIYAQ FRACTION ONE SIXTH + {0x1EE00, 0x1EE03, prN}, // Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL + {0x1EE05, 0x1EE1F, prN}, // Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF + {0x1EE21, 0x1EE22, prN}, // Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM + {0x1EE24, 0x1EE24, prN}, // Lo ARABIC MATHEMATICAL INITIAL HEH + {0x1EE27, 0x1EE27, prN}, // Lo ARABIC MATHEMATICAL INITIAL HAH + {0x1EE29, 0x1EE32, prN}, // Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF + {0x1EE34, 0x1EE37, prN}, // Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH + {0x1EE39, 0x1EE39, prN}, // Lo ARABIC MATHEMATICAL INITIAL DAD + {0x1EE3B, 0x1EE3B, prN}, // Lo ARABIC MATHEMATICAL INITIAL GHAIN + {0x1EE42, 0x1EE42, prN}, // Lo ARABIC MATHEMATICAL TAILED JEEM + {0x1EE47, 0x1EE47, prN}, // Lo ARABIC MATHEMATICAL TAILED HAH + {0x1EE49, 0x1EE49, prN}, // Lo ARABIC MATHEMATICAL TAILED YEH + {0x1EE4B, 0x1EE4B, prN}, // Lo ARABIC MATHEMATICAL TAILED LAM + {0x1EE4D, 0x1EE4F, prN}, // Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN + {0x1EE51, 0x1EE52, prN}, // Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF + {0x1EE54, 0x1EE54, prN}, // Lo ARABIC MATHEMATICAL TAILED SHEEN + {0x1EE57, 0x1EE57, prN}, // Lo ARABIC MATHEMATICAL TAILED KHAH + {0x1EE59, 0x1EE59, prN}, // Lo ARABIC MATHEMATICAL TAILED DAD + {0x1EE5B, 0x1EE5B, prN}, // Lo ARABIC MATHEMATICAL TAILED GHAIN + {0x1EE5D, 0x1EE5D, prN}, // Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON + {0x1EE5F, 0x1EE5F, prN}, // Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF + {0x1EE61, 0x1EE62, prN}, // Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM + {0x1EE64, 0x1EE64, prN}, // Lo ARABIC MATHEMATICAL STRETCHED HEH + {0x1EE67, 0x1EE6A, prN}, // Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF + {0x1EE6C, 0x1EE72, prN}, // Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF + {0x1EE74, 0x1EE77, prN}, // Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH + {0x1EE79, 0x1EE7C, prN}, // Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH + {0x1EE7E, 0x1EE7E, prN}, // Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH + {0x1EE80, 0x1EE89, prN}, // Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH + {0x1EE8B, 0x1EE9B, prN}, // Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN + {0x1EEA1, 0x1EEA3, prN}, // Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL + {0x1EEA5, 0x1EEA9, prN}, // Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH + {0x1EEAB, 0x1EEBB, prN}, // Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN + {0x1EEF0, 0x1EEF1, prN}, // Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL + {0x1F000, 0x1F003, prN}, // So [4] MAHJONG TILE EAST WIND..MAHJONG TILE NORTH WIND + {0x1F004, 0x1F004, prW}, // So MAHJONG TILE RED DRAGON + {0x1F005, 0x1F02B, prN}, // So [39] MAHJONG TILE GREEN DRAGON..MAHJONG TILE BACK + {0x1F030, 0x1F093, prN}, // So [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 + {0x1F0A0, 0x1F0AE, prN}, // So [15] PLAYING CARD BACK..PLAYING CARD KING OF SPADES + {0x1F0B1, 0x1F0BF, prN}, // So [15] PLAYING CARD ACE OF HEARTS..PLAYING CARD RED JOKER + {0x1F0C1, 0x1F0CE, prN}, // So [14] PLAYING CARD ACE OF DIAMONDS..PLAYING CARD KING OF DIAMONDS + {0x1F0CF, 0x1F0CF, prW}, // So PLAYING CARD BLACK JOKER + {0x1F0D1, 0x1F0F5, prN}, // So [37] PLAYING CARD ACE OF CLUBS..PLAYING CARD TRUMP-21 + {0x1F100, 0x1F10A, prA}, // No [11] DIGIT ZERO FULL STOP..DIGIT NINE COMMA + {0x1F10B, 0x1F10C, prN}, // No [2] DINGBAT CIRCLED SANS-SERIF DIGIT ZERO..DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ZERO + {0x1F10D, 0x1F10F, prN}, // So [3] CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH + {0x1F110, 0x1F12D, prA}, // So [30] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED CD + {0x1F12E, 0x1F12F, prN}, // So [2] CIRCLED WZ..COPYLEFT SYMBOL + {0x1F130, 0x1F169, prA}, // So [58] SQUARED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z + {0x1F16A, 0x1F16F, prN}, // So [6] RAISED MC SIGN..CIRCLED HUMAN FIGURE + {0x1F170, 0x1F18D, prA}, // So [30] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED SA + {0x1F18E, 0x1F18E, prW}, // So NEGATIVE SQUARED AB + {0x1F18F, 0x1F190, prA}, // So [2] NEGATIVE SQUARED WC..SQUARE DJ + {0x1F191, 0x1F19A, prW}, // So [10] SQUARED CL..SQUARED VS + {0x1F19B, 0x1F1AC, prA}, // So [18] SQUARED THREE D..SQUARED VOD + {0x1F1AD, 0x1F1AD, prN}, // So MASK WORK SYMBOL + {0x1F1E6, 0x1F1FF, prN}, // So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + {0x1F200, 0x1F202, prW}, // So [3] SQUARE HIRAGANA HOKA..SQUARED KATAKANA SA + {0x1F210, 0x1F23B, prW}, // So [44] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-914D + {0x1F240, 0x1F248, prW}, // So [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 + {0x1F250, 0x1F251, prW}, // So [2] CIRCLED IDEOGRAPH ADVANTAGE..CIRCLED IDEOGRAPH ACCEPT + {0x1F260, 0x1F265, prW}, // So [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI + {0x1F300, 0x1F320, prW}, // So [33] CYCLONE..SHOOTING STAR + {0x1F321, 0x1F32C, prN}, // So [12] THERMOMETER..WIND BLOWING FACE + {0x1F32D, 0x1F335, prW}, // So [9] HOT DOG..CACTUS + {0x1F336, 0x1F336, prN}, // So HOT PEPPER + {0x1F337, 0x1F37C, prW}, // So [70] TULIP..BABY BOTTLE + {0x1F37D, 0x1F37D, prN}, // So FORK AND KNIFE WITH PLATE + {0x1F37E, 0x1F393, prW}, // So [22] BOTTLE WITH POPPING CORK..GRADUATION CAP + {0x1F394, 0x1F39F, prN}, // So [12] HEART WITH TIP ON THE LEFT..ADMISSION TICKETS + {0x1F3A0, 0x1F3CA, prW}, // So [43] CAROUSEL HORSE..SWIMMER + {0x1F3CB, 0x1F3CE, prN}, // So [4] WEIGHT LIFTER..RACING CAR + {0x1F3CF, 0x1F3D3, prW}, // So [5] CRICKET BAT AND BALL..TABLE TENNIS PADDLE AND BALL + {0x1F3D4, 0x1F3DF, prN}, // So [12] SNOW CAPPED MOUNTAIN..STADIUM + {0x1F3E0, 0x1F3F0, prW}, // So [17] HOUSE BUILDING..EUROPEAN CASTLE + {0x1F3F1, 0x1F3F3, prN}, // So [3] WHITE PENNANT..WAVING WHITE FLAG + {0x1F3F4, 0x1F3F4, prW}, // So WAVING BLACK FLAG + {0x1F3F5, 0x1F3F7, prN}, // So [3] ROSETTE..LABEL + {0x1F3F8, 0x1F3FA, prW}, // So [3] BADMINTON RACQUET AND SHUTTLECOCK..AMPHORA + {0x1F3FB, 0x1F3FF, prW}, // Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 + {0x1F400, 0x1F43E, prW}, // So [63] RAT..PAW PRINTS + {0x1F43F, 0x1F43F, prN}, // So CHIPMUNK + {0x1F440, 0x1F440, prW}, // So EYES + {0x1F441, 0x1F441, prN}, // So EYE + {0x1F442, 0x1F4FC, prW}, // So [187] EAR..VIDEOCASSETTE + {0x1F4FD, 0x1F4FE, prN}, // So [2] FILM PROJECTOR..PORTABLE STEREO + {0x1F4FF, 0x1F53D, prW}, // So [63] PRAYER BEADS..DOWN-POINTING SMALL RED TRIANGLE + {0x1F53E, 0x1F54A, prN}, // So [13] LOWER RIGHT SHADOWED WHITE CIRCLE..DOVE OF PEACE + {0x1F54B, 0x1F54E, prW}, // So [4] KAABA..MENORAH WITH NINE BRANCHES + {0x1F54F, 0x1F54F, prN}, // So BOWL OF HYGIEIA + {0x1F550, 0x1F567, prW}, // So [24] CLOCK FACE ONE OCLOCK..CLOCK FACE TWELVE-THIRTY + {0x1F568, 0x1F579, prN}, // So [18] RIGHT SPEAKER..JOYSTICK + {0x1F57A, 0x1F57A, prW}, // So MAN DANCING + {0x1F57B, 0x1F594, prN}, // So [26] LEFT HAND TELEPHONE RECEIVER..REVERSED VICTORY HAND + {0x1F595, 0x1F596, prW}, // So [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS + {0x1F597, 0x1F5A3, prN}, // So [13] WHITE DOWN POINTING LEFT HAND INDEX..BLACK DOWN POINTING BACKHAND INDEX + {0x1F5A4, 0x1F5A4, prW}, // So BLACK HEART + {0x1F5A5, 0x1F5FA, prN}, // So [86] DESKTOP COMPUTER..WORLD MAP + {0x1F5FB, 0x1F5FF, prW}, // So [5] MOUNT FUJI..MOYAI + {0x1F600, 0x1F64F, prW}, // So [80] GRINNING FACE..PERSON WITH FOLDED HANDS + {0x1F650, 0x1F67F, prN}, // So [48] NORTH WEST POINTING LEAF..REVERSE CHECKER BOARD + {0x1F680, 0x1F6C5, prW}, // So [70] ROCKET..LEFT LUGGAGE + {0x1F6C6, 0x1F6CB, prN}, // So [6] TRIANGLE WITH ROUNDED CORNERS..COUCH AND LAMP + {0x1F6CC, 0x1F6CC, prW}, // So SLEEPING ACCOMMODATION + {0x1F6CD, 0x1F6CF, prN}, // So [3] SHOPPING BAGS..BED + {0x1F6D0, 0x1F6D2, prW}, // So [3] PLACE OF WORSHIP..SHOPPING TROLLEY + {0x1F6D3, 0x1F6D4, prN}, // So [2] STUPA..PAGODA + {0x1F6D5, 0x1F6D7, prW}, // So [3] HINDU TEMPLE..ELEVATOR + {0x1F6DC, 0x1F6DF, prW}, // So [4] WIRELESS..RING BUOY + {0x1F6E0, 0x1F6EA, prN}, // So [11] HAMMER AND WRENCH..NORTHEAST-POINTING AIRPLANE + {0x1F6EB, 0x1F6EC, prW}, // So [2] AIRPLANE DEPARTURE..AIRPLANE ARRIVING + {0x1F6F0, 0x1F6F3, prN}, // So [4] SATELLITE..PASSENGER SHIP + {0x1F6F4, 0x1F6FC, prW}, // So [9] SCOOTER..ROLLER SKATE + {0x1F700, 0x1F776, prN}, // So [119] ALCHEMICAL SYMBOL FOR QUINTESSENCE..LUNAR ECLIPSE + {0x1F77B, 0x1F77F, prN}, // So [5] HAUMEA..ORCUS + {0x1F780, 0x1F7D9, prN}, // So [90] BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE..NINE POINTED WHITE STAR + {0x1F7E0, 0x1F7EB, prW}, // So [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE + {0x1F7F0, 0x1F7F0, prW}, // So HEAVY EQUALS SIGN + {0x1F800, 0x1F80B, prN}, // So [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD + {0x1F810, 0x1F847, prN}, // So [56] LEFTWARDS ARROW WITH SMALL EQUILATERAL ARROWHEAD..DOWNWARDS HEAVY ARROW + {0x1F850, 0x1F859, prN}, // So [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW + {0x1F860, 0x1F887, prN}, // So [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW + {0x1F890, 0x1F8AD, prN}, // So [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS + {0x1F8B0, 0x1F8B1, prN}, // So [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST + {0x1F900, 0x1F90B, prN}, // So [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT + {0x1F90C, 0x1F93A, prW}, // So [47] PINCHED FINGERS..FENCER + {0x1F93B, 0x1F93B, prN}, // So MODERN PENTATHLON + {0x1F93C, 0x1F945, prW}, // So [10] WRESTLERS..GOAL NET + {0x1F946, 0x1F946, prN}, // So RIFLE + {0x1F947, 0x1F9FF, prW}, // So [185] FIRST PLACE MEDAL..NAZAR AMULET + {0x1FA00, 0x1FA53, prN}, // So [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP + {0x1FA60, 0x1FA6D, prN}, // So [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER + {0x1FA70, 0x1FA7C, prW}, // So [13] BALLET SHOES..CRUTCH + {0x1FA80, 0x1FA88, prW}, // So [9] YO-YO..FLUTE + {0x1FA90, 0x1FABD, prW}, // So [46] RINGED PLANET..WING + {0x1FABF, 0x1FAC5, prW}, // So [7] GOOSE..PERSON WITH CROWN + {0x1FACE, 0x1FADB, prW}, // So [14] MOOSE..PEA POD + {0x1FAE0, 0x1FAE8, prW}, // So [9] MELTING FACE..SHAKING FACE + {0x1FAF0, 0x1FAF8, prW}, // So [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND + {0x1FB00, 0x1FB92, prN}, // So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK + {0x1FB94, 0x1FBCA, prN}, // So [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON + {0x1FBF0, 0x1FBF9, prN}, // Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE + {0x20000, 0x2A6DF, prW}, // Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF + {0x2A6E0, 0x2A6FF, prW}, // Cn [32] .. + {0x2A700, 0x2B739, prW}, // Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 + {0x2B73A, 0x2B73F, prW}, // Cn [6] .. + {0x2B740, 0x2B81D, prW}, // Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D + {0x2B81E, 0x2B81F, prW}, // Cn [2] .. + {0x2B820, 0x2CEA1, prW}, // Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 + {0x2CEA2, 0x2CEAF, prW}, // Cn [14] .. + {0x2CEB0, 0x2EBE0, prW}, // Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 + {0x2EBE1, 0x2F7FF, prW}, // Cn [3103] .. + {0x2F800, 0x2FA1D, prW}, // Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + {0x2FA1E, 0x2FA1F, prW}, // Cn [2] .. + {0x2FA20, 0x2FFFD, prW}, // Cn [1502] .. + {0x30000, 0x3134A, prW}, // Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A + {0x3134B, 0x3134F, prW}, // Cn [5] .. + {0x31350, 0x323AF, prW}, // Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF + {0x323B0, 0x3FFFD, prW}, // Cn [56398] .. + {0xE0001, 0xE0001, prN}, // Cf LANGUAGE TAG + {0xE0020, 0xE007F, prN}, // Cf [96] TAG SPACE..CANCEL TAG + {0xE0100, 0xE01EF, prA}, // Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + {0xF0000, 0xFFFFD, prA}, // Co [65534] .. + {0x100000, 0x10FFFD, prA}, // Co [65534] .. +} diff --git a/vendor/github.com/rivo/uniseg/emojipresentation.go b/vendor/github.com/rivo/uniseg/emojipresentation.go new file mode 100644 index 0000000000..9b5f499c4a --- /dev/null +++ b/vendor/github.com/rivo/uniseg/emojipresentation.go @@ -0,0 +1,295 @@ +// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// emojiPresentation are taken from +// +// and +// https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt +// ("Extended_Pictographic" only) +// on September 5, 2023. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var emojiPresentation = [][3]int{ + {0x231A, 0x231B, prEmojiPresentation}, // E0.6 [2] (โŒš..โŒ›) watch..hourglass done + {0x23E9, 0x23EC, prEmojiPresentation}, // E0.6 [4] (โฉ..โฌ) fast-forward button..fast down button + {0x23F0, 0x23F0, prEmojiPresentation}, // E0.6 [1] (โฐ) alarm clock + {0x23F3, 0x23F3, prEmojiPresentation}, // E0.6 [1] (โณ) hourglass not done + {0x25FD, 0x25FE, prEmojiPresentation}, // E0.6 [2] (โ—ฝ..โ—พ) white medium-small square..black medium-small square + {0x2614, 0x2615, prEmojiPresentation}, // E0.6 [2] (โ˜”..โ˜•) umbrella with rain drops..hot beverage + {0x2648, 0x2653, prEmojiPresentation}, // E0.6 [12] (โ™ˆ..โ™“) Aries..Pisces + {0x267F, 0x267F, prEmojiPresentation}, // E0.6 [1] (โ™ฟ) wheelchair symbol + {0x2693, 0x2693, prEmojiPresentation}, // E0.6 [1] (โš“) anchor + {0x26A1, 0x26A1, prEmojiPresentation}, // E0.6 [1] (โšก) high voltage + {0x26AA, 0x26AB, prEmojiPresentation}, // E0.6 [2] (โšช..โšซ) white circle..black circle + {0x26BD, 0x26BE, prEmojiPresentation}, // E0.6 [2] (โšฝ..โšพ) soccer ball..baseball + {0x26C4, 0x26C5, prEmojiPresentation}, // E0.6 [2] (โ›„..โ›…) snowman without snow..sun behind cloud + {0x26CE, 0x26CE, prEmojiPresentation}, // E0.6 [1] (โ›Ž) Ophiuchus + {0x26D4, 0x26D4, prEmojiPresentation}, // E0.6 [1] (โ›”) no entry + {0x26EA, 0x26EA, prEmojiPresentation}, // E0.6 [1] (โ›ช) church + {0x26F2, 0x26F3, prEmojiPresentation}, // E0.6 [2] (โ›ฒ..โ›ณ) fountain..flag in hole + {0x26F5, 0x26F5, prEmojiPresentation}, // E0.6 [1] (โ›ต) sailboat + {0x26FA, 0x26FA, prEmojiPresentation}, // E0.6 [1] (โ›บ) tent + {0x26FD, 0x26FD, prEmojiPresentation}, // E0.6 [1] (โ›ฝ) fuel pump + {0x2705, 0x2705, prEmojiPresentation}, // E0.6 [1] (โœ…) check mark button + {0x270A, 0x270B, prEmojiPresentation}, // E0.6 [2] (โœŠ..โœ‹) raised fist..raised hand + {0x2728, 0x2728, prEmojiPresentation}, // E0.6 [1] (โœจ) sparkles + {0x274C, 0x274C, prEmojiPresentation}, // E0.6 [1] (โŒ) cross mark + {0x274E, 0x274E, prEmojiPresentation}, // E0.6 [1] (โŽ) cross mark button + {0x2753, 0x2755, prEmojiPresentation}, // E0.6 [3] (โ“..โ•) red question mark..white exclamation mark + {0x2757, 0x2757, prEmojiPresentation}, // E0.6 [1] (โ—) red exclamation mark + {0x2795, 0x2797, prEmojiPresentation}, // E0.6 [3] (โž•..โž—) plus..divide + {0x27B0, 0x27B0, prEmojiPresentation}, // E0.6 [1] (โžฐ) curly loop + {0x27BF, 0x27BF, prEmojiPresentation}, // E1.0 [1] (โžฟ) double curly loop + {0x2B1B, 0x2B1C, prEmojiPresentation}, // E0.6 [2] (โฌ›..โฌœ) black large square..white large square + {0x2B50, 0x2B50, prEmojiPresentation}, // E0.6 [1] (โญ) star + {0x2B55, 0x2B55, prEmojiPresentation}, // E0.6 [1] (โญ•) hollow red circle + {0x1F004, 0x1F004, prEmojiPresentation}, // E0.6 [1] (๐Ÿ€„) mahjong red dragon + {0x1F0CF, 0x1F0CF, prEmojiPresentation}, // E0.6 [1] (๐Ÿƒ) joker + {0x1F18E, 0x1F18E, prEmojiPresentation}, // E0.6 [1] (๐Ÿ†Ž) AB button (blood type) + {0x1F191, 0x1F19A, prEmojiPresentation}, // E0.6 [10] (๐Ÿ†‘..๐Ÿ†š) CL button..VS button + {0x1F1E6, 0x1F1FF, prEmojiPresentation}, // E0.0 [26] (๐Ÿ‡ฆ..๐Ÿ‡ฟ) regional indicator symbol letter a..regional indicator symbol letter z + {0x1F201, 0x1F201, prEmojiPresentation}, // E0.6 [1] (๐Ÿˆ) Japanese โ€œhereโ€ button + {0x1F21A, 0x1F21A, prEmojiPresentation}, // E0.6 [1] (๐Ÿˆš) Japanese โ€œfree of chargeโ€ button + {0x1F22F, 0x1F22F, prEmojiPresentation}, // E0.6 [1] (๐Ÿˆฏ) Japanese โ€œreservedโ€ button + {0x1F232, 0x1F236, prEmojiPresentation}, // E0.6 [5] (๐Ÿˆฒ..๐Ÿˆถ) Japanese โ€œprohibitedโ€ button..Japanese โ€œnot free of chargeโ€ button + {0x1F238, 0x1F23A, prEmojiPresentation}, // E0.6 [3] (๐Ÿˆธ..๐Ÿˆบ) Japanese โ€œapplicationโ€ button..Japanese โ€œopen for businessโ€ button + {0x1F250, 0x1F251, prEmojiPresentation}, // E0.6 [2] (๐Ÿ‰..๐Ÿ‰‘) Japanese โ€œbargainโ€ button..Japanese โ€œacceptableโ€ button + {0x1F300, 0x1F30C, prEmojiPresentation}, // E0.6 [13] (๐ŸŒ€..๐ŸŒŒ) cyclone..milky way + {0x1F30D, 0x1F30E, prEmojiPresentation}, // E0.7 [2] (๐ŸŒ..๐ŸŒŽ) globe showing Europe-Africa..globe showing Americas + {0x1F30F, 0x1F30F, prEmojiPresentation}, // E0.6 [1] (๐ŸŒ) globe showing Asia-Australia + {0x1F310, 0x1F310, prEmojiPresentation}, // E1.0 [1] (๐ŸŒ) globe with meridians + {0x1F311, 0x1F311, prEmojiPresentation}, // E0.6 [1] (๐ŸŒ‘) new moon + {0x1F312, 0x1F312, prEmojiPresentation}, // E1.0 [1] (๐ŸŒ’) waxing crescent moon + {0x1F313, 0x1F315, prEmojiPresentation}, // E0.6 [3] (๐ŸŒ“..๐ŸŒ•) first quarter moon..full moon + {0x1F316, 0x1F318, prEmojiPresentation}, // E1.0 [3] (๐ŸŒ–..๐ŸŒ˜) waning gibbous moon..waning crescent moon + {0x1F319, 0x1F319, prEmojiPresentation}, // E0.6 [1] (๐ŸŒ™) crescent moon + {0x1F31A, 0x1F31A, prEmojiPresentation}, // E1.0 [1] (๐ŸŒš) new moon face + {0x1F31B, 0x1F31B, prEmojiPresentation}, // E0.6 [1] (๐ŸŒ›) first quarter moon face + {0x1F31C, 0x1F31C, prEmojiPresentation}, // E0.7 [1] (๐ŸŒœ) last quarter moon face + {0x1F31D, 0x1F31E, prEmojiPresentation}, // E1.0 [2] (๐ŸŒ..๐ŸŒž) full moon face..sun with face + {0x1F31F, 0x1F320, prEmojiPresentation}, // E0.6 [2] (๐ŸŒŸ..๐ŸŒ ) glowing star..shooting star + {0x1F32D, 0x1F32F, prEmojiPresentation}, // E1.0 [3] (๐ŸŒญ..๐ŸŒฏ) hot dog..burrito + {0x1F330, 0x1F331, prEmojiPresentation}, // E0.6 [2] (๐ŸŒฐ..๐ŸŒฑ) chestnut..seedling + {0x1F332, 0x1F333, prEmojiPresentation}, // E1.0 [2] (๐ŸŒฒ..๐ŸŒณ) evergreen tree..deciduous tree + {0x1F334, 0x1F335, prEmojiPresentation}, // E0.6 [2] (๐ŸŒด..๐ŸŒต) palm tree..cactus + {0x1F337, 0x1F34A, prEmojiPresentation}, // E0.6 [20] (๐ŸŒท..๐ŸŠ) tulip..tangerine + {0x1F34B, 0x1F34B, prEmojiPresentation}, // E1.0 [1] (๐Ÿ‹) lemon + {0x1F34C, 0x1F34F, prEmojiPresentation}, // E0.6 [4] (๐ŸŒ..๐Ÿ) banana..green apple + {0x1F350, 0x1F350, prEmojiPresentation}, // E1.0 [1] (๐Ÿ) pear + {0x1F351, 0x1F37B, prEmojiPresentation}, // E0.6 [43] (๐Ÿ‘..๐Ÿป) peach..clinking beer mugs + {0x1F37C, 0x1F37C, prEmojiPresentation}, // E1.0 [1] (๐Ÿผ) baby bottle + {0x1F37E, 0x1F37F, prEmojiPresentation}, // E1.0 [2] (๐Ÿพ..๐Ÿฟ) bottle with popping cork..popcorn + {0x1F380, 0x1F393, prEmojiPresentation}, // E0.6 [20] (๐ŸŽ€..๐ŸŽ“) ribbon..graduation cap + {0x1F3A0, 0x1F3C4, prEmojiPresentation}, // E0.6 [37] (๐ŸŽ ..๐Ÿ„) carousel horse..person surfing + {0x1F3C5, 0x1F3C5, prEmojiPresentation}, // E1.0 [1] (๐Ÿ…) sports medal + {0x1F3C6, 0x1F3C6, prEmojiPresentation}, // E0.6 [1] (๐Ÿ†) trophy + {0x1F3C7, 0x1F3C7, prEmojiPresentation}, // E1.0 [1] (๐Ÿ‡) horse racing + {0x1F3C8, 0x1F3C8, prEmojiPresentation}, // E0.6 [1] (๐Ÿˆ) american football + {0x1F3C9, 0x1F3C9, prEmojiPresentation}, // E1.0 [1] (๐Ÿ‰) rugby football + {0x1F3CA, 0x1F3CA, prEmojiPresentation}, // E0.6 [1] (๐ŸŠ) person swimming + {0x1F3CF, 0x1F3D3, prEmojiPresentation}, // E1.0 [5] (๐Ÿ..๐Ÿ“) cricket game..ping pong + {0x1F3E0, 0x1F3E3, prEmojiPresentation}, // E0.6 [4] (๐Ÿ ..๐Ÿฃ) house..Japanese post office + {0x1F3E4, 0x1F3E4, prEmojiPresentation}, // E1.0 [1] (๐Ÿค) post office + {0x1F3E5, 0x1F3F0, prEmojiPresentation}, // E0.6 [12] (๐Ÿฅ..๐Ÿฐ) hospital..castle + {0x1F3F4, 0x1F3F4, prEmojiPresentation}, // E1.0 [1] (๐Ÿด) black flag + {0x1F3F8, 0x1F407, prEmojiPresentation}, // E1.0 [16] (๐Ÿธ..๐Ÿ‡) badminton..rabbit + {0x1F408, 0x1F408, prEmojiPresentation}, // E0.7 [1] (๐Ÿˆ) cat + {0x1F409, 0x1F40B, prEmojiPresentation}, // E1.0 [3] (๐Ÿ‰..๐Ÿ‹) dragon..whale + {0x1F40C, 0x1F40E, prEmojiPresentation}, // E0.6 [3] (๐ŸŒ..๐ŸŽ) snail..horse + {0x1F40F, 0x1F410, prEmojiPresentation}, // E1.0 [2] (๐Ÿ..๐Ÿ) ram..goat + {0x1F411, 0x1F412, prEmojiPresentation}, // E0.6 [2] (๐Ÿ‘..๐Ÿ’) ewe..monkey + {0x1F413, 0x1F413, prEmojiPresentation}, // E1.0 [1] (๐Ÿ“) rooster + {0x1F414, 0x1F414, prEmojiPresentation}, // E0.6 [1] (๐Ÿ”) chicken + {0x1F415, 0x1F415, prEmojiPresentation}, // E0.7 [1] (๐Ÿ•) dog + {0x1F416, 0x1F416, prEmojiPresentation}, // E1.0 [1] (๐Ÿ–) pig + {0x1F417, 0x1F429, prEmojiPresentation}, // E0.6 [19] (๐Ÿ—..๐Ÿฉ) boar..poodle + {0x1F42A, 0x1F42A, prEmojiPresentation}, // E1.0 [1] (๐Ÿช) camel + {0x1F42B, 0x1F43E, prEmojiPresentation}, // E0.6 [20] (๐Ÿซ..๐Ÿพ) two-hump camel..paw prints + {0x1F440, 0x1F440, prEmojiPresentation}, // E0.6 [1] (๐Ÿ‘€) eyes + {0x1F442, 0x1F464, prEmojiPresentation}, // E0.6 [35] (๐Ÿ‘‚..๐Ÿ‘ค) ear..bust in silhouette + {0x1F465, 0x1F465, prEmojiPresentation}, // E1.0 [1] (๐Ÿ‘ฅ) busts in silhouette + {0x1F466, 0x1F46B, prEmojiPresentation}, // E0.6 [6] (๐Ÿ‘ฆ..๐Ÿ‘ซ) boy..woman and man holding hands + {0x1F46C, 0x1F46D, prEmojiPresentation}, // E1.0 [2] (๐Ÿ‘ฌ..๐Ÿ‘ญ) men holding hands..women holding hands + {0x1F46E, 0x1F4AC, prEmojiPresentation}, // E0.6 [63] (๐Ÿ‘ฎ..๐Ÿ’ฌ) police officer..speech balloon + {0x1F4AD, 0x1F4AD, prEmojiPresentation}, // E1.0 [1] (๐Ÿ’ญ) thought balloon + {0x1F4AE, 0x1F4B5, prEmojiPresentation}, // E0.6 [8] (๐Ÿ’ฎ..๐Ÿ’ต) white flower..dollar banknote + {0x1F4B6, 0x1F4B7, prEmojiPresentation}, // E1.0 [2] (๐Ÿ’ถ..๐Ÿ’ท) euro banknote..pound banknote + {0x1F4B8, 0x1F4EB, prEmojiPresentation}, // E0.6 [52] (๐Ÿ’ธ..๐Ÿ“ซ) money with wings..closed mailbox with raised flag + {0x1F4EC, 0x1F4ED, prEmojiPresentation}, // E0.7 [2] (๐Ÿ“ฌ..๐Ÿ“ญ) open mailbox with raised flag..open mailbox with lowered flag + {0x1F4EE, 0x1F4EE, prEmojiPresentation}, // E0.6 [1] (๐Ÿ“ฎ) postbox + {0x1F4EF, 0x1F4EF, prEmojiPresentation}, // E1.0 [1] (๐Ÿ“ฏ) postal horn + {0x1F4F0, 0x1F4F4, prEmojiPresentation}, // E0.6 [5] (๐Ÿ“ฐ..๐Ÿ“ด) newspaper..mobile phone off + {0x1F4F5, 0x1F4F5, prEmojiPresentation}, // E1.0 [1] (๐Ÿ“ต) no mobile phones + {0x1F4F6, 0x1F4F7, prEmojiPresentation}, // E0.6 [2] (๐Ÿ“ถ..๐Ÿ“ท) antenna bars..camera + {0x1F4F8, 0x1F4F8, prEmojiPresentation}, // E1.0 [1] (๐Ÿ“ธ) camera with flash + {0x1F4F9, 0x1F4FC, prEmojiPresentation}, // E0.6 [4] (๐Ÿ“น..๐Ÿ“ผ) video camera..videocassette + {0x1F4FF, 0x1F502, prEmojiPresentation}, // E1.0 [4] (๐Ÿ“ฟ..๐Ÿ”‚) prayer beads..repeat single button + {0x1F503, 0x1F503, prEmojiPresentation}, // E0.6 [1] (๐Ÿ”ƒ) clockwise vertical arrows + {0x1F504, 0x1F507, prEmojiPresentation}, // E1.0 [4] (๐Ÿ”„..๐Ÿ”‡) counterclockwise arrows button..muted speaker + {0x1F508, 0x1F508, prEmojiPresentation}, // E0.7 [1] (๐Ÿ”ˆ) speaker low volume + {0x1F509, 0x1F509, prEmojiPresentation}, // E1.0 [1] (๐Ÿ”‰) speaker medium volume + {0x1F50A, 0x1F514, prEmojiPresentation}, // E0.6 [11] (๐Ÿ”Š..๐Ÿ””) speaker high volume..bell + {0x1F515, 0x1F515, prEmojiPresentation}, // E1.0 [1] (๐Ÿ”•) bell with slash + {0x1F516, 0x1F52B, prEmojiPresentation}, // E0.6 [22] (๐Ÿ”–..๐Ÿ”ซ) bookmark..water pistol + {0x1F52C, 0x1F52D, prEmojiPresentation}, // E1.0 [2] (๐Ÿ”ฌ..๐Ÿ”ญ) microscope..telescope + {0x1F52E, 0x1F53D, prEmojiPresentation}, // E0.6 [16] (๐Ÿ”ฎ..๐Ÿ”ฝ) crystal ball..downwards button + {0x1F54B, 0x1F54E, prEmojiPresentation}, // E1.0 [4] (๐Ÿ•‹..๐Ÿ•Ž) kaaba..menorah + {0x1F550, 0x1F55B, prEmojiPresentation}, // E0.6 [12] (๐Ÿ•..๐Ÿ•›) one oโ€™clock..twelve oโ€™clock + {0x1F55C, 0x1F567, prEmojiPresentation}, // E0.7 [12] (๐Ÿ•œ..๐Ÿ•ง) one-thirty..twelve-thirty + {0x1F57A, 0x1F57A, prEmojiPresentation}, // E3.0 [1] (๐Ÿ•บ) man dancing + {0x1F595, 0x1F596, prEmojiPresentation}, // E1.0 [2] (๐Ÿ–•..๐Ÿ––) middle finger..vulcan salute + {0x1F5A4, 0x1F5A4, prEmojiPresentation}, // E3.0 [1] (๐Ÿ–ค) black heart + {0x1F5FB, 0x1F5FF, prEmojiPresentation}, // E0.6 [5] (๐Ÿ—ป..๐Ÿ—ฟ) mount fuji..moai + {0x1F600, 0x1F600, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜€) grinning face + {0x1F601, 0x1F606, prEmojiPresentation}, // E0.6 [6] (๐Ÿ˜..๐Ÿ˜†) beaming face with smiling eyes..grinning squinting face + {0x1F607, 0x1F608, prEmojiPresentation}, // E1.0 [2] (๐Ÿ˜‡..๐Ÿ˜ˆ) smiling face with halo..smiling face with horns + {0x1F609, 0x1F60D, prEmojiPresentation}, // E0.6 [5] (๐Ÿ˜‰..๐Ÿ˜) winking face..smiling face with heart-eyes + {0x1F60E, 0x1F60E, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜Ž) smiling face with sunglasses + {0x1F60F, 0x1F60F, prEmojiPresentation}, // E0.6 [1] (๐Ÿ˜) smirking face + {0x1F610, 0x1F610, prEmojiPresentation}, // E0.7 [1] (๐Ÿ˜) neutral face + {0x1F611, 0x1F611, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜‘) expressionless face + {0x1F612, 0x1F614, prEmojiPresentation}, // E0.6 [3] (๐Ÿ˜’..๐Ÿ˜”) unamused face..pensive face + {0x1F615, 0x1F615, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜•) confused face + {0x1F616, 0x1F616, prEmojiPresentation}, // E0.6 [1] (๐Ÿ˜–) confounded face + {0x1F617, 0x1F617, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜—) kissing face + {0x1F618, 0x1F618, prEmojiPresentation}, // E0.6 [1] (๐Ÿ˜˜) face blowing a kiss + {0x1F619, 0x1F619, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜™) kissing face with smiling eyes + {0x1F61A, 0x1F61A, prEmojiPresentation}, // E0.6 [1] (๐Ÿ˜š) kissing face with closed eyes + {0x1F61B, 0x1F61B, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜›) face with tongue + {0x1F61C, 0x1F61E, prEmojiPresentation}, // E0.6 [3] (๐Ÿ˜œ..๐Ÿ˜ž) winking face with tongue..disappointed face + {0x1F61F, 0x1F61F, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜Ÿ) worried face + {0x1F620, 0x1F625, prEmojiPresentation}, // E0.6 [6] (๐Ÿ˜ ..๐Ÿ˜ฅ) angry face..sad but relieved face + {0x1F626, 0x1F627, prEmojiPresentation}, // E1.0 [2] (๐Ÿ˜ฆ..๐Ÿ˜ง) frowning face with open mouth..anguished face + {0x1F628, 0x1F62B, prEmojiPresentation}, // E0.6 [4] (๐Ÿ˜จ..๐Ÿ˜ซ) fearful face..tired face + {0x1F62C, 0x1F62C, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜ฌ) grimacing face + {0x1F62D, 0x1F62D, prEmojiPresentation}, // E0.6 [1] (๐Ÿ˜ญ) loudly crying face + {0x1F62E, 0x1F62F, prEmojiPresentation}, // E1.0 [2] (๐Ÿ˜ฎ..๐Ÿ˜ฏ) face with open mouth..hushed face + {0x1F630, 0x1F633, prEmojiPresentation}, // E0.6 [4] (๐Ÿ˜ฐ..๐Ÿ˜ณ) anxious face with sweat..flushed face + {0x1F634, 0x1F634, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜ด) sleeping face + {0x1F635, 0x1F635, prEmojiPresentation}, // E0.6 [1] (๐Ÿ˜ต) face with crossed-out eyes + {0x1F636, 0x1F636, prEmojiPresentation}, // E1.0 [1] (๐Ÿ˜ถ) face without mouth + {0x1F637, 0x1F640, prEmojiPresentation}, // E0.6 [10] (๐Ÿ˜ท..๐Ÿ™€) face with medical mask..weary cat + {0x1F641, 0x1F644, prEmojiPresentation}, // E1.0 [4] (๐Ÿ™..๐Ÿ™„) slightly frowning face..face with rolling eyes + {0x1F645, 0x1F64F, prEmojiPresentation}, // E0.6 [11] (๐Ÿ™…..๐Ÿ™) person gesturing NO..folded hands + {0x1F680, 0x1F680, prEmojiPresentation}, // E0.6 [1] (๐Ÿš€) rocket + {0x1F681, 0x1F682, prEmojiPresentation}, // E1.0 [2] (๐Ÿš..๐Ÿš‚) helicopter..locomotive + {0x1F683, 0x1F685, prEmojiPresentation}, // E0.6 [3] (๐Ÿšƒ..๐Ÿš…) railway car..bullet train + {0x1F686, 0x1F686, prEmojiPresentation}, // E1.0 [1] (๐Ÿš†) train + {0x1F687, 0x1F687, prEmojiPresentation}, // E0.6 [1] (๐Ÿš‡) metro + {0x1F688, 0x1F688, prEmojiPresentation}, // E1.0 [1] (๐Ÿšˆ) light rail + {0x1F689, 0x1F689, prEmojiPresentation}, // E0.6 [1] (๐Ÿš‰) station + {0x1F68A, 0x1F68B, prEmojiPresentation}, // E1.0 [2] (๐ŸšŠ..๐Ÿš‹) tram..tram car + {0x1F68C, 0x1F68C, prEmojiPresentation}, // E0.6 [1] (๐ŸšŒ) bus + {0x1F68D, 0x1F68D, prEmojiPresentation}, // E0.7 [1] (๐Ÿš) oncoming bus + {0x1F68E, 0x1F68E, prEmojiPresentation}, // E1.0 [1] (๐ŸšŽ) trolleybus + {0x1F68F, 0x1F68F, prEmojiPresentation}, // E0.6 [1] (๐Ÿš) bus stop + {0x1F690, 0x1F690, prEmojiPresentation}, // E1.0 [1] (๐Ÿš) minibus + {0x1F691, 0x1F693, prEmojiPresentation}, // E0.6 [3] (๐Ÿš‘..๐Ÿš“) ambulance..police car + {0x1F694, 0x1F694, prEmojiPresentation}, // E0.7 [1] (๐Ÿš”) oncoming police car + {0x1F695, 0x1F695, prEmojiPresentation}, // E0.6 [1] (๐Ÿš•) taxi + {0x1F696, 0x1F696, prEmojiPresentation}, // E1.0 [1] (๐Ÿš–) oncoming taxi + {0x1F697, 0x1F697, prEmojiPresentation}, // E0.6 [1] (๐Ÿš—) automobile + {0x1F698, 0x1F698, prEmojiPresentation}, // E0.7 [1] (๐Ÿš˜) oncoming automobile + {0x1F699, 0x1F69A, prEmojiPresentation}, // E0.6 [2] (๐Ÿš™..๐Ÿšš) sport utility vehicle..delivery truck + {0x1F69B, 0x1F6A1, prEmojiPresentation}, // E1.0 [7] (๐Ÿš›..๐Ÿšก) articulated lorry..aerial tramway + {0x1F6A2, 0x1F6A2, prEmojiPresentation}, // E0.6 [1] (๐Ÿšข) ship + {0x1F6A3, 0x1F6A3, prEmojiPresentation}, // E1.0 [1] (๐Ÿšฃ) person rowing boat + {0x1F6A4, 0x1F6A5, prEmojiPresentation}, // E0.6 [2] (๐Ÿšค..๐Ÿšฅ) speedboat..horizontal traffic light + {0x1F6A6, 0x1F6A6, prEmojiPresentation}, // E1.0 [1] (๐Ÿšฆ) vertical traffic light + {0x1F6A7, 0x1F6AD, prEmojiPresentation}, // E0.6 [7] (๐Ÿšง..๐Ÿšญ) construction..no smoking + {0x1F6AE, 0x1F6B1, prEmojiPresentation}, // E1.0 [4] (๐Ÿšฎ..๐Ÿšฑ) litter in bin sign..non-potable water + {0x1F6B2, 0x1F6B2, prEmojiPresentation}, // E0.6 [1] (๐Ÿšฒ) bicycle + {0x1F6B3, 0x1F6B5, prEmojiPresentation}, // E1.0 [3] (๐Ÿšณ..๐Ÿšต) no bicycles..person mountain biking + {0x1F6B6, 0x1F6B6, prEmojiPresentation}, // E0.6 [1] (๐Ÿšถ) person walking + {0x1F6B7, 0x1F6B8, prEmojiPresentation}, // E1.0 [2] (๐Ÿšท..๐Ÿšธ) no pedestrians..children crossing + {0x1F6B9, 0x1F6BE, prEmojiPresentation}, // E0.6 [6] (๐Ÿšน..๐Ÿšพ) menโ€™s room..water closet + {0x1F6BF, 0x1F6BF, prEmojiPresentation}, // E1.0 [1] (๐Ÿšฟ) shower + {0x1F6C0, 0x1F6C0, prEmojiPresentation}, // E0.6 [1] (๐Ÿ›€) person taking bath + {0x1F6C1, 0x1F6C5, prEmojiPresentation}, // E1.0 [5] (๐Ÿ›..๐Ÿ›…) bathtub..left luggage + {0x1F6CC, 0x1F6CC, prEmojiPresentation}, // E1.0 [1] (๐Ÿ›Œ) person in bed + {0x1F6D0, 0x1F6D0, prEmojiPresentation}, // E1.0 [1] (๐Ÿ›) place of worship + {0x1F6D1, 0x1F6D2, prEmojiPresentation}, // E3.0 [2] (๐Ÿ›‘..๐Ÿ›’) stop sign..shopping cart + {0x1F6D5, 0x1F6D5, prEmojiPresentation}, // E12.0 [1] (๐Ÿ›•) hindu temple + {0x1F6D6, 0x1F6D7, prEmojiPresentation}, // E13.0 [2] (๐Ÿ›–..๐Ÿ›—) hut..elevator + {0x1F6DC, 0x1F6DC, prEmojiPresentation}, // E15.0 [1] (๐Ÿ›œ) wireless + {0x1F6DD, 0x1F6DF, prEmojiPresentation}, // E14.0 [3] (๐Ÿ›..๐Ÿ›Ÿ) playground slide..ring buoy + {0x1F6EB, 0x1F6EC, prEmojiPresentation}, // E1.0 [2] (๐Ÿ›ซ..๐Ÿ›ฌ) airplane departure..airplane arrival + {0x1F6F4, 0x1F6F6, prEmojiPresentation}, // E3.0 [3] (๐Ÿ›ด..๐Ÿ›ถ) kick scooter..canoe + {0x1F6F7, 0x1F6F8, prEmojiPresentation}, // E5.0 [2] (๐Ÿ›ท..๐Ÿ›ธ) sled..flying saucer + {0x1F6F9, 0x1F6F9, prEmojiPresentation}, // E11.0 [1] (๐Ÿ›น) skateboard + {0x1F6FA, 0x1F6FA, prEmojiPresentation}, // E12.0 [1] (๐Ÿ›บ) auto rickshaw + {0x1F6FB, 0x1F6FC, prEmojiPresentation}, // E13.0 [2] (๐Ÿ›ป..๐Ÿ›ผ) pickup truck..roller skate + {0x1F7E0, 0x1F7EB, prEmojiPresentation}, // E12.0 [12] (๐ŸŸ ..๐ŸŸซ) orange circle..brown square + {0x1F7F0, 0x1F7F0, prEmojiPresentation}, // E14.0 [1] (๐ŸŸฐ) heavy equals sign + {0x1F90C, 0x1F90C, prEmojiPresentation}, // E13.0 [1] (๐ŸคŒ) pinched fingers + {0x1F90D, 0x1F90F, prEmojiPresentation}, // E12.0 [3] (๐Ÿค..๐Ÿค) white heart..pinching hand + {0x1F910, 0x1F918, prEmojiPresentation}, // E1.0 [9] (๐Ÿค..๐Ÿค˜) zipper-mouth face..sign of the horns + {0x1F919, 0x1F91E, prEmojiPresentation}, // E3.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers + {0x1F91F, 0x1F91F, prEmojiPresentation}, // E5.0 [1] (๐ŸคŸ) love-you gesture + {0x1F920, 0x1F927, prEmojiPresentation}, // E3.0 [8] (๐Ÿค ..๐Ÿคง) cowboy hat face..sneezing face + {0x1F928, 0x1F92F, prEmojiPresentation}, // E5.0 [8] (๐Ÿคจ..๐Ÿคฏ) face with raised eyebrow..exploding head + {0x1F930, 0x1F930, prEmojiPresentation}, // E3.0 [1] (๐Ÿคฐ) pregnant woman + {0x1F931, 0x1F932, prEmojiPresentation}, // E5.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together + {0x1F933, 0x1F93A, prEmojiPresentation}, // E3.0 [8] (๐Ÿคณ..๐Ÿคบ) selfie..person fencing + {0x1F93C, 0x1F93E, prEmojiPresentation}, // E3.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball + {0x1F93F, 0x1F93F, prEmojiPresentation}, // E12.0 [1] (๐Ÿคฟ) diving mask + {0x1F940, 0x1F945, prEmojiPresentation}, // E3.0 [6] (๐Ÿฅ€..๐Ÿฅ…) wilted flower..goal net + {0x1F947, 0x1F94B, prEmojiPresentation}, // E3.0 [5] (๐Ÿฅ‡..๐Ÿฅ‹) 1st place medal..martial arts uniform + {0x1F94C, 0x1F94C, prEmojiPresentation}, // E5.0 [1] (๐ŸฅŒ) curling stone + {0x1F94D, 0x1F94F, prEmojiPresentation}, // E11.0 [3] (๐Ÿฅ..๐Ÿฅ) lacrosse..flying disc + {0x1F950, 0x1F95E, prEmojiPresentation}, // E3.0 [15] (๐Ÿฅ..๐Ÿฅž) croissant..pancakes + {0x1F95F, 0x1F96B, prEmojiPresentation}, // E5.0 [13] (๐ŸฅŸ..๐Ÿฅซ) dumpling..canned food + {0x1F96C, 0x1F970, prEmojiPresentation}, // E11.0 [5] (๐Ÿฅฌ..๐Ÿฅฐ) leafy green..smiling face with hearts + {0x1F971, 0x1F971, prEmojiPresentation}, // E12.0 [1] (๐Ÿฅฑ) yawning face + {0x1F972, 0x1F972, prEmojiPresentation}, // E13.0 [1] (๐Ÿฅฒ) smiling face with tear + {0x1F973, 0x1F976, prEmojiPresentation}, // E11.0 [4] (๐Ÿฅณ..๐Ÿฅถ) partying face..cold face + {0x1F977, 0x1F978, prEmojiPresentation}, // E13.0 [2] (๐Ÿฅท..๐Ÿฅธ) ninja..disguised face + {0x1F979, 0x1F979, prEmojiPresentation}, // E14.0 [1] (๐Ÿฅน) face holding back tears + {0x1F97A, 0x1F97A, prEmojiPresentation}, // E11.0 [1] (๐Ÿฅบ) pleading face + {0x1F97B, 0x1F97B, prEmojiPresentation}, // E12.0 [1] (๐Ÿฅป) sari + {0x1F97C, 0x1F97F, prEmojiPresentation}, // E11.0 [4] (๐Ÿฅผ..๐Ÿฅฟ) lab coat..flat shoe + {0x1F980, 0x1F984, prEmojiPresentation}, // E1.0 [5] (๐Ÿฆ€..๐Ÿฆ„) crab..unicorn + {0x1F985, 0x1F991, prEmojiPresentation}, // E3.0 [13] (๐Ÿฆ…..๐Ÿฆ‘) eagle..squid + {0x1F992, 0x1F997, prEmojiPresentation}, // E5.0 [6] (๐Ÿฆ’..๐Ÿฆ—) giraffe..cricket + {0x1F998, 0x1F9A2, prEmojiPresentation}, // E11.0 [11] (๐Ÿฆ˜..๐Ÿฆข) kangaroo..swan + {0x1F9A3, 0x1F9A4, prEmojiPresentation}, // E13.0 [2] (๐Ÿฆฃ..๐Ÿฆค) mammoth..dodo + {0x1F9A5, 0x1F9AA, prEmojiPresentation}, // E12.0 [6] (๐Ÿฆฅ..๐Ÿฆช) sloth..oyster + {0x1F9AB, 0x1F9AD, prEmojiPresentation}, // E13.0 [3] (๐Ÿฆซ..๐Ÿฆญ) beaver..seal + {0x1F9AE, 0x1F9AF, prEmojiPresentation}, // E12.0 [2] (๐Ÿฆฎ..๐Ÿฆฏ) guide dog..white cane + {0x1F9B0, 0x1F9B9, prEmojiPresentation}, // E11.0 [10] (๐Ÿฆฐ..๐Ÿฆน) red hair..supervillain + {0x1F9BA, 0x1F9BF, prEmojiPresentation}, // E12.0 [6] (๐Ÿฆบ..๐Ÿฆฟ) safety vest..mechanical leg + {0x1F9C0, 0x1F9C0, prEmojiPresentation}, // E1.0 [1] (๐Ÿง€) cheese wedge + {0x1F9C1, 0x1F9C2, prEmojiPresentation}, // E11.0 [2] (๐Ÿง..๐Ÿง‚) cupcake..salt + {0x1F9C3, 0x1F9CA, prEmojiPresentation}, // E12.0 [8] (๐Ÿงƒ..๐ŸงŠ) beverage box..ice + {0x1F9CB, 0x1F9CB, prEmojiPresentation}, // E13.0 [1] (๐Ÿง‹) bubble tea + {0x1F9CC, 0x1F9CC, prEmojiPresentation}, // E14.0 [1] (๐ŸงŒ) troll + {0x1F9CD, 0x1F9CF, prEmojiPresentation}, // E12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person + {0x1F9D0, 0x1F9E6, prEmojiPresentation}, // E5.0 [23] (๐Ÿง..๐Ÿงฆ) face with monocle..socks + {0x1F9E7, 0x1F9FF, prEmojiPresentation}, // E11.0 [25] (๐Ÿงง..๐Ÿงฟ) red envelope..nazar amulet + {0x1FA70, 0x1FA73, prEmojiPresentation}, // E12.0 [4] (๐Ÿฉฐ..๐Ÿฉณ) ballet shoes..shorts + {0x1FA74, 0x1FA74, prEmojiPresentation}, // E13.0 [1] (๐Ÿฉด) thong sandal + {0x1FA75, 0x1FA77, prEmojiPresentation}, // E15.0 [3] (๐Ÿฉต..๐Ÿฉท) light blue heart..pink heart + {0x1FA78, 0x1FA7A, prEmojiPresentation}, // E12.0 [3] (๐Ÿฉธ..๐Ÿฉบ) drop of blood..stethoscope + {0x1FA7B, 0x1FA7C, prEmojiPresentation}, // E14.0 [2] (๐Ÿฉป..๐Ÿฉผ) x-ray..crutch + {0x1FA80, 0x1FA82, prEmojiPresentation}, // E12.0 [3] (๐Ÿช€..๐Ÿช‚) yo-yo..parachute + {0x1FA83, 0x1FA86, prEmojiPresentation}, // E13.0 [4] (๐Ÿชƒ..๐Ÿช†) boomerang..nesting dolls + {0x1FA87, 0x1FA88, prEmojiPresentation}, // E15.0 [2] (๐Ÿช‡..๐Ÿชˆ) maracas..flute + {0x1FA90, 0x1FA95, prEmojiPresentation}, // E12.0 [6] (๐Ÿช..๐Ÿช•) ringed planet..banjo + {0x1FA96, 0x1FAA8, prEmojiPresentation}, // E13.0 [19] (๐Ÿช–..๐Ÿชจ) military helmet..rock + {0x1FAA9, 0x1FAAC, prEmojiPresentation}, // E14.0 [4] (๐Ÿชฉ..๐Ÿชฌ) mirror ball..hamsa + {0x1FAAD, 0x1FAAF, prEmojiPresentation}, // E15.0 [3] (๐Ÿชญ..๐Ÿชฏ) folding hand fan..khanda + {0x1FAB0, 0x1FAB6, prEmojiPresentation}, // E13.0 [7] (๐Ÿชฐ..๐Ÿชถ) fly..feather + {0x1FAB7, 0x1FABA, prEmojiPresentation}, // E14.0 [4] (๐Ÿชท..๐Ÿชบ) lotus..nest with eggs + {0x1FABB, 0x1FABD, prEmojiPresentation}, // E15.0 [3] (๐Ÿชป..๐Ÿชฝ) hyacinth..wing + {0x1FABF, 0x1FABF, prEmojiPresentation}, // E15.0 [1] (๐Ÿชฟ) goose + {0x1FAC0, 0x1FAC2, prEmojiPresentation}, // E13.0 [3] (๐Ÿซ€..๐Ÿซ‚) anatomical heart..people hugging + {0x1FAC3, 0x1FAC5, prEmojiPresentation}, // E14.0 [3] (๐Ÿซƒ..๐Ÿซ…) pregnant man..person with crown + {0x1FACE, 0x1FACF, prEmojiPresentation}, // E15.0 [2] (๐ŸซŽ..๐Ÿซ) moose..donkey + {0x1FAD0, 0x1FAD6, prEmojiPresentation}, // E13.0 [7] (๐Ÿซ..๐Ÿซ–) blueberries..teapot + {0x1FAD7, 0x1FAD9, prEmojiPresentation}, // E14.0 [3] (๐Ÿซ—..๐Ÿซ™) pouring liquid..jar + {0x1FADA, 0x1FADB, prEmojiPresentation}, // E15.0 [2] (๐Ÿซš..๐Ÿซ›) ginger root..pea pod + {0x1FAE0, 0x1FAE7, prEmojiPresentation}, // E14.0 [8] (๐Ÿซ ..๐Ÿซง) melting face..bubbles + {0x1FAE8, 0x1FAE8, prEmojiPresentation}, // E15.0 [1] (๐Ÿซจ) shaking face + {0x1FAF0, 0x1FAF6, prEmojiPresentation}, // E14.0 [7] (๐Ÿซฐ..๐Ÿซถ) hand with index finger and thumb crossed..heart hands + {0x1FAF7, 0x1FAF8, prEmojiPresentation}, // E15.0 [2] (๐Ÿซท..๐Ÿซธ) leftwards pushing hand..rightwards pushing hand +} diff --git a/vendor/github.com/rivo/uniseg/gen_breaktest.go b/vendor/github.com/rivo/uniseg/gen_breaktest.go new file mode 100644 index 0000000000..6bfbeb5e7f --- /dev/null +++ b/vendor/github.com/rivo/uniseg/gen_breaktest.go @@ -0,0 +1,215 @@ +//go:build generate + +// This program generates a Go containing a slice of test cases based on the +// Unicode Character Database auxiliary data files. The command line arguments +// are as follows: +// +// 1. The name of the Unicode data file (just the filename, without extension). +// 2. The name of the locally generated Go file. +// 3. The name of the slice containing the test cases. +// 4. The name of the generator, for logging purposes. +// +//go:generate go run gen_breaktest.go GraphemeBreakTest graphemebreak_test.go graphemeBreakTestCases graphemes +//go:generate go run gen_breaktest.go WordBreakTest wordbreak_test.go wordBreakTestCases words +//go:generate go run gen_breaktest.go SentenceBreakTest sentencebreak_test.go sentenceBreakTestCases sentences +//go:generate go run gen_breaktest.go LineBreakTest linebreak_test.go lineBreakTestCases lines + +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "go/format" + "io/ioutil" + "log" + "net/http" + "os" + "time" +) + +// We want to test against a specific version rather than the latest. When the +// package is upgraded to a new version, change these to generate new tests. +const ( + testCaseURL = `https://www.unicode.org/Public/15.0.0/ucd/auxiliary/%s.txt` +) + +func main() { + if len(os.Args) < 5 { + fmt.Println("Not enough arguments, see code for details") + os.Exit(1) + } + + log.SetPrefix("gen_breaktest (" + os.Args[4] + "): ") + log.SetFlags(0) + + // Read text of testcases and parse into Go source code. + src, err := parse(fmt.Sprintf(testCaseURL, os.Args[1])) + if err != nil { + log.Fatal(err) + } + + // Format the Go code. + formatted, err := format.Source(src) + if err != nil { + log.Fatalln("gofmt:", err) + } + + // Write it out. + log.Print("Writing to ", os.Args[2]) + if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil { + log.Fatal(err) + } +} + +// parse reads a break text file, either from a local file or from a URL. It +// parses the file data into Go source code representing the test cases. +func parse(url string) ([]byte, error) { + log.Printf("Parsing %s", url) + res, err := http.Get(url) + if err != nil { + return nil, err + } + body := res.Body + defer body.Close() + + buf := new(bytes.Buffer) + buf.Grow(120 << 10) + buf.WriteString(`// Code generated via go generate from gen_breaktest.go. DO NOT EDIT. + +package uniseg + +// ` + os.Args[3] + ` are Grapheme testcases taken from +// ` + url + ` +// on ` + time.Now().Format("January 2, 2006") + `. See +// https://www.unicode.org/license.html for the Unicode license agreement. +var ` + os.Args[3] + ` = []testCase { +`) + + sc := bufio.NewScanner(body) + num := 1 + var line []byte + original := make([]byte, 0, 64) + expected := make([]byte, 0, 64) + for sc.Scan() { + num++ + line = sc.Bytes() + if len(line) == 0 || line[0] == '#' { + continue + } + var comment []byte + if i := bytes.IndexByte(line, '#'); i >= 0 { + comment = bytes.TrimSpace(line[i+1:]) + line = bytes.TrimSpace(line[:i]) + } + original, expected, err := parseRuneSequence(line, original[:0], expected[:0]) + if err != nil { + return nil, fmt.Errorf(`line %d: %v: %q`, num, err, line) + } + fmt.Fprintf(buf, "\t{original: \"%s\", expected: %s}, // %s\n", original, expected, comment) + } + if err := sc.Err(); err != nil { + return nil, err + } + + // Check for final "# EOF", useful check if we're streaming via HTTP + if !bytes.Equal(line, []byte("# EOF")) { + return nil, fmt.Errorf(`line %d: exected "# EOF" as final line, got %q`, num, line) + } + buf.WriteString("}\n") + return buf.Bytes(), nil +} + +// Used by parseRuneSequence to match input via bytes.HasPrefix. +var ( + prefixBreak = []byte("รท ") + prefixDontBreak = []byte("ร— ") + breakOk = []byte("รท") + breakNo = []byte("ร—") +) + +// parseRuneSequence parses a rune + breaking opportunity sequence from b +// and appends the Go code for testcase.original to orig +// and appends the Go code for testcase.expected to exp. +// It retuns the new orig and exp slices. +// +// E.g.ย for the inputย b="รท 0020 ร— 0308 รท 1F1E6 รท" +// it will append +// +// "\u0020\u0308\U0001F1E6" +// +// and "[][]rune{{0x0020,0x0308},{0x1F1E6},}" +// to orig and exp respectively. +// +// The formatting of exp is expected to be cleaned up by gofmt or format.Source. +// Note we explicitly require the sequence to start with รท and we implicitly +// require it to end with รท. +func parseRuneSequence(b, orig, exp []byte) ([]byte, []byte, error) { + // Check for and remove first รท or ร—. + if !bytes.HasPrefix(b, prefixBreak) && !bytes.HasPrefix(b, prefixDontBreak) { + return nil, nil, errors.New("expected รท or ร— as first character") + } + if bytes.HasPrefix(b, prefixBreak) { + b = b[len(prefixBreak):] + } else { + b = b[len(prefixDontBreak):] + } + + boundary := true + exp = append(exp, "[][]rune{"...) + for len(b) > 0 { + if boundary { + exp = append(exp, '{') + } + exp = append(exp, "0x"...) + // Find end of hex digits. + var i int + for i = 0; i < len(b) && b[i] != ' '; i++ { + if d := b[i]; ('0' <= d || d <= '9') || + ('A' <= d || d <= 'F') || + ('a' <= d || d <= 'f') { + continue + } + return nil, nil, errors.New("bad hex digit") + } + switch i { + case 4: + orig = append(orig, "\\u"...) + case 5: + orig = append(orig, "\\U000"...) + default: + return nil, nil, errors.New("unsupport code point hex length") + } + orig = append(orig, b[:i]...) + exp = append(exp, b[:i]...) + b = b[i:] + + // Check for space between hex and รท or ร—. + if len(b) < 1 || b[0] != ' ' { + return nil, nil, errors.New("bad input") + } + b = b[1:] + + // Check for next boundary. + switch { + case bytes.HasPrefix(b, breakOk): + boundary = true + b = b[len(breakOk):] + case bytes.HasPrefix(b, breakNo): + boundary = false + b = b[len(breakNo):] + default: + return nil, nil, errors.New("missing รท or ร—") + } + if boundary { + exp = append(exp, '}') + } + exp = append(exp, ',') + if len(b) > 0 && b[0] == ' ' { + b = b[1:] + } + } + exp = append(exp, '}') + return orig, exp, nil +} diff --git a/vendor/github.com/rivo/uniseg/gen_properties.go b/vendor/github.com/rivo/uniseg/gen_properties.go new file mode 100644 index 0000000000..8992d2c5f8 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/gen_properties.go @@ -0,0 +1,261 @@ +//go:build generate + +// This program generates a property file in Go file from Unicode Character +// Database auxiliary data files. The command line arguments are as follows: +// +// 1. The name of the Unicode data file (just the filename, without extension). +// Can be "-" (to skip) if the emoji flag is included. +// 2. The name of the locally generated Go file. +// 3. The name of the slice mapping code points to properties. +// 4. The name of the generator, for logging purposes. +// 5. (Optional) Flags, comma-separated. The following flags are available: +// - "emojis=": include the specified emoji properties (e.g. +// "Extended_Pictographic"). +// - "gencat": include general category properties. +// +//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic +//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic +//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences +//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat +//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth +//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "go/format" + "io/ioutil" + "log" + "net/http" + "os" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// We want to test against a specific version rather than the latest. When the +// package is upgraded to a new version, change these to generate new tests. +const ( + propertyURL = `https://www.unicode.org/Public/15.0.0/ucd/%s.txt` + emojiURL = `https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt` +) + +// The regular expression for a line containing a code point range property. +var propertyPattern = regexp.MustCompile(`^([0-9A-F]{4,6})(\.\.([0-9A-F]{4,6}))?\s*;\s*([A-Za-z0-9_]+)\s*#\s(.+)$`) + +func main() { + if len(os.Args) < 5 { + fmt.Println("Not enough arguments, see code for details") + os.Exit(1) + } + + log.SetPrefix("gen_properties (" + os.Args[4] + "): ") + log.SetFlags(0) + + // Parse flags. + flags := make(map[string]string) + if len(os.Args) >= 6 { + for _, flag := range strings.Split(os.Args[5], ",") { + flagFields := strings.Split(flag, "=") + if len(flagFields) == 1 { + flags[flagFields[0]] = "yes" + } else { + flags[flagFields[0]] = flagFields[1] + } + } + } + + // Parse the text file and generate Go source code from it. + _, includeGeneralCategory := flags["gencat"] + var mainURL string + if os.Args[1] != "-" { + mainURL = fmt.Sprintf(propertyURL, os.Args[1]) + } + src, err := parse(mainURL, flags["emojis"], includeGeneralCategory) + if err != nil { + log.Fatal(err) + } + + // Format the Go code. + formatted, err := format.Source([]byte(src)) + if err != nil { + log.Fatal("gofmt:", err) + } + + // Save it to the (local) target file. + log.Print("Writing to ", os.Args[2]) + if err := ioutil.WriteFile(os.Args[2], formatted, 0644); err != nil { + log.Fatal(err) + } +} + +// parse parses the Unicode Properties text files located at the given URLs and +// returns their equivalent Go source code to be used in the uniseg package. If +// "emojiProperty" is not an empty string, emoji code points for that emoji +// property (e.g. "Extended_Pictographic") will be included. In those cases, you +// may pass an empty "propertyURL" to skip parsing the main properties file. If +// "includeGeneralCategory" is true, the Unicode General Category property will +// be extracted from the comments and included in the output. +func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) { + if propertyURL == "" && emojiProperty == "" { + return "", errors.New("no properties to parse") + } + + // Temporary buffer to hold properties. + var properties [][4]string + + // Open the first URL. + if propertyURL != "" { + log.Printf("Parsing %s", propertyURL) + res, err := http.Get(propertyURL) + if err != nil { + return "", err + } + in1 := res.Body + defer in1.Close() + + // Parse it. + scanner := bufio.NewScanner(in1) + num := 0 + for scanner.Scan() { + num++ + line := strings.TrimSpace(scanner.Text()) + + // Skip comments and empty lines. + if strings.HasPrefix(line, "#") || line == "" { + continue + } + + // Everything else must be a code point range, a property and a comment. + from, to, property, comment, err := parseProperty(line) + if err != nil { + return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err) + } + properties = append(properties, [4]string{from, to, property, comment}) + } + if err := scanner.Err(); err != nil { + return "", err + } + } + + // Open the second URL. + if emojiProperty != "" { + log.Printf("Parsing %s", emojiURL) + res, err := http.Get(emojiURL) + if err != nil { + return "", err + } + in2 := res.Body + defer in2.Close() + + // Parse it. + scanner := bufio.NewScanner(in2) + num := 0 + for scanner.Scan() { + num++ + line := scanner.Text() + + // Skip comments, empty lines, and everything not containing + // "Extended_Pictographic". + if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) { + continue + } + + // Everything else must be a code point range, a property and a comment. + from, to, property, comment, err := parseProperty(line) + if err != nil { + return "", fmt.Errorf("emojis line %d: %v", num, err) + } + properties = append(properties, [4]string{from, to, property, comment}) + } + if err := scanner.Err(); err != nil { + return "", err + } + } + + // Avoid overflow during binary search. + if len(properties) >= 1<<31 { + return "", errors.New("too many properties") + } + + // Sort properties. + sort.Slice(properties, func(i, j int) bool { + left, _ := strconv.ParseUint(properties[i][0], 16, 64) + right, _ := strconv.ParseUint(properties[j][0], 16, 64) + return left < right + }) + + // Header. + var ( + buf bytes.Buffer + emojiComment string + ) + columns := 3 + if includeGeneralCategory { + columns = 4 + } + if emojiURL != "" { + emojiComment = ` +// and +// ` + emojiURL + ` +// ("Extended_Pictographic" only)` + } + buf.WriteString(`// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// ` + os.Args[3] + ` are taken from +// ` + propertyURL + emojiComment + ` +// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{ + `) + + // Properties. + for _, prop := range properties { + if includeGeneralCategory { + generalCategory := "gc" + prop[3][:2] + if generalCategory == "gcL&" { + generalCategory = "gcLC" + } + prop[3] = prop[3][3:] + fmt.Fprintf(&buf, "{0x%s,0x%s,%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), generalCategory, prop[3]) + } else { + fmt.Fprintf(&buf, "{0x%s,0x%s,%s}, // %s\n", prop[0], prop[1], translateProperty("pr", prop[2]), prop[3]) + } + } + + // Tail. + buf.WriteString("}") + + return buf.String(), nil +} + +// parseProperty parses a line of the Unicode properties text file containing a +// property for a code point range and returns it along with its comment. +func parseProperty(line string) (from, to, property, comment string, err error) { + fields := propertyPattern.FindStringSubmatch(line) + if fields == nil { + err = errors.New("no property found") + return + } + from = fields[1] + to = fields[3] + if to == "" { + to = from + } + property = fields[4] + comment = fields[5] + return +} + +// translateProperty translates a property name as used in the Unicode data file +// to a variable used in the Go code. +func translateProperty(prefix, property string) string { + return prefix + strings.ReplaceAll(property, "_", "") +} diff --git a/vendor/github.com/rivo/uniseg/grapheme.go b/vendor/github.com/rivo/uniseg/grapheme.go new file mode 100644 index 0000000000..b12403d43c --- /dev/null +++ b/vendor/github.com/rivo/uniseg/grapheme.go @@ -0,0 +1,331 @@ +package uniseg + +import "unicode/utf8" + +// Graphemes implements an iterator over Unicode grapheme clusters, or +// user-perceived characters. While iterating, it also provides information +// about word boundaries, sentence boundaries, line breaks, and monospace +// character widths. +// +// After constructing the class via [NewGraphemes] for a given string "str", +// [Graphemes.Next] is called for every grapheme cluster in a loop until it +// returns false. Inside the loop, information about the grapheme cluster as +// well as boundary information and character width is available via the various +// methods (see examples below). +// +// This class basically wraps the [StepString] parser and provides a convenient +// interface to it. If you are only interested in some parts of this package's +// functionality, using the specialized functions starting with "First" is +// almost always faster. +type Graphemes struct { + // The original string. + original string + + // The remaining string to be parsed. + remaining string + + // The current grapheme cluster. + cluster string + + // The byte offset of the current grapheme cluster relative to the original + // string. + offset int + + // The current boundary information of the [Step] parser. + boundaries int + + // The current state of the [Step] parser. + state int +} + +// NewGraphemes returns a new grapheme cluster iterator. +func NewGraphemes(str string) *Graphemes { + return &Graphemes{ + original: str, + remaining: str, + state: -1, + } +} + +// Next advances the iterator by one grapheme cluster and returns false if no +// clusters are left. This function must be called before the first cluster is +// accessed. +func (g *Graphemes) Next() bool { + if len(g.remaining) == 0 { + // We're already past the end. + g.state = -2 + g.cluster = "" + return false + } + g.offset += len(g.cluster) + g.cluster, g.remaining, g.boundaries, g.state = StepString(g.remaining, g.state) + return true +} + +// Runes returns a slice of runes (code points) which corresponds to the current +// grapheme cluster. If the iterator is already past the end or [Graphemes.Next] +// has not yet been called, nil is returned. +func (g *Graphemes) Runes() []rune { + if g.state < 0 { + return nil + } + return []rune(g.cluster) +} + +// Str returns a substring of the original string which corresponds to the +// current grapheme cluster. If the iterator is already past the end or +// [Graphemes.Next] has not yet been called, an empty string is returned. +func (g *Graphemes) Str() string { + return g.cluster +} + +// Bytes returns a byte slice which corresponds to the current grapheme cluster. +// If the iterator is already past the end or [Graphemes.Next] has not yet been +// called, nil is returned. +func (g *Graphemes) Bytes() []byte { + if g.state < 0 { + return nil + } + return []byte(g.cluster) +} + +// Positions returns the interval of the current grapheme cluster as byte +// positions into the original string. The first returned value "from" indexes +// the first byte and the second returned value "to" indexes the first byte that +// is not included anymore, i.e. str[from:to] is the current grapheme cluster of +// the original string "str". If [Graphemes.Next] has not yet been called, both +// values are 0. If the iterator is already past the end, both values are 1. +func (g *Graphemes) Positions() (int, int) { + if g.state == -1 { + return 0, 0 + } else if g.state == -2 { + return 1, 1 + } + return g.offset, g.offset + len(g.cluster) +} + +// IsWordBoundary returns true if a word ends after the current grapheme +// cluster. +func (g *Graphemes) IsWordBoundary() bool { + if g.state < 0 { + return true + } + return g.boundaries&MaskWord != 0 +} + +// IsSentenceBoundary returns true if a sentence ends after the current +// grapheme cluster. +func (g *Graphemes) IsSentenceBoundary() bool { + if g.state < 0 { + return true + } + return g.boundaries&MaskSentence != 0 +} + +// LineBreak returns whether the line can be broken after the current grapheme +// cluster. A value of [LineDontBreak] means the line may not be broken, a value +// of [LineMustBreak] means the line must be broken, and a value of +// [LineCanBreak] means the line may or may not be broken. +func (g *Graphemes) LineBreak() int { + if g.state == -1 { + return LineDontBreak + } + if g.state == -2 { + return LineMustBreak + } + return g.boundaries & MaskLine +} + +// Width returns the monospace width of the current grapheme cluster. +func (g *Graphemes) Width() int { + if g.state < 0 { + return 0 + } + return g.boundaries >> ShiftWidth +} + +// Reset puts the iterator into its initial state such that the next call to +// [Graphemes.Next] sets it to the first grapheme cluster again. +func (g *Graphemes) Reset() { + g.state = -1 + g.offset = 0 + g.cluster = "" + g.remaining = g.original +} + +// GraphemeClusterCount returns the number of user-perceived characters +// (grapheme clusters) for the given string. +func GraphemeClusterCount(s string) (n int) { + state := -1 + for len(s) > 0 { + _, s, _, state = FirstGraphemeClusterInString(s, state) + n++ + } + return +} + +// ReverseString reverses the given string while observing grapheme cluster +// boundaries. +func ReverseString(s string) string { + str := []byte(s) + reversed := make([]byte, len(str)) + state := -1 + index := len(str) + for len(str) > 0 { + var cluster []byte + cluster, str, _, state = FirstGraphemeCluster(str, state) + index -= len(cluster) + copy(reversed[index:], cluster) + if index <= len(str)/2 { + break + } + } + return string(reversed) +} + +// The number of bits the grapheme property must be shifted to make place for +// grapheme states. +const shiftGraphemePropState = 4 + +// FirstGraphemeCluster returns the first grapheme cluster found in the given +// byte slice according to the rules of [Unicode Standard Annex #29, Grapheme +// Cluster Boundaries]. This function can be called continuously to extract all +// grapheme clusters from a byte slice, as illustrated in the example below. +// +// If you don't know the current state, for example when calling the function +// for the first time, you must pass -1. For consecutive calls, pass the state +// and rest slice returned by the previous call. +// +// The "rest" slice is the sub-slice of the original byte slice "b" starting +// after the last byte of the identified grapheme cluster. If the length of the +// "rest" slice is 0, the entire byte slice "b" has been processed. The +// "cluster" byte slice is the sub-slice of the input slice containing the +// identified grapheme cluster. +// +// The returned width is the width of the grapheme cluster for most monospace +// fonts where a value of 1 represents one character cell. +// +// Given an empty byte slice "b", the function returns nil values. +// +// While slightly less convenient than using the Graphemes class, this function +// has much better performance and makes no allocations. It lends itself well to +// large byte slices. +// +// [Unicode Standard Annex #29, Grapheme Cluster Boundaries]: http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries +func FirstGraphemeCluster(b []byte, state int) (cluster, rest []byte, width, newState int) { + // An empty byte slice returns nothing. + if len(b) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRune(b) + if len(b) <= length { // If we're already past the end, there is nothing else to parse. + var prop int + if state < 0 { + prop = propertyGraphemes(r) + } else { + prop = state >> shiftGraphemePropState + } + return b, nil, runeWidth(r, prop), grAny | (prop << shiftGraphemePropState) + } + + // If we don't know the state, determine it now. + var firstProp int + if state < 0 { + state, firstProp, _ = transitionGraphemeState(state, r) + } else { + firstProp = state >> shiftGraphemePropState + } + width += runeWidth(r, firstProp) + + // Transition until we find a boundary. + for { + var ( + prop int + boundary bool + ) + + r, l := utf8.DecodeRune(b[length:]) + state, prop, boundary = transitionGraphemeState(state&maskGraphemeState, r) + + if boundary { + return b[:length], b[length:], width, state | (prop << shiftGraphemePropState) + } + + if firstProp == prExtendedPictographic { + if r == vs15 { + width = 1 + } else if r == vs16 { + width = 2 + } + } else if firstProp != prRegionalIndicator && firstProp != prL { + width += runeWidth(r, prop) + } + + length += l + if len(b) <= length { + return b, nil, width, grAny | (prop << shiftGraphemePropState) + } + } +} + +// FirstGraphemeClusterInString is like [FirstGraphemeCluster] but its input and +// outputs are strings. +func FirstGraphemeClusterInString(str string, state int) (cluster, rest string, width, newState int) { + // An empty string returns nothing. + if len(str) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRuneInString(str) + if len(str) <= length { // If we're already past the end, there is nothing else to parse. + var prop int + if state < 0 { + prop = propertyGraphemes(r) + } else { + prop = state >> shiftGraphemePropState + } + return str, "", runeWidth(r, prop), grAny | (prop << shiftGraphemePropState) + } + + // If we don't know the state, determine it now. + var firstProp int + if state < 0 { + state, firstProp, _ = transitionGraphemeState(state, r) + } else { + firstProp = state >> shiftGraphemePropState + } + width += runeWidth(r, firstProp) + + // Transition until we find a boundary. + for { + var ( + prop int + boundary bool + ) + + r, l := utf8.DecodeRuneInString(str[length:]) + state, prop, boundary = transitionGraphemeState(state&maskGraphemeState, r) + + if boundary { + return str[:length], str[length:], width, state | (prop << shiftGraphemePropState) + } + + if firstProp == prExtendedPictographic { + if r == vs15 { + width = 1 + } else if r == vs16 { + width = 2 + } + } else if firstProp != prRegionalIndicator && firstProp != prL { + width += runeWidth(r, prop) + } + + length += l + if len(str) <= length { + return str, "", width, grAny | (prop << shiftGraphemePropState) + } + } +} diff --git a/vendor/github.com/rivo/uniseg/graphemeproperties.go b/vendor/github.com/rivo/uniseg/graphemeproperties.go new file mode 100644 index 0000000000..0aff4a619a --- /dev/null +++ b/vendor/github.com/rivo/uniseg/graphemeproperties.go @@ -0,0 +1,1915 @@ +// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// graphemeCodePoints are taken from +// https://www.unicode.org/Public/15.0.0/ucd/auxiliary/GraphemeBreakProperty.txt +// and +// https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt +// ("Extended_Pictographic" only) +// on September 5, 2023. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var graphemeCodePoints = [][3]int{ + {0x0000, 0x0009, prControl}, // Cc [10] .. + {0x000A, 0x000A, prLF}, // Cc + {0x000B, 0x000C, prControl}, // Cc [2] .. + {0x000D, 0x000D, prCR}, // Cc + {0x000E, 0x001F, prControl}, // Cc [18] .. + {0x007F, 0x009F, prControl}, // Cc [33] .. + {0x00A9, 0x00A9, prExtendedPictographic}, // E0.6 [1] (ยฉ๏ธ) copyright + {0x00AD, 0x00AD, prControl}, // Cf SOFT HYPHEN + {0x00AE, 0x00AE, prExtendedPictographic}, // E0.6 [1] (ยฎ๏ธ) registered + {0x0300, 0x036F, prExtend}, // Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X + {0x0483, 0x0487, prExtend}, // Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE + {0x0488, 0x0489, prExtend}, // Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN + {0x0591, 0x05BD, prExtend}, // Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG + {0x05BF, 0x05BF, prExtend}, // Mn HEBREW POINT RAFE + {0x05C1, 0x05C2, prExtend}, // Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT + {0x05C4, 0x05C5, prExtend}, // Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT + {0x05C7, 0x05C7, prExtend}, // Mn HEBREW POINT QAMATS QATAN + {0x0600, 0x0605, prPrepend}, // Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE + {0x0610, 0x061A, prExtend}, // Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA + {0x061C, 0x061C, prControl}, // Cf ARABIC LETTER MARK + {0x064B, 0x065F, prExtend}, // Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW + {0x0670, 0x0670, prExtend}, // Mn ARABIC LETTER SUPERSCRIPT ALEF + {0x06D6, 0x06DC, prExtend}, // Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN + {0x06DD, 0x06DD, prPrepend}, // Cf ARABIC END OF AYAH + {0x06DF, 0x06E4, prExtend}, // Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA + {0x06E7, 0x06E8, prExtend}, // Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON + {0x06EA, 0x06ED, prExtend}, // Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM + {0x070F, 0x070F, prPrepend}, // Cf SYRIAC ABBREVIATION MARK + {0x0711, 0x0711, prExtend}, // Mn SYRIAC LETTER SUPERSCRIPT ALAPH + {0x0730, 0x074A, prExtend}, // Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH + {0x07A6, 0x07B0, prExtend}, // Mn [11] THAANA ABAFILI..THAANA SUKUN + {0x07EB, 0x07F3, prExtend}, // Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE + {0x07FD, 0x07FD, prExtend}, // Mn NKO DANTAYALAN + {0x0816, 0x0819, prExtend}, // Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH + {0x081B, 0x0823, prExtend}, // Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A + {0x0825, 0x0827, prExtend}, // Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U + {0x0829, 0x082D, prExtend}, // Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA + {0x0859, 0x085B, prExtend}, // Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK + {0x0890, 0x0891, prPrepend}, // Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE + {0x0898, 0x089F, prExtend}, // Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA + {0x08CA, 0x08E1, prExtend}, // Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA + {0x08E2, 0x08E2, prPrepend}, // Cf ARABIC DISPUTED END OF AYAH + {0x08E3, 0x0902, prExtend}, // Mn [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA + {0x0903, 0x0903, prSpacingMark}, // Mc DEVANAGARI SIGN VISARGA + {0x093A, 0x093A, prExtend}, // Mn DEVANAGARI VOWEL SIGN OE + {0x093B, 0x093B, prSpacingMark}, // Mc DEVANAGARI VOWEL SIGN OOE + {0x093C, 0x093C, prExtend}, // Mn DEVANAGARI SIGN NUKTA + {0x093E, 0x0940, prSpacingMark}, // Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II + {0x0941, 0x0948, prExtend}, // Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI + {0x0949, 0x094C, prSpacingMark}, // Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU + {0x094D, 0x094D, prExtend}, // Mn DEVANAGARI SIGN VIRAMA + {0x094E, 0x094F, prSpacingMark}, // Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW + {0x0951, 0x0957, prExtend}, // Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE + {0x0962, 0x0963, prExtend}, // Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL + {0x0981, 0x0981, prExtend}, // Mn BENGALI SIGN CANDRABINDU + {0x0982, 0x0983, prSpacingMark}, // Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA + {0x09BC, 0x09BC, prExtend}, // Mn BENGALI SIGN NUKTA + {0x09BE, 0x09BE, prExtend}, // Mc BENGALI VOWEL SIGN AA + {0x09BF, 0x09C0, prSpacingMark}, // Mc [2] BENGALI VOWEL SIGN I..BENGALI VOWEL SIGN II + {0x09C1, 0x09C4, prExtend}, // Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR + {0x09C7, 0x09C8, prSpacingMark}, // Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI + {0x09CB, 0x09CC, prSpacingMark}, // Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU + {0x09CD, 0x09CD, prExtend}, // Mn BENGALI SIGN VIRAMA + {0x09D7, 0x09D7, prExtend}, // Mc BENGALI AU LENGTH MARK + {0x09E2, 0x09E3, prExtend}, // Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL + {0x09FE, 0x09FE, prExtend}, // Mn BENGALI SANDHI MARK + {0x0A01, 0x0A02, prExtend}, // Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI + {0x0A03, 0x0A03, prSpacingMark}, // Mc GURMUKHI SIGN VISARGA + {0x0A3C, 0x0A3C, prExtend}, // Mn GURMUKHI SIGN NUKTA + {0x0A3E, 0x0A40, prSpacingMark}, // Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II + {0x0A41, 0x0A42, prExtend}, // Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU + {0x0A47, 0x0A48, prExtend}, // Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI + {0x0A4B, 0x0A4D, prExtend}, // Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA + {0x0A51, 0x0A51, prExtend}, // Mn GURMUKHI SIGN UDAAT + {0x0A70, 0x0A71, prExtend}, // Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK + {0x0A75, 0x0A75, prExtend}, // Mn GURMUKHI SIGN YAKASH + {0x0A81, 0x0A82, prExtend}, // Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA + {0x0A83, 0x0A83, prSpacingMark}, // Mc GUJARATI SIGN VISARGA + {0x0ABC, 0x0ABC, prExtend}, // Mn GUJARATI SIGN NUKTA + {0x0ABE, 0x0AC0, prSpacingMark}, // Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II + {0x0AC1, 0x0AC5, prExtend}, // Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E + {0x0AC7, 0x0AC8, prExtend}, // Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI + {0x0AC9, 0x0AC9, prSpacingMark}, // Mc GUJARATI VOWEL SIGN CANDRA O + {0x0ACB, 0x0ACC, prSpacingMark}, // Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU + {0x0ACD, 0x0ACD, prExtend}, // Mn GUJARATI SIGN VIRAMA + {0x0AE2, 0x0AE3, prExtend}, // Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL + {0x0AFA, 0x0AFF, prExtend}, // Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE + {0x0B01, 0x0B01, prExtend}, // Mn ORIYA SIGN CANDRABINDU + {0x0B02, 0x0B03, prSpacingMark}, // Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA + {0x0B3C, 0x0B3C, prExtend}, // Mn ORIYA SIGN NUKTA + {0x0B3E, 0x0B3E, prExtend}, // Mc ORIYA VOWEL SIGN AA + {0x0B3F, 0x0B3F, prExtend}, // Mn ORIYA VOWEL SIGN I + {0x0B40, 0x0B40, prSpacingMark}, // Mc ORIYA VOWEL SIGN II + {0x0B41, 0x0B44, prExtend}, // Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR + {0x0B47, 0x0B48, prSpacingMark}, // Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI + {0x0B4B, 0x0B4C, prSpacingMark}, // Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU + {0x0B4D, 0x0B4D, prExtend}, // Mn ORIYA SIGN VIRAMA + {0x0B55, 0x0B56, prExtend}, // Mn [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK + {0x0B57, 0x0B57, prExtend}, // Mc ORIYA AU LENGTH MARK + {0x0B62, 0x0B63, prExtend}, // Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL + {0x0B82, 0x0B82, prExtend}, // Mn TAMIL SIGN ANUSVARA + {0x0BBE, 0x0BBE, prExtend}, // Mc TAMIL VOWEL SIGN AA + {0x0BBF, 0x0BBF, prSpacingMark}, // Mc TAMIL VOWEL SIGN I + {0x0BC0, 0x0BC0, prExtend}, // Mn TAMIL VOWEL SIGN II + {0x0BC1, 0x0BC2, prSpacingMark}, // Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU + {0x0BC6, 0x0BC8, prSpacingMark}, // Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI + {0x0BCA, 0x0BCC, prSpacingMark}, // Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU + {0x0BCD, 0x0BCD, prExtend}, // Mn TAMIL SIGN VIRAMA + {0x0BD7, 0x0BD7, prExtend}, // Mc TAMIL AU LENGTH MARK + {0x0C00, 0x0C00, prExtend}, // Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE + {0x0C01, 0x0C03, prSpacingMark}, // Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA + {0x0C04, 0x0C04, prExtend}, // Mn TELUGU SIGN COMBINING ANUSVARA ABOVE + {0x0C3C, 0x0C3C, prExtend}, // Mn TELUGU SIGN NUKTA + {0x0C3E, 0x0C40, prExtend}, // Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II + {0x0C41, 0x0C44, prSpacingMark}, // Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR + {0x0C46, 0x0C48, prExtend}, // Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI + {0x0C4A, 0x0C4D, prExtend}, // Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA + {0x0C55, 0x0C56, prExtend}, // Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK + {0x0C62, 0x0C63, prExtend}, // Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL + {0x0C81, 0x0C81, prExtend}, // Mn KANNADA SIGN CANDRABINDU + {0x0C82, 0x0C83, prSpacingMark}, // Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA + {0x0CBC, 0x0CBC, prExtend}, // Mn KANNADA SIGN NUKTA + {0x0CBE, 0x0CBE, prSpacingMark}, // Mc KANNADA VOWEL SIGN AA + {0x0CBF, 0x0CBF, prExtend}, // Mn KANNADA VOWEL SIGN I + {0x0CC0, 0x0CC1, prSpacingMark}, // Mc [2] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN U + {0x0CC2, 0x0CC2, prExtend}, // Mc KANNADA VOWEL SIGN UU + {0x0CC3, 0x0CC4, prSpacingMark}, // Mc [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR + {0x0CC6, 0x0CC6, prExtend}, // Mn KANNADA VOWEL SIGN E + {0x0CC7, 0x0CC8, prSpacingMark}, // Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI + {0x0CCA, 0x0CCB, prSpacingMark}, // Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO + {0x0CCC, 0x0CCD, prExtend}, // Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA + {0x0CD5, 0x0CD6, prExtend}, // Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK + {0x0CE2, 0x0CE3, prExtend}, // Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL + {0x0CF3, 0x0CF3, prSpacingMark}, // Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT + {0x0D00, 0x0D01, prExtend}, // Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU + {0x0D02, 0x0D03, prSpacingMark}, // Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA + {0x0D3B, 0x0D3C, prExtend}, // Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA + {0x0D3E, 0x0D3E, prExtend}, // Mc MALAYALAM VOWEL SIGN AA + {0x0D3F, 0x0D40, prSpacingMark}, // Mc [2] MALAYALAM VOWEL SIGN I..MALAYALAM VOWEL SIGN II + {0x0D41, 0x0D44, prExtend}, // Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR + {0x0D46, 0x0D48, prSpacingMark}, // Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI + {0x0D4A, 0x0D4C, prSpacingMark}, // Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU + {0x0D4D, 0x0D4D, prExtend}, // Mn MALAYALAM SIGN VIRAMA + {0x0D4E, 0x0D4E, prPrepend}, // Lo MALAYALAM LETTER DOT REPH + {0x0D57, 0x0D57, prExtend}, // Mc MALAYALAM AU LENGTH MARK + {0x0D62, 0x0D63, prExtend}, // Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL + {0x0D81, 0x0D81, prExtend}, // Mn SINHALA SIGN CANDRABINDU + {0x0D82, 0x0D83, prSpacingMark}, // Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA + {0x0DCA, 0x0DCA, prExtend}, // Mn SINHALA SIGN AL-LAKUNA + {0x0DCF, 0x0DCF, prExtend}, // Mc SINHALA VOWEL SIGN AELA-PILLA + {0x0DD0, 0x0DD1, prSpacingMark}, // Mc [2] SINHALA VOWEL SIGN KETTI AEDA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA + {0x0DD2, 0x0DD4, prExtend}, // Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA + {0x0DD6, 0x0DD6, prExtend}, // Mn SINHALA VOWEL SIGN DIGA PAA-PILLA + {0x0DD8, 0x0DDE, prSpacingMark}, // Mc [7] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA + {0x0DDF, 0x0DDF, prExtend}, // Mc SINHALA VOWEL SIGN GAYANUKITTA + {0x0DF2, 0x0DF3, prSpacingMark}, // Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA + {0x0E31, 0x0E31, prExtend}, // Mn THAI CHARACTER MAI HAN-AKAT + {0x0E33, 0x0E33, prSpacingMark}, // Lo THAI CHARACTER SARA AM + {0x0E34, 0x0E3A, prExtend}, // Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU + {0x0E47, 0x0E4E, prExtend}, // Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN + {0x0EB1, 0x0EB1, prExtend}, // Mn LAO VOWEL SIGN MAI KAN + {0x0EB3, 0x0EB3, prSpacingMark}, // Lo LAO VOWEL SIGN AM + {0x0EB4, 0x0EBC, prExtend}, // Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO + {0x0EC8, 0x0ECE, prExtend}, // Mn [7] LAO TONE MAI EK..LAO YAMAKKAN + {0x0F18, 0x0F19, prExtend}, // Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS + {0x0F35, 0x0F35, prExtend}, // Mn TIBETAN MARK NGAS BZUNG NYI ZLA + {0x0F37, 0x0F37, prExtend}, // Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS + {0x0F39, 0x0F39, prExtend}, // Mn TIBETAN MARK TSA -PHRU + {0x0F3E, 0x0F3F, prSpacingMark}, // Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES + {0x0F71, 0x0F7E, prExtend}, // Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO + {0x0F7F, 0x0F7F, prSpacingMark}, // Mc TIBETAN SIGN RNAM BCAD + {0x0F80, 0x0F84, prExtend}, // Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA + {0x0F86, 0x0F87, prExtend}, // Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS + {0x0F8D, 0x0F97, prExtend}, // Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA + {0x0F99, 0x0FBC, prExtend}, // Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA + {0x0FC6, 0x0FC6, prExtend}, // Mn TIBETAN SYMBOL PADMA GDAN + {0x102D, 0x1030, prExtend}, // Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU + {0x1031, 0x1031, prSpacingMark}, // Mc MYANMAR VOWEL SIGN E + {0x1032, 0x1037, prExtend}, // Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW + {0x1039, 0x103A, prExtend}, // Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT + {0x103B, 0x103C, prSpacingMark}, // Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA + {0x103D, 0x103E, prExtend}, // Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA + {0x1056, 0x1057, prSpacingMark}, // Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR + {0x1058, 0x1059, prExtend}, // Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL + {0x105E, 0x1060, prExtend}, // Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA + {0x1071, 0x1074, prExtend}, // Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE + {0x1082, 0x1082, prExtend}, // Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA + {0x1084, 0x1084, prSpacingMark}, // Mc MYANMAR VOWEL SIGN SHAN E + {0x1085, 0x1086, prExtend}, // Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y + {0x108D, 0x108D, prExtend}, // Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE + {0x109D, 0x109D, prExtend}, // Mn MYANMAR VOWEL SIGN AITON AI + {0x1100, 0x115F, prL}, // Lo [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER + {0x1160, 0x11A7, prV}, // Lo [72] HANGUL JUNGSEONG FILLER..HANGUL JUNGSEONG O-YAE + {0x11A8, 0x11FF, prT}, // Lo [88] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG SSANGNIEUN + {0x135D, 0x135F, prExtend}, // Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK + {0x1712, 0x1714, prExtend}, // Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA + {0x1715, 0x1715, prSpacingMark}, // Mc TAGALOG SIGN PAMUDPOD + {0x1732, 0x1733, prExtend}, // Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U + {0x1734, 0x1734, prSpacingMark}, // Mc HANUNOO SIGN PAMUDPOD + {0x1752, 0x1753, prExtend}, // Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U + {0x1772, 0x1773, prExtend}, // Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U + {0x17B4, 0x17B5, prExtend}, // Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA + {0x17B6, 0x17B6, prSpacingMark}, // Mc KHMER VOWEL SIGN AA + {0x17B7, 0x17BD, prExtend}, // Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA + {0x17BE, 0x17C5, prSpacingMark}, // Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU + {0x17C6, 0x17C6, prExtend}, // Mn KHMER SIGN NIKAHIT + {0x17C7, 0x17C8, prSpacingMark}, // Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU + {0x17C9, 0x17D3, prExtend}, // Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT + {0x17DD, 0x17DD, prExtend}, // Mn KHMER SIGN ATTHACAN + {0x180B, 0x180D, prExtend}, // Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE + {0x180E, 0x180E, prControl}, // Cf MONGOLIAN VOWEL SEPARATOR + {0x180F, 0x180F, prExtend}, // Mn MONGOLIAN FREE VARIATION SELECTOR FOUR + {0x1885, 0x1886, prExtend}, // Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA + {0x18A9, 0x18A9, prExtend}, // Mn MONGOLIAN LETTER ALI GALI DAGALGA + {0x1920, 0x1922, prExtend}, // Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U + {0x1923, 0x1926, prSpacingMark}, // Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU + {0x1927, 0x1928, prExtend}, // Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O + {0x1929, 0x192B, prSpacingMark}, // Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA + {0x1930, 0x1931, prSpacingMark}, // Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA + {0x1932, 0x1932, prExtend}, // Mn LIMBU SMALL LETTER ANUSVARA + {0x1933, 0x1938, prSpacingMark}, // Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA + {0x1939, 0x193B, prExtend}, // Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I + {0x1A17, 0x1A18, prExtend}, // Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U + {0x1A19, 0x1A1A, prSpacingMark}, // Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O + {0x1A1B, 0x1A1B, prExtend}, // Mn BUGINESE VOWEL SIGN AE + {0x1A55, 0x1A55, prSpacingMark}, // Mc TAI THAM CONSONANT SIGN MEDIAL RA + {0x1A56, 0x1A56, prExtend}, // Mn TAI THAM CONSONANT SIGN MEDIAL LA + {0x1A57, 0x1A57, prSpacingMark}, // Mc TAI THAM CONSONANT SIGN LA TANG LAI + {0x1A58, 0x1A5E, prExtend}, // Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA + {0x1A60, 0x1A60, prExtend}, // Mn TAI THAM SIGN SAKOT + {0x1A62, 0x1A62, prExtend}, // Mn TAI THAM VOWEL SIGN MAI SAT + {0x1A65, 0x1A6C, prExtend}, // Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW + {0x1A6D, 0x1A72, prSpacingMark}, // Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI + {0x1A73, 0x1A7C, prExtend}, // Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN + {0x1A7F, 0x1A7F, prExtend}, // Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT + {0x1AB0, 0x1ABD, prExtend}, // Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW + {0x1ABE, 0x1ABE, prExtend}, // Me COMBINING PARENTHESES OVERLAY + {0x1ABF, 0x1ACE, prExtend}, // Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T + {0x1B00, 0x1B03, prExtend}, // Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG + {0x1B04, 0x1B04, prSpacingMark}, // Mc BALINESE SIGN BISAH + {0x1B34, 0x1B34, prExtend}, // Mn BALINESE SIGN REREKAN + {0x1B35, 0x1B35, prExtend}, // Mc BALINESE VOWEL SIGN TEDUNG + {0x1B36, 0x1B3A, prExtend}, // Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA + {0x1B3B, 0x1B3B, prSpacingMark}, // Mc BALINESE VOWEL SIGN RA REPA TEDUNG + {0x1B3C, 0x1B3C, prExtend}, // Mn BALINESE VOWEL SIGN LA LENGA + {0x1B3D, 0x1B41, prSpacingMark}, // Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG + {0x1B42, 0x1B42, prExtend}, // Mn BALINESE VOWEL SIGN PEPET + {0x1B43, 0x1B44, prSpacingMark}, // Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG + {0x1B6B, 0x1B73, prExtend}, // Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG + {0x1B80, 0x1B81, prExtend}, // Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR + {0x1B82, 0x1B82, prSpacingMark}, // Mc SUNDANESE SIGN PANGWISAD + {0x1BA1, 0x1BA1, prSpacingMark}, // Mc SUNDANESE CONSONANT SIGN PAMINGKAL + {0x1BA2, 0x1BA5, prExtend}, // Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU + {0x1BA6, 0x1BA7, prSpacingMark}, // Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG + {0x1BA8, 0x1BA9, prExtend}, // Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG + {0x1BAA, 0x1BAA, prSpacingMark}, // Mc SUNDANESE SIGN PAMAAEH + {0x1BAB, 0x1BAD, prExtend}, // Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA + {0x1BE6, 0x1BE6, prExtend}, // Mn BATAK SIGN TOMPI + {0x1BE7, 0x1BE7, prSpacingMark}, // Mc BATAK VOWEL SIGN E + {0x1BE8, 0x1BE9, prExtend}, // Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE + {0x1BEA, 0x1BEC, prSpacingMark}, // Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O + {0x1BED, 0x1BED, prExtend}, // Mn BATAK VOWEL SIGN KARO O + {0x1BEE, 0x1BEE, prSpacingMark}, // Mc BATAK VOWEL SIGN U + {0x1BEF, 0x1BF1, prExtend}, // Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H + {0x1BF2, 0x1BF3, prSpacingMark}, // Mc [2] BATAK PANGOLAT..BATAK PANONGONAN + {0x1C24, 0x1C2B, prSpacingMark}, // Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU + {0x1C2C, 0x1C33, prExtend}, // Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T + {0x1C34, 0x1C35, prSpacingMark}, // Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG + {0x1C36, 0x1C37, prExtend}, // Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA + {0x1CD0, 0x1CD2, prExtend}, // Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA + {0x1CD4, 0x1CE0, prExtend}, // Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA + {0x1CE1, 0x1CE1, prSpacingMark}, // Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA + {0x1CE2, 0x1CE8, prExtend}, // Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL + {0x1CED, 0x1CED, prExtend}, // Mn VEDIC SIGN TIRYAK + {0x1CF4, 0x1CF4, prExtend}, // Mn VEDIC TONE CANDRA ABOVE + {0x1CF7, 0x1CF7, prSpacingMark}, // Mc VEDIC SIGN ATIKRAMA + {0x1CF8, 0x1CF9, prExtend}, // Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE + {0x1DC0, 0x1DFF, prExtend}, // Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW + {0x200B, 0x200B, prControl}, // Cf ZERO WIDTH SPACE + {0x200C, 0x200C, prExtend}, // Cf ZERO WIDTH NON-JOINER + {0x200D, 0x200D, prZWJ}, // Cf ZERO WIDTH JOINER + {0x200E, 0x200F, prControl}, // Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK + {0x2028, 0x2028, prControl}, // Zl LINE SEPARATOR + {0x2029, 0x2029, prControl}, // Zp PARAGRAPH SEPARATOR + {0x202A, 0x202E, prControl}, // Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE + {0x203C, 0x203C, prExtendedPictographic}, // E0.6 [1] (โ€ผ๏ธ) double exclamation mark + {0x2049, 0x2049, prExtendedPictographic}, // E0.6 [1] (โ‰๏ธ) exclamation question mark + {0x2060, 0x2064, prControl}, // Cf [5] WORD JOINER..INVISIBLE PLUS + {0x2065, 0x2065, prControl}, // Cn + {0x2066, 0x206F, prControl}, // Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES + {0x20D0, 0x20DC, prExtend}, // Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE + {0x20DD, 0x20E0, prExtend}, // Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH + {0x20E1, 0x20E1, prExtend}, // Mn COMBINING LEFT RIGHT ARROW ABOVE + {0x20E2, 0x20E4, prExtend}, // Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE + {0x20E5, 0x20F0, prExtend}, // Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE + {0x2122, 0x2122, prExtendedPictographic}, // E0.6 [1] (โ„ข๏ธ) trade mark + {0x2139, 0x2139, prExtendedPictographic}, // E0.6 [1] (โ„น๏ธ) information + {0x2194, 0x2199, prExtendedPictographic}, // E0.6 [6] (โ†”๏ธ..โ†™๏ธ) left-right arrow..down-left arrow + {0x21A9, 0x21AA, prExtendedPictographic}, // E0.6 [2] (โ†ฉ๏ธ..โ†ช๏ธ) right arrow curving left..left arrow curving right + {0x231A, 0x231B, prExtendedPictographic}, // E0.6 [2] (โŒš..โŒ›) watch..hourglass done + {0x2328, 0x2328, prExtendedPictographic}, // E1.0 [1] (โŒจ๏ธ) keyboard + {0x2388, 0x2388, prExtendedPictographic}, // E0.0 [1] (โŽˆ) HELM SYMBOL + {0x23CF, 0x23CF, prExtendedPictographic}, // E1.0 [1] (โ๏ธ) eject button + {0x23E9, 0x23EC, prExtendedPictographic}, // E0.6 [4] (โฉ..โฌ) fast-forward button..fast down button + {0x23ED, 0x23EE, prExtendedPictographic}, // E0.7 [2] (โญ๏ธ..โฎ๏ธ) next track button..last track button + {0x23EF, 0x23EF, prExtendedPictographic}, // E1.0 [1] (โฏ๏ธ) play or pause button + {0x23F0, 0x23F0, prExtendedPictographic}, // E0.6 [1] (โฐ) alarm clock + {0x23F1, 0x23F2, prExtendedPictographic}, // E1.0 [2] (โฑ๏ธ..โฒ๏ธ) stopwatch..timer clock + {0x23F3, 0x23F3, prExtendedPictographic}, // E0.6 [1] (โณ) hourglass not done + {0x23F8, 0x23FA, prExtendedPictographic}, // E0.7 [3] (โธ๏ธ..โบ๏ธ) pause button..record button + {0x24C2, 0x24C2, prExtendedPictographic}, // E0.6 [1] (โ“‚๏ธ) circled M + {0x25AA, 0x25AB, prExtendedPictographic}, // E0.6 [2] (โ–ช๏ธ..โ–ซ๏ธ) black small square..white small square + {0x25B6, 0x25B6, prExtendedPictographic}, // E0.6 [1] (โ–ถ๏ธ) play button + {0x25C0, 0x25C0, prExtendedPictographic}, // E0.6 [1] (โ—€๏ธ) reverse button + {0x25FB, 0x25FE, prExtendedPictographic}, // E0.6 [4] (โ—ป๏ธ..โ—พ) white medium square..black medium-small square + {0x2600, 0x2601, prExtendedPictographic}, // E0.6 [2] (โ˜€๏ธ..โ˜๏ธ) sun..cloud + {0x2602, 0x2603, prExtendedPictographic}, // E0.7 [2] (โ˜‚๏ธ..โ˜ƒ๏ธ) umbrella..snowman + {0x2604, 0x2604, prExtendedPictographic}, // E1.0 [1] (โ˜„๏ธ) comet + {0x2605, 0x2605, prExtendedPictographic}, // E0.0 [1] (โ˜…) BLACK STAR + {0x2607, 0x260D, prExtendedPictographic}, // E0.0 [7] (โ˜‡..โ˜) LIGHTNING..OPPOSITION + {0x260E, 0x260E, prExtendedPictographic}, // E0.6 [1] (โ˜Ž๏ธ) telephone + {0x260F, 0x2610, prExtendedPictographic}, // E0.0 [2] (โ˜..โ˜) WHITE TELEPHONE..BALLOT BOX + {0x2611, 0x2611, prExtendedPictographic}, // E0.6 [1] (โ˜‘๏ธ) check box with check + {0x2612, 0x2612, prExtendedPictographic}, // E0.0 [1] (โ˜’) BALLOT BOX WITH X + {0x2614, 0x2615, prExtendedPictographic}, // E0.6 [2] (โ˜”..โ˜•) umbrella with rain drops..hot beverage + {0x2616, 0x2617, prExtendedPictographic}, // E0.0 [2] (โ˜–..โ˜—) WHITE SHOGI PIECE..BLACK SHOGI PIECE + {0x2618, 0x2618, prExtendedPictographic}, // E1.0 [1] (โ˜˜๏ธ) shamrock + {0x2619, 0x261C, prExtendedPictographic}, // E0.0 [4] (โ˜™..โ˜œ) REVERSED ROTATED FLORAL HEART BULLET..WHITE LEFT POINTING INDEX + {0x261D, 0x261D, prExtendedPictographic}, // E0.6 [1] (โ˜๏ธ) index pointing up + {0x261E, 0x261F, prExtendedPictographic}, // E0.0 [2] (โ˜ž..โ˜Ÿ) WHITE RIGHT POINTING INDEX..WHITE DOWN POINTING INDEX + {0x2620, 0x2620, prExtendedPictographic}, // E1.0 [1] (โ˜ ๏ธ) skull and crossbones + {0x2621, 0x2621, prExtendedPictographic}, // E0.0 [1] (โ˜ก) CAUTION SIGN + {0x2622, 0x2623, prExtendedPictographic}, // E1.0 [2] (โ˜ข๏ธ..โ˜ฃ๏ธ) radioactive..biohazard + {0x2624, 0x2625, prExtendedPictographic}, // E0.0 [2] (โ˜ค..โ˜ฅ) CADUCEUS..ANKH + {0x2626, 0x2626, prExtendedPictographic}, // E1.0 [1] (โ˜ฆ๏ธ) orthodox cross + {0x2627, 0x2629, prExtendedPictographic}, // E0.0 [3] (โ˜ง..โ˜ฉ) CHI RHO..CROSS OF JERUSALEM + {0x262A, 0x262A, prExtendedPictographic}, // E0.7 [1] (โ˜ช๏ธ) star and crescent + {0x262B, 0x262D, prExtendedPictographic}, // E0.0 [3] (โ˜ซ..โ˜ญ) FARSI SYMBOL..HAMMER AND SICKLE + {0x262E, 0x262E, prExtendedPictographic}, // E1.0 [1] (โ˜ฎ๏ธ) peace symbol + {0x262F, 0x262F, prExtendedPictographic}, // E0.7 [1] (โ˜ฏ๏ธ) yin yang + {0x2630, 0x2637, prExtendedPictographic}, // E0.0 [8] (โ˜ฐ..โ˜ท) TRIGRAM FOR HEAVEN..TRIGRAM FOR EARTH + {0x2638, 0x2639, prExtendedPictographic}, // E0.7 [2] (โ˜ธ๏ธ..โ˜น๏ธ) wheel of dharma..frowning face + {0x263A, 0x263A, prExtendedPictographic}, // E0.6 [1] (โ˜บ๏ธ) smiling face + {0x263B, 0x263F, prExtendedPictographic}, // E0.0 [5] (โ˜ป..โ˜ฟ) BLACK SMILING FACE..MERCURY + {0x2640, 0x2640, prExtendedPictographic}, // E4.0 [1] (โ™€๏ธ) female sign + {0x2641, 0x2641, prExtendedPictographic}, // E0.0 [1] (โ™) EARTH + {0x2642, 0x2642, prExtendedPictographic}, // E4.0 [1] (โ™‚๏ธ) male sign + {0x2643, 0x2647, prExtendedPictographic}, // E0.0 [5] (โ™ƒ..โ™‡) JUPITER..PLUTO + {0x2648, 0x2653, prExtendedPictographic}, // E0.6 [12] (โ™ˆ..โ™“) Aries..Pisces + {0x2654, 0x265E, prExtendedPictographic}, // E0.0 [11] (โ™”..โ™ž) WHITE CHESS KING..BLACK CHESS KNIGHT + {0x265F, 0x265F, prExtendedPictographic}, // E11.0 [1] (โ™Ÿ๏ธ) chess pawn + {0x2660, 0x2660, prExtendedPictographic}, // E0.6 [1] (โ™ ๏ธ) spade suit + {0x2661, 0x2662, prExtendedPictographic}, // E0.0 [2] (โ™ก..โ™ข) WHITE HEART SUIT..WHITE DIAMOND SUIT + {0x2663, 0x2663, prExtendedPictographic}, // E0.6 [1] (โ™ฃ๏ธ) club suit + {0x2664, 0x2664, prExtendedPictographic}, // E0.0 [1] (โ™ค) WHITE SPADE SUIT + {0x2665, 0x2666, prExtendedPictographic}, // E0.6 [2] (โ™ฅ๏ธ..โ™ฆ๏ธ) heart suit..diamond suit + {0x2667, 0x2667, prExtendedPictographic}, // E0.0 [1] (โ™ง) WHITE CLUB SUIT + {0x2668, 0x2668, prExtendedPictographic}, // E0.6 [1] (โ™จ๏ธ) hot springs + {0x2669, 0x267A, prExtendedPictographic}, // E0.0 [18] (โ™ฉ..โ™บ) QUARTER NOTE..RECYCLING SYMBOL FOR GENERIC MATERIALS + {0x267B, 0x267B, prExtendedPictographic}, // E0.6 [1] (โ™ป๏ธ) recycling symbol + {0x267C, 0x267D, prExtendedPictographic}, // E0.0 [2] (โ™ผ..โ™ฝ) RECYCLED PAPER SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL + {0x267E, 0x267E, prExtendedPictographic}, // E11.0 [1] (โ™พ๏ธ) infinity + {0x267F, 0x267F, prExtendedPictographic}, // E0.6 [1] (โ™ฟ) wheelchair symbol + {0x2680, 0x2685, prExtendedPictographic}, // E0.0 [6] (โš€..โš…) DIE FACE-1..DIE FACE-6 + {0x2690, 0x2691, prExtendedPictographic}, // E0.0 [2] (โš..โš‘) WHITE FLAG..BLACK FLAG + {0x2692, 0x2692, prExtendedPictographic}, // E1.0 [1] (โš’๏ธ) hammer and pick + {0x2693, 0x2693, prExtendedPictographic}, // E0.6 [1] (โš“) anchor + {0x2694, 0x2694, prExtendedPictographic}, // E1.0 [1] (โš”๏ธ) crossed swords + {0x2695, 0x2695, prExtendedPictographic}, // E4.0 [1] (โš•๏ธ) medical symbol + {0x2696, 0x2697, prExtendedPictographic}, // E1.0 [2] (โš–๏ธ..โš—๏ธ) balance scale..alembic + {0x2698, 0x2698, prExtendedPictographic}, // E0.0 [1] (โš˜) FLOWER + {0x2699, 0x2699, prExtendedPictographic}, // E1.0 [1] (โš™๏ธ) gear + {0x269A, 0x269A, prExtendedPictographic}, // E0.0 [1] (โšš) STAFF OF HERMES + {0x269B, 0x269C, prExtendedPictographic}, // E1.0 [2] (โš›๏ธ..โšœ๏ธ) atom symbol..fleur-de-lis + {0x269D, 0x269F, prExtendedPictographic}, // E0.0 [3] (โš..โšŸ) OUTLINED WHITE STAR..THREE LINES CONVERGING LEFT + {0x26A0, 0x26A1, prExtendedPictographic}, // E0.6 [2] (โš ๏ธ..โšก) warning..high voltage + {0x26A2, 0x26A6, prExtendedPictographic}, // E0.0 [5] (โšข..โšฆ) DOUBLED FEMALE SIGN..MALE WITH STROKE SIGN + {0x26A7, 0x26A7, prExtendedPictographic}, // E13.0 [1] (โšง๏ธ) transgender symbol + {0x26A8, 0x26A9, prExtendedPictographic}, // E0.0 [2] (โšจ..โšฉ) VERTICAL MALE WITH STROKE SIGN..HORIZONTAL MALE WITH STROKE SIGN + {0x26AA, 0x26AB, prExtendedPictographic}, // E0.6 [2] (โšช..โšซ) white circle..black circle + {0x26AC, 0x26AF, prExtendedPictographic}, // E0.0 [4] (โšฌ..โšฏ) MEDIUM SMALL WHITE CIRCLE..UNMARRIED PARTNERSHIP SYMBOL + {0x26B0, 0x26B1, prExtendedPictographic}, // E1.0 [2] (โšฐ๏ธ..โšฑ๏ธ) coffin..funeral urn + {0x26B2, 0x26BC, prExtendedPictographic}, // E0.0 [11] (โšฒ..โšผ) NEUTER..SESQUIQUADRATE + {0x26BD, 0x26BE, prExtendedPictographic}, // E0.6 [2] (โšฝ..โšพ) soccer ball..baseball + {0x26BF, 0x26C3, prExtendedPictographic}, // E0.0 [5] (โšฟ..โ›ƒ) SQUARED KEY..BLACK DRAUGHTS KING + {0x26C4, 0x26C5, prExtendedPictographic}, // E0.6 [2] (โ›„..โ›…) snowman without snow..sun behind cloud + {0x26C6, 0x26C7, prExtendedPictographic}, // E0.0 [2] (โ›†..โ›‡) RAIN..BLACK SNOWMAN + {0x26C8, 0x26C8, prExtendedPictographic}, // E0.7 [1] (โ›ˆ๏ธ) cloud with lightning and rain + {0x26C9, 0x26CD, prExtendedPictographic}, // E0.0 [5] (โ›‰..โ›) TURNED WHITE SHOGI PIECE..DISABLED CAR + {0x26CE, 0x26CE, prExtendedPictographic}, // E0.6 [1] (โ›Ž) Ophiuchus + {0x26CF, 0x26CF, prExtendedPictographic}, // E0.7 [1] (โ›๏ธ) pick + {0x26D0, 0x26D0, prExtendedPictographic}, // E0.0 [1] (โ›) CAR SLIDING + {0x26D1, 0x26D1, prExtendedPictographic}, // E0.7 [1] (โ›‘๏ธ) rescue workerโ€™s helmet + {0x26D2, 0x26D2, prExtendedPictographic}, // E0.0 [1] (โ›’) CIRCLED CROSSING LANES + {0x26D3, 0x26D3, prExtendedPictographic}, // E0.7 [1] (โ›“๏ธ) chains + {0x26D4, 0x26D4, prExtendedPictographic}, // E0.6 [1] (โ›”) no entry + {0x26D5, 0x26E8, prExtendedPictographic}, // E0.0 [20] (โ›•..โ›จ) ALTERNATE ONE-WAY LEFT WAY TRAFFIC..BLACK CROSS ON SHIELD + {0x26E9, 0x26E9, prExtendedPictographic}, // E0.7 [1] (โ›ฉ๏ธ) shinto shrine + {0x26EA, 0x26EA, prExtendedPictographic}, // E0.6 [1] (โ›ช) church + {0x26EB, 0x26EF, prExtendedPictographic}, // E0.0 [5] (โ›ซ..โ›ฏ) CASTLE..MAP SYMBOL FOR LIGHTHOUSE + {0x26F0, 0x26F1, prExtendedPictographic}, // E0.7 [2] (โ›ฐ๏ธ..โ›ฑ๏ธ) mountain..umbrella on ground + {0x26F2, 0x26F3, prExtendedPictographic}, // E0.6 [2] (โ›ฒ..โ›ณ) fountain..flag in hole + {0x26F4, 0x26F4, prExtendedPictographic}, // E0.7 [1] (โ›ด๏ธ) ferry + {0x26F5, 0x26F5, prExtendedPictographic}, // E0.6 [1] (โ›ต) sailboat + {0x26F6, 0x26F6, prExtendedPictographic}, // E0.0 [1] (โ›ถ) SQUARE FOUR CORNERS + {0x26F7, 0x26F9, prExtendedPictographic}, // E0.7 [3] (โ›ท๏ธ..โ›น๏ธ) skier..person bouncing ball + {0x26FA, 0x26FA, prExtendedPictographic}, // E0.6 [1] (โ›บ) tent + {0x26FB, 0x26FC, prExtendedPictographic}, // E0.0 [2] (โ›ป..โ›ผ) JAPANESE BANK SYMBOL..HEADSTONE GRAVEYARD SYMBOL + {0x26FD, 0x26FD, prExtendedPictographic}, // E0.6 [1] (โ›ฝ) fuel pump + {0x26FE, 0x2701, prExtendedPictographic}, // E0.0 [4] (โ›พ..โœ) CUP ON BLACK SQUARE..UPPER BLADE SCISSORS + {0x2702, 0x2702, prExtendedPictographic}, // E0.6 [1] (โœ‚๏ธ) scissors + {0x2703, 0x2704, prExtendedPictographic}, // E0.0 [2] (โœƒ..โœ„) LOWER BLADE SCISSORS..WHITE SCISSORS + {0x2705, 0x2705, prExtendedPictographic}, // E0.6 [1] (โœ…) check mark button + {0x2708, 0x270C, prExtendedPictographic}, // E0.6 [5] (โœˆ๏ธ..โœŒ๏ธ) airplane..victory hand + {0x270D, 0x270D, prExtendedPictographic}, // E0.7 [1] (โœ๏ธ) writing hand + {0x270E, 0x270E, prExtendedPictographic}, // E0.0 [1] (โœŽ) LOWER RIGHT PENCIL + {0x270F, 0x270F, prExtendedPictographic}, // E0.6 [1] (โœ๏ธ) pencil + {0x2710, 0x2711, prExtendedPictographic}, // E0.0 [2] (โœ..โœ‘) UPPER RIGHT PENCIL..WHITE NIB + {0x2712, 0x2712, prExtendedPictographic}, // E0.6 [1] (โœ’๏ธ) black nib + {0x2714, 0x2714, prExtendedPictographic}, // E0.6 [1] (โœ”๏ธ) check mark + {0x2716, 0x2716, prExtendedPictographic}, // E0.6 [1] (โœ–๏ธ) multiply + {0x271D, 0x271D, prExtendedPictographic}, // E0.7 [1] (โœ๏ธ) latin cross + {0x2721, 0x2721, prExtendedPictographic}, // E0.7 [1] (โœก๏ธ) star of David + {0x2728, 0x2728, prExtendedPictographic}, // E0.6 [1] (โœจ) sparkles + {0x2733, 0x2734, prExtendedPictographic}, // E0.6 [2] (โœณ๏ธ..โœด๏ธ) eight-spoked asterisk..eight-pointed star + {0x2744, 0x2744, prExtendedPictographic}, // E0.6 [1] (โ„๏ธ) snowflake + {0x2747, 0x2747, prExtendedPictographic}, // E0.6 [1] (โ‡๏ธ) sparkle + {0x274C, 0x274C, prExtendedPictographic}, // E0.6 [1] (โŒ) cross mark + {0x274E, 0x274E, prExtendedPictographic}, // E0.6 [1] (โŽ) cross mark button + {0x2753, 0x2755, prExtendedPictographic}, // E0.6 [3] (โ“..โ•) red question mark..white exclamation mark + {0x2757, 0x2757, prExtendedPictographic}, // E0.6 [1] (โ—) red exclamation mark + {0x2763, 0x2763, prExtendedPictographic}, // E1.0 [1] (โฃ๏ธ) heart exclamation + {0x2764, 0x2764, prExtendedPictographic}, // E0.6 [1] (โค๏ธ) red heart + {0x2765, 0x2767, prExtendedPictographic}, // E0.0 [3] (โฅ..โง) ROTATED HEAVY BLACK HEART BULLET..ROTATED FLORAL HEART BULLET + {0x2795, 0x2797, prExtendedPictographic}, // E0.6 [3] (โž•..โž—) plus..divide + {0x27A1, 0x27A1, prExtendedPictographic}, // E0.6 [1] (โžก๏ธ) right arrow + {0x27B0, 0x27B0, prExtendedPictographic}, // E0.6 [1] (โžฐ) curly loop + {0x27BF, 0x27BF, prExtendedPictographic}, // E1.0 [1] (โžฟ) double curly loop + {0x2934, 0x2935, prExtendedPictographic}, // E0.6 [2] (โคด๏ธ..โคต๏ธ) right arrow curving up..right arrow curving down + {0x2B05, 0x2B07, prExtendedPictographic}, // E0.6 [3] (โฌ…๏ธ..โฌ‡๏ธ) left arrow..down arrow + {0x2B1B, 0x2B1C, prExtendedPictographic}, // E0.6 [2] (โฌ›..โฌœ) black large square..white large square + {0x2B50, 0x2B50, prExtendedPictographic}, // E0.6 [1] (โญ) star + {0x2B55, 0x2B55, prExtendedPictographic}, // E0.6 [1] (โญ•) hollow red circle + {0x2CEF, 0x2CF1, prExtend}, // Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS + {0x2D7F, 0x2D7F, prExtend}, // Mn TIFINAGH CONSONANT JOINER + {0x2DE0, 0x2DFF, prExtend}, // Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS + {0x302A, 0x302D, prExtend}, // Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK + {0x302E, 0x302F, prExtend}, // Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK + {0x3030, 0x3030, prExtendedPictographic}, // E0.6 [1] (ใ€ฐ๏ธ) wavy dash + {0x303D, 0x303D, prExtendedPictographic}, // E0.6 [1] (ใ€ฝ๏ธ) part alternation mark + {0x3099, 0x309A, prExtend}, // Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x3297, 0x3297, prExtendedPictographic}, // E0.6 [1] (ใŠ—๏ธ) Japanese โ€œcongratulationsโ€ button + {0x3299, 0x3299, prExtendedPictographic}, // E0.6 [1] (ใŠ™๏ธ) Japanese โ€œsecretโ€ button + {0xA66F, 0xA66F, prExtend}, // Mn COMBINING CYRILLIC VZMET + {0xA670, 0xA672, prExtend}, // Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN + {0xA674, 0xA67D, prExtend}, // Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK + {0xA69E, 0xA69F, prExtend}, // Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E + {0xA6F0, 0xA6F1, prExtend}, // Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS + {0xA802, 0xA802, prExtend}, // Mn SYLOTI NAGRI SIGN DVISVARA + {0xA806, 0xA806, prExtend}, // Mn SYLOTI NAGRI SIGN HASANTA + {0xA80B, 0xA80B, prExtend}, // Mn SYLOTI NAGRI SIGN ANUSVARA + {0xA823, 0xA824, prSpacingMark}, // Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I + {0xA825, 0xA826, prExtend}, // Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E + {0xA827, 0xA827, prSpacingMark}, // Mc SYLOTI NAGRI VOWEL SIGN OO + {0xA82C, 0xA82C, prExtend}, // Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA + {0xA880, 0xA881, prSpacingMark}, // Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA + {0xA8B4, 0xA8C3, prSpacingMark}, // Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU + {0xA8C4, 0xA8C5, prExtend}, // Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU + {0xA8E0, 0xA8F1, prExtend}, // Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA + {0xA8FF, 0xA8FF, prExtend}, // Mn DEVANAGARI VOWEL SIGN AY + {0xA926, 0xA92D, prExtend}, // Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU + {0xA947, 0xA951, prExtend}, // Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R + {0xA952, 0xA953, prSpacingMark}, // Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA + {0xA960, 0xA97C, prL}, // Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH + {0xA980, 0xA982, prExtend}, // Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR + {0xA983, 0xA983, prSpacingMark}, // Mc JAVANESE SIGN WIGNYAN + {0xA9B3, 0xA9B3, prExtend}, // Mn JAVANESE SIGN CECAK TELU + {0xA9B4, 0xA9B5, prSpacingMark}, // Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG + {0xA9B6, 0xA9B9, prExtend}, // Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT + {0xA9BA, 0xA9BB, prSpacingMark}, // Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE + {0xA9BC, 0xA9BD, prExtend}, // Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET + {0xA9BE, 0xA9C0, prSpacingMark}, // Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON + {0xA9E5, 0xA9E5, prExtend}, // Mn MYANMAR SIGN SHAN SAW + {0xAA29, 0xAA2E, prExtend}, // Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE + {0xAA2F, 0xAA30, prSpacingMark}, // Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI + {0xAA31, 0xAA32, prExtend}, // Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE + {0xAA33, 0xAA34, prSpacingMark}, // Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA + {0xAA35, 0xAA36, prExtend}, // Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA + {0xAA43, 0xAA43, prExtend}, // Mn CHAM CONSONANT SIGN FINAL NG + {0xAA4C, 0xAA4C, prExtend}, // Mn CHAM CONSONANT SIGN FINAL M + {0xAA4D, 0xAA4D, prSpacingMark}, // Mc CHAM CONSONANT SIGN FINAL H + {0xAA7C, 0xAA7C, prExtend}, // Mn MYANMAR SIGN TAI LAING TONE-2 + {0xAAB0, 0xAAB0, prExtend}, // Mn TAI VIET MAI KANG + {0xAAB2, 0xAAB4, prExtend}, // Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U + {0xAAB7, 0xAAB8, prExtend}, // Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA + {0xAABE, 0xAABF, prExtend}, // Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK + {0xAAC1, 0xAAC1, prExtend}, // Mn TAI VIET TONE MAI THO + {0xAAEB, 0xAAEB, prSpacingMark}, // Mc MEETEI MAYEK VOWEL SIGN II + {0xAAEC, 0xAAED, prExtend}, // Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI + {0xAAEE, 0xAAEF, prSpacingMark}, // Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU + {0xAAF5, 0xAAF5, prSpacingMark}, // Mc MEETEI MAYEK VOWEL SIGN VISARGA + {0xAAF6, 0xAAF6, prExtend}, // Mn MEETEI MAYEK VIRAMA + {0xABE3, 0xABE4, prSpacingMark}, // Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP + {0xABE5, 0xABE5, prExtend}, // Mn MEETEI MAYEK VOWEL SIGN ANAP + {0xABE6, 0xABE7, prSpacingMark}, // Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP + {0xABE8, 0xABE8, prExtend}, // Mn MEETEI MAYEK VOWEL SIGN UNAP + {0xABE9, 0xABEA, prSpacingMark}, // Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG + {0xABEC, 0xABEC, prSpacingMark}, // Mc MEETEI MAYEK LUM IYEK + {0xABED, 0xABED, prExtend}, // Mn MEETEI MAYEK APUN IYEK + {0xAC00, 0xAC00, prLV}, // Lo HANGUL SYLLABLE GA + {0xAC01, 0xAC1B, prLVT}, // Lo [27] HANGUL SYLLABLE GAG..HANGUL SYLLABLE GAH + {0xAC1C, 0xAC1C, prLV}, // Lo HANGUL SYLLABLE GAE + {0xAC1D, 0xAC37, prLVT}, // Lo [27] HANGUL SYLLABLE GAEG..HANGUL SYLLABLE GAEH + {0xAC38, 0xAC38, prLV}, // Lo HANGUL SYLLABLE GYA + {0xAC39, 0xAC53, prLVT}, // Lo [27] HANGUL SYLLABLE GYAG..HANGUL SYLLABLE GYAH + {0xAC54, 0xAC54, prLV}, // Lo HANGUL SYLLABLE GYAE + {0xAC55, 0xAC6F, prLVT}, // Lo [27] HANGUL SYLLABLE GYAEG..HANGUL SYLLABLE GYAEH + {0xAC70, 0xAC70, prLV}, // Lo HANGUL SYLLABLE GEO + {0xAC71, 0xAC8B, prLVT}, // Lo [27] HANGUL SYLLABLE GEOG..HANGUL SYLLABLE GEOH + {0xAC8C, 0xAC8C, prLV}, // Lo HANGUL SYLLABLE GE + {0xAC8D, 0xACA7, prLVT}, // Lo [27] HANGUL SYLLABLE GEG..HANGUL SYLLABLE GEH + {0xACA8, 0xACA8, prLV}, // Lo HANGUL SYLLABLE GYEO + {0xACA9, 0xACC3, prLVT}, // Lo [27] HANGUL SYLLABLE GYEOG..HANGUL SYLLABLE GYEOH + {0xACC4, 0xACC4, prLV}, // Lo HANGUL SYLLABLE GYE + {0xACC5, 0xACDF, prLVT}, // Lo [27] HANGUL SYLLABLE GYEG..HANGUL SYLLABLE GYEH + {0xACE0, 0xACE0, prLV}, // Lo HANGUL SYLLABLE GO + {0xACE1, 0xACFB, prLVT}, // Lo [27] HANGUL SYLLABLE GOG..HANGUL SYLLABLE GOH + {0xACFC, 0xACFC, prLV}, // Lo HANGUL SYLLABLE GWA + {0xACFD, 0xAD17, prLVT}, // Lo [27] HANGUL SYLLABLE GWAG..HANGUL SYLLABLE GWAH + {0xAD18, 0xAD18, prLV}, // Lo HANGUL SYLLABLE GWAE + {0xAD19, 0xAD33, prLVT}, // Lo [27] HANGUL SYLLABLE GWAEG..HANGUL SYLLABLE GWAEH + {0xAD34, 0xAD34, prLV}, // Lo HANGUL SYLLABLE GOE + {0xAD35, 0xAD4F, prLVT}, // Lo [27] HANGUL SYLLABLE GOEG..HANGUL SYLLABLE GOEH + {0xAD50, 0xAD50, prLV}, // Lo HANGUL SYLLABLE GYO + {0xAD51, 0xAD6B, prLVT}, // Lo [27] HANGUL SYLLABLE GYOG..HANGUL SYLLABLE GYOH + {0xAD6C, 0xAD6C, prLV}, // Lo HANGUL SYLLABLE GU + {0xAD6D, 0xAD87, prLVT}, // Lo [27] HANGUL SYLLABLE GUG..HANGUL SYLLABLE GUH + {0xAD88, 0xAD88, prLV}, // Lo HANGUL SYLLABLE GWEO + {0xAD89, 0xADA3, prLVT}, // Lo [27] HANGUL SYLLABLE GWEOG..HANGUL SYLLABLE GWEOH + {0xADA4, 0xADA4, prLV}, // Lo HANGUL SYLLABLE GWE + {0xADA5, 0xADBF, prLVT}, // Lo [27] HANGUL SYLLABLE GWEG..HANGUL SYLLABLE GWEH + {0xADC0, 0xADC0, prLV}, // Lo HANGUL SYLLABLE GWI + {0xADC1, 0xADDB, prLVT}, // Lo [27] HANGUL SYLLABLE GWIG..HANGUL SYLLABLE GWIH + {0xADDC, 0xADDC, prLV}, // Lo HANGUL SYLLABLE GYU + {0xADDD, 0xADF7, prLVT}, // Lo [27] HANGUL SYLLABLE GYUG..HANGUL SYLLABLE GYUH + {0xADF8, 0xADF8, prLV}, // Lo HANGUL SYLLABLE GEU + {0xADF9, 0xAE13, prLVT}, // Lo [27] HANGUL SYLLABLE GEUG..HANGUL SYLLABLE GEUH + {0xAE14, 0xAE14, prLV}, // Lo HANGUL SYLLABLE GYI + {0xAE15, 0xAE2F, prLVT}, // Lo [27] HANGUL SYLLABLE GYIG..HANGUL SYLLABLE GYIH + {0xAE30, 0xAE30, prLV}, // Lo HANGUL SYLLABLE GI + {0xAE31, 0xAE4B, prLVT}, // Lo [27] HANGUL SYLLABLE GIG..HANGUL SYLLABLE GIH + {0xAE4C, 0xAE4C, prLV}, // Lo HANGUL SYLLABLE GGA + {0xAE4D, 0xAE67, prLVT}, // Lo [27] HANGUL SYLLABLE GGAG..HANGUL SYLLABLE GGAH + {0xAE68, 0xAE68, prLV}, // Lo HANGUL SYLLABLE GGAE + {0xAE69, 0xAE83, prLVT}, // Lo [27] HANGUL SYLLABLE GGAEG..HANGUL SYLLABLE GGAEH + {0xAE84, 0xAE84, prLV}, // Lo HANGUL SYLLABLE GGYA + {0xAE85, 0xAE9F, prLVT}, // Lo [27] HANGUL SYLLABLE GGYAG..HANGUL SYLLABLE GGYAH + {0xAEA0, 0xAEA0, prLV}, // Lo HANGUL SYLLABLE GGYAE + {0xAEA1, 0xAEBB, prLVT}, // Lo [27] HANGUL SYLLABLE GGYAEG..HANGUL SYLLABLE GGYAEH + {0xAEBC, 0xAEBC, prLV}, // Lo HANGUL SYLLABLE GGEO + {0xAEBD, 0xAED7, prLVT}, // Lo [27] HANGUL SYLLABLE GGEOG..HANGUL SYLLABLE GGEOH + {0xAED8, 0xAED8, prLV}, // Lo HANGUL SYLLABLE GGE + {0xAED9, 0xAEF3, prLVT}, // Lo [27] HANGUL SYLLABLE GGEG..HANGUL SYLLABLE GGEH + {0xAEF4, 0xAEF4, prLV}, // Lo HANGUL SYLLABLE GGYEO + {0xAEF5, 0xAF0F, prLVT}, // Lo [27] HANGUL SYLLABLE GGYEOG..HANGUL SYLLABLE GGYEOH + {0xAF10, 0xAF10, prLV}, // Lo HANGUL SYLLABLE GGYE + {0xAF11, 0xAF2B, prLVT}, // Lo [27] HANGUL SYLLABLE GGYEG..HANGUL SYLLABLE GGYEH + {0xAF2C, 0xAF2C, prLV}, // Lo HANGUL SYLLABLE GGO + {0xAF2D, 0xAF47, prLVT}, // Lo [27] HANGUL SYLLABLE GGOG..HANGUL SYLLABLE GGOH + {0xAF48, 0xAF48, prLV}, // Lo HANGUL SYLLABLE GGWA + {0xAF49, 0xAF63, prLVT}, // Lo [27] HANGUL SYLLABLE GGWAG..HANGUL SYLLABLE GGWAH + {0xAF64, 0xAF64, prLV}, // Lo HANGUL SYLLABLE GGWAE + {0xAF65, 0xAF7F, prLVT}, // Lo [27] HANGUL SYLLABLE GGWAEG..HANGUL SYLLABLE GGWAEH + {0xAF80, 0xAF80, prLV}, // Lo HANGUL SYLLABLE GGOE + {0xAF81, 0xAF9B, prLVT}, // Lo [27] HANGUL SYLLABLE GGOEG..HANGUL SYLLABLE GGOEH + {0xAF9C, 0xAF9C, prLV}, // Lo HANGUL SYLLABLE GGYO + {0xAF9D, 0xAFB7, prLVT}, // Lo [27] HANGUL SYLLABLE GGYOG..HANGUL SYLLABLE GGYOH + {0xAFB8, 0xAFB8, prLV}, // Lo HANGUL SYLLABLE GGU + {0xAFB9, 0xAFD3, prLVT}, // Lo [27] HANGUL SYLLABLE GGUG..HANGUL SYLLABLE GGUH + {0xAFD4, 0xAFD4, prLV}, // Lo HANGUL SYLLABLE GGWEO + {0xAFD5, 0xAFEF, prLVT}, // Lo [27] HANGUL SYLLABLE GGWEOG..HANGUL SYLLABLE GGWEOH + {0xAFF0, 0xAFF0, prLV}, // Lo HANGUL SYLLABLE GGWE + {0xAFF1, 0xB00B, prLVT}, // Lo [27] HANGUL SYLLABLE GGWEG..HANGUL SYLLABLE GGWEH + {0xB00C, 0xB00C, prLV}, // Lo HANGUL SYLLABLE GGWI + {0xB00D, 0xB027, prLVT}, // Lo [27] HANGUL SYLLABLE GGWIG..HANGUL SYLLABLE GGWIH + {0xB028, 0xB028, prLV}, // Lo HANGUL SYLLABLE GGYU + {0xB029, 0xB043, prLVT}, // Lo [27] HANGUL SYLLABLE GGYUG..HANGUL SYLLABLE GGYUH + {0xB044, 0xB044, prLV}, // Lo HANGUL SYLLABLE GGEU + {0xB045, 0xB05F, prLVT}, // Lo [27] HANGUL SYLLABLE GGEUG..HANGUL SYLLABLE GGEUH + {0xB060, 0xB060, prLV}, // Lo HANGUL SYLLABLE GGYI + {0xB061, 0xB07B, prLVT}, // Lo [27] HANGUL SYLLABLE GGYIG..HANGUL SYLLABLE GGYIH + {0xB07C, 0xB07C, prLV}, // Lo HANGUL SYLLABLE GGI + {0xB07D, 0xB097, prLVT}, // Lo [27] HANGUL SYLLABLE GGIG..HANGUL SYLLABLE GGIH + {0xB098, 0xB098, prLV}, // Lo HANGUL SYLLABLE NA + {0xB099, 0xB0B3, prLVT}, // Lo [27] HANGUL SYLLABLE NAG..HANGUL SYLLABLE NAH + {0xB0B4, 0xB0B4, prLV}, // Lo HANGUL SYLLABLE NAE + {0xB0B5, 0xB0CF, prLVT}, // Lo [27] HANGUL SYLLABLE NAEG..HANGUL SYLLABLE NAEH + {0xB0D0, 0xB0D0, prLV}, // Lo HANGUL SYLLABLE NYA + {0xB0D1, 0xB0EB, prLVT}, // Lo [27] HANGUL SYLLABLE NYAG..HANGUL SYLLABLE NYAH + {0xB0EC, 0xB0EC, prLV}, // Lo HANGUL SYLLABLE NYAE + {0xB0ED, 0xB107, prLVT}, // Lo [27] HANGUL SYLLABLE NYAEG..HANGUL SYLLABLE NYAEH + {0xB108, 0xB108, prLV}, // Lo HANGUL SYLLABLE NEO + {0xB109, 0xB123, prLVT}, // Lo [27] HANGUL SYLLABLE NEOG..HANGUL SYLLABLE NEOH + {0xB124, 0xB124, prLV}, // Lo HANGUL SYLLABLE NE + {0xB125, 0xB13F, prLVT}, // Lo [27] HANGUL SYLLABLE NEG..HANGUL SYLLABLE NEH + {0xB140, 0xB140, prLV}, // Lo HANGUL SYLLABLE NYEO + {0xB141, 0xB15B, prLVT}, // Lo [27] HANGUL SYLLABLE NYEOG..HANGUL SYLLABLE NYEOH + {0xB15C, 0xB15C, prLV}, // Lo HANGUL SYLLABLE NYE + {0xB15D, 0xB177, prLVT}, // Lo [27] HANGUL SYLLABLE NYEG..HANGUL SYLLABLE NYEH + {0xB178, 0xB178, prLV}, // Lo HANGUL SYLLABLE NO + {0xB179, 0xB193, prLVT}, // Lo [27] HANGUL SYLLABLE NOG..HANGUL SYLLABLE NOH + {0xB194, 0xB194, prLV}, // Lo HANGUL SYLLABLE NWA + {0xB195, 0xB1AF, prLVT}, // Lo [27] HANGUL SYLLABLE NWAG..HANGUL SYLLABLE NWAH + {0xB1B0, 0xB1B0, prLV}, // Lo HANGUL SYLLABLE NWAE + {0xB1B1, 0xB1CB, prLVT}, // Lo [27] HANGUL SYLLABLE NWAEG..HANGUL SYLLABLE NWAEH + {0xB1CC, 0xB1CC, prLV}, // Lo HANGUL SYLLABLE NOE + {0xB1CD, 0xB1E7, prLVT}, // Lo [27] HANGUL SYLLABLE NOEG..HANGUL SYLLABLE NOEH + {0xB1E8, 0xB1E8, prLV}, // Lo HANGUL SYLLABLE NYO + {0xB1E9, 0xB203, prLVT}, // Lo [27] HANGUL SYLLABLE NYOG..HANGUL SYLLABLE NYOH + {0xB204, 0xB204, prLV}, // Lo HANGUL SYLLABLE NU + {0xB205, 0xB21F, prLVT}, // Lo [27] HANGUL SYLLABLE NUG..HANGUL SYLLABLE NUH + {0xB220, 0xB220, prLV}, // Lo HANGUL SYLLABLE NWEO + {0xB221, 0xB23B, prLVT}, // Lo [27] HANGUL SYLLABLE NWEOG..HANGUL SYLLABLE NWEOH + {0xB23C, 0xB23C, prLV}, // Lo HANGUL SYLLABLE NWE + {0xB23D, 0xB257, prLVT}, // Lo [27] HANGUL SYLLABLE NWEG..HANGUL SYLLABLE NWEH + {0xB258, 0xB258, prLV}, // Lo HANGUL SYLLABLE NWI + {0xB259, 0xB273, prLVT}, // Lo [27] HANGUL SYLLABLE NWIG..HANGUL SYLLABLE NWIH + {0xB274, 0xB274, prLV}, // Lo HANGUL SYLLABLE NYU + {0xB275, 0xB28F, prLVT}, // Lo [27] HANGUL SYLLABLE NYUG..HANGUL SYLLABLE NYUH + {0xB290, 0xB290, prLV}, // Lo HANGUL SYLLABLE NEU + {0xB291, 0xB2AB, prLVT}, // Lo [27] HANGUL SYLLABLE NEUG..HANGUL SYLLABLE NEUH + {0xB2AC, 0xB2AC, prLV}, // Lo HANGUL SYLLABLE NYI + {0xB2AD, 0xB2C7, prLVT}, // Lo [27] HANGUL SYLLABLE NYIG..HANGUL SYLLABLE NYIH + {0xB2C8, 0xB2C8, prLV}, // Lo HANGUL SYLLABLE NI + {0xB2C9, 0xB2E3, prLVT}, // Lo [27] HANGUL SYLLABLE NIG..HANGUL SYLLABLE NIH + {0xB2E4, 0xB2E4, prLV}, // Lo HANGUL SYLLABLE DA + {0xB2E5, 0xB2FF, prLVT}, // Lo [27] HANGUL SYLLABLE DAG..HANGUL SYLLABLE DAH + {0xB300, 0xB300, prLV}, // Lo HANGUL SYLLABLE DAE + {0xB301, 0xB31B, prLVT}, // Lo [27] HANGUL SYLLABLE DAEG..HANGUL SYLLABLE DAEH + {0xB31C, 0xB31C, prLV}, // Lo HANGUL SYLLABLE DYA + {0xB31D, 0xB337, prLVT}, // Lo [27] HANGUL SYLLABLE DYAG..HANGUL SYLLABLE DYAH + {0xB338, 0xB338, prLV}, // Lo HANGUL SYLLABLE DYAE + {0xB339, 0xB353, prLVT}, // Lo [27] HANGUL SYLLABLE DYAEG..HANGUL SYLLABLE DYAEH + {0xB354, 0xB354, prLV}, // Lo HANGUL SYLLABLE DEO + {0xB355, 0xB36F, prLVT}, // Lo [27] HANGUL SYLLABLE DEOG..HANGUL SYLLABLE DEOH + {0xB370, 0xB370, prLV}, // Lo HANGUL SYLLABLE DE + {0xB371, 0xB38B, prLVT}, // Lo [27] HANGUL SYLLABLE DEG..HANGUL SYLLABLE DEH + {0xB38C, 0xB38C, prLV}, // Lo HANGUL SYLLABLE DYEO + {0xB38D, 0xB3A7, prLVT}, // Lo [27] HANGUL SYLLABLE DYEOG..HANGUL SYLLABLE DYEOH + {0xB3A8, 0xB3A8, prLV}, // Lo HANGUL SYLLABLE DYE + {0xB3A9, 0xB3C3, prLVT}, // Lo [27] HANGUL SYLLABLE DYEG..HANGUL SYLLABLE DYEH + {0xB3C4, 0xB3C4, prLV}, // Lo HANGUL SYLLABLE DO + {0xB3C5, 0xB3DF, prLVT}, // Lo [27] HANGUL SYLLABLE DOG..HANGUL SYLLABLE DOH + {0xB3E0, 0xB3E0, prLV}, // Lo HANGUL SYLLABLE DWA + {0xB3E1, 0xB3FB, prLVT}, // Lo [27] HANGUL SYLLABLE DWAG..HANGUL SYLLABLE DWAH + {0xB3FC, 0xB3FC, prLV}, // Lo HANGUL SYLLABLE DWAE + {0xB3FD, 0xB417, prLVT}, // Lo [27] HANGUL SYLLABLE DWAEG..HANGUL SYLLABLE DWAEH + {0xB418, 0xB418, prLV}, // Lo HANGUL SYLLABLE DOE + {0xB419, 0xB433, prLVT}, // Lo [27] HANGUL SYLLABLE DOEG..HANGUL SYLLABLE DOEH + {0xB434, 0xB434, prLV}, // Lo HANGUL SYLLABLE DYO + {0xB435, 0xB44F, prLVT}, // Lo [27] HANGUL SYLLABLE DYOG..HANGUL SYLLABLE DYOH + {0xB450, 0xB450, prLV}, // Lo HANGUL SYLLABLE DU + {0xB451, 0xB46B, prLVT}, // Lo [27] HANGUL SYLLABLE DUG..HANGUL SYLLABLE DUH + {0xB46C, 0xB46C, prLV}, // Lo HANGUL SYLLABLE DWEO + {0xB46D, 0xB487, prLVT}, // Lo [27] HANGUL SYLLABLE DWEOG..HANGUL SYLLABLE DWEOH + {0xB488, 0xB488, prLV}, // Lo HANGUL SYLLABLE DWE + {0xB489, 0xB4A3, prLVT}, // Lo [27] HANGUL SYLLABLE DWEG..HANGUL SYLLABLE DWEH + {0xB4A4, 0xB4A4, prLV}, // Lo HANGUL SYLLABLE DWI + {0xB4A5, 0xB4BF, prLVT}, // Lo [27] HANGUL SYLLABLE DWIG..HANGUL SYLLABLE DWIH + {0xB4C0, 0xB4C0, prLV}, // Lo HANGUL SYLLABLE DYU + {0xB4C1, 0xB4DB, prLVT}, // Lo [27] HANGUL SYLLABLE DYUG..HANGUL SYLLABLE DYUH + {0xB4DC, 0xB4DC, prLV}, // Lo HANGUL SYLLABLE DEU + {0xB4DD, 0xB4F7, prLVT}, // Lo [27] HANGUL SYLLABLE DEUG..HANGUL SYLLABLE DEUH + {0xB4F8, 0xB4F8, prLV}, // Lo HANGUL SYLLABLE DYI + {0xB4F9, 0xB513, prLVT}, // Lo [27] HANGUL SYLLABLE DYIG..HANGUL SYLLABLE DYIH + {0xB514, 0xB514, prLV}, // Lo HANGUL SYLLABLE DI + {0xB515, 0xB52F, prLVT}, // Lo [27] HANGUL SYLLABLE DIG..HANGUL SYLLABLE DIH + {0xB530, 0xB530, prLV}, // Lo HANGUL SYLLABLE DDA + {0xB531, 0xB54B, prLVT}, // Lo [27] HANGUL SYLLABLE DDAG..HANGUL SYLLABLE DDAH + {0xB54C, 0xB54C, prLV}, // Lo HANGUL SYLLABLE DDAE + {0xB54D, 0xB567, prLVT}, // Lo [27] HANGUL SYLLABLE DDAEG..HANGUL SYLLABLE DDAEH + {0xB568, 0xB568, prLV}, // Lo HANGUL SYLLABLE DDYA + {0xB569, 0xB583, prLVT}, // Lo [27] HANGUL SYLLABLE DDYAG..HANGUL SYLLABLE DDYAH + {0xB584, 0xB584, prLV}, // Lo HANGUL SYLLABLE DDYAE + {0xB585, 0xB59F, prLVT}, // Lo [27] HANGUL SYLLABLE DDYAEG..HANGUL SYLLABLE DDYAEH + {0xB5A0, 0xB5A0, prLV}, // Lo HANGUL SYLLABLE DDEO + {0xB5A1, 0xB5BB, prLVT}, // Lo [27] HANGUL SYLLABLE DDEOG..HANGUL SYLLABLE DDEOH + {0xB5BC, 0xB5BC, prLV}, // Lo HANGUL SYLLABLE DDE + {0xB5BD, 0xB5D7, prLVT}, // Lo [27] HANGUL SYLLABLE DDEG..HANGUL SYLLABLE DDEH + {0xB5D8, 0xB5D8, prLV}, // Lo HANGUL SYLLABLE DDYEO + {0xB5D9, 0xB5F3, prLVT}, // Lo [27] HANGUL SYLLABLE DDYEOG..HANGUL SYLLABLE DDYEOH + {0xB5F4, 0xB5F4, prLV}, // Lo HANGUL SYLLABLE DDYE + {0xB5F5, 0xB60F, prLVT}, // Lo [27] HANGUL SYLLABLE DDYEG..HANGUL SYLLABLE DDYEH + {0xB610, 0xB610, prLV}, // Lo HANGUL SYLLABLE DDO + {0xB611, 0xB62B, prLVT}, // Lo [27] HANGUL SYLLABLE DDOG..HANGUL SYLLABLE DDOH + {0xB62C, 0xB62C, prLV}, // Lo HANGUL SYLLABLE DDWA + {0xB62D, 0xB647, prLVT}, // Lo [27] HANGUL SYLLABLE DDWAG..HANGUL SYLLABLE DDWAH + {0xB648, 0xB648, prLV}, // Lo HANGUL SYLLABLE DDWAE + {0xB649, 0xB663, prLVT}, // Lo [27] HANGUL SYLLABLE DDWAEG..HANGUL SYLLABLE DDWAEH + {0xB664, 0xB664, prLV}, // Lo HANGUL SYLLABLE DDOE + {0xB665, 0xB67F, prLVT}, // Lo [27] HANGUL SYLLABLE DDOEG..HANGUL SYLLABLE DDOEH + {0xB680, 0xB680, prLV}, // Lo HANGUL SYLLABLE DDYO + {0xB681, 0xB69B, prLVT}, // Lo [27] HANGUL SYLLABLE DDYOG..HANGUL SYLLABLE DDYOH + {0xB69C, 0xB69C, prLV}, // Lo HANGUL SYLLABLE DDU + {0xB69D, 0xB6B7, prLVT}, // Lo [27] HANGUL SYLLABLE DDUG..HANGUL SYLLABLE DDUH + {0xB6B8, 0xB6B8, prLV}, // Lo HANGUL SYLLABLE DDWEO + {0xB6B9, 0xB6D3, prLVT}, // Lo [27] HANGUL SYLLABLE DDWEOG..HANGUL SYLLABLE DDWEOH + {0xB6D4, 0xB6D4, prLV}, // Lo HANGUL SYLLABLE DDWE + {0xB6D5, 0xB6EF, prLVT}, // Lo [27] HANGUL SYLLABLE DDWEG..HANGUL SYLLABLE DDWEH + {0xB6F0, 0xB6F0, prLV}, // Lo HANGUL SYLLABLE DDWI + {0xB6F1, 0xB70B, prLVT}, // Lo [27] HANGUL SYLLABLE DDWIG..HANGUL SYLLABLE DDWIH + {0xB70C, 0xB70C, prLV}, // Lo HANGUL SYLLABLE DDYU + {0xB70D, 0xB727, prLVT}, // Lo [27] HANGUL SYLLABLE DDYUG..HANGUL SYLLABLE DDYUH + {0xB728, 0xB728, prLV}, // Lo HANGUL SYLLABLE DDEU + {0xB729, 0xB743, prLVT}, // Lo [27] HANGUL SYLLABLE DDEUG..HANGUL SYLLABLE DDEUH + {0xB744, 0xB744, prLV}, // Lo HANGUL SYLLABLE DDYI + {0xB745, 0xB75F, prLVT}, // Lo [27] HANGUL SYLLABLE DDYIG..HANGUL SYLLABLE DDYIH + {0xB760, 0xB760, prLV}, // Lo HANGUL SYLLABLE DDI + {0xB761, 0xB77B, prLVT}, // Lo [27] HANGUL SYLLABLE DDIG..HANGUL SYLLABLE DDIH + {0xB77C, 0xB77C, prLV}, // Lo HANGUL SYLLABLE RA + {0xB77D, 0xB797, prLVT}, // Lo [27] HANGUL SYLLABLE RAG..HANGUL SYLLABLE RAH + {0xB798, 0xB798, prLV}, // Lo HANGUL SYLLABLE RAE + {0xB799, 0xB7B3, prLVT}, // Lo [27] HANGUL SYLLABLE RAEG..HANGUL SYLLABLE RAEH + {0xB7B4, 0xB7B4, prLV}, // Lo HANGUL SYLLABLE RYA + {0xB7B5, 0xB7CF, prLVT}, // Lo [27] HANGUL SYLLABLE RYAG..HANGUL SYLLABLE RYAH + {0xB7D0, 0xB7D0, prLV}, // Lo HANGUL SYLLABLE RYAE + {0xB7D1, 0xB7EB, prLVT}, // Lo [27] HANGUL SYLLABLE RYAEG..HANGUL SYLLABLE RYAEH + {0xB7EC, 0xB7EC, prLV}, // Lo HANGUL SYLLABLE REO + {0xB7ED, 0xB807, prLVT}, // Lo [27] HANGUL SYLLABLE REOG..HANGUL SYLLABLE REOH + {0xB808, 0xB808, prLV}, // Lo HANGUL SYLLABLE RE + {0xB809, 0xB823, prLVT}, // Lo [27] HANGUL SYLLABLE REG..HANGUL SYLLABLE REH + {0xB824, 0xB824, prLV}, // Lo HANGUL SYLLABLE RYEO + {0xB825, 0xB83F, prLVT}, // Lo [27] HANGUL SYLLABLE RYEOG..HANGUL SYLLABLE RYEOH + {0xB840, 0xB840, prLV}, // Lo HANGUL SYLLABLE RYE + {0xB841, 0xB85B, prLVT}, // Lo [27] HANGUL SYLLABLE RYEG..HANGUL SYLLABLE RYEH + {0xB85C, 0xB85C, prLV}, // Lo HANGUL SYLLABLE RO + {0xB85D, 0xB877, prLVT}, // Lo [27] HANGUL SYLLABLE ROG..HANGUL SYLLABLE ROH + {0xB878, 0xB878, prLV}, // Lo HANGUL SYLLABLE RWA + {0xB879, 0xB893, prLVT}, // Lo [27] HANGUL SYLLABLE RWAG..HANGUL SYLLABLE RWAH + {0xB894, 0xB894, prLV}, // Lo HANGUL SYLLABLE RWAE + {0xB895, 0xB8AF, prLVT}, // Lo [27] HANGUL SYLLABLE RWAEG..HANGUL SYLLABLE RWAEH + {0xB8B0, 0xB8B0, prLV}, // Lo HANGUL SYLLABLE ROE + {0xB8B1, 0xB8CB, prLVT}, // Lo [27] HANGUL SYLLABLE ROEG..HANGUL SYLLABLE ROEH + {0xB8CC, 0xB8CC, prLV}, // Lo HANGUL SYLLABLE RYO + {0xB8CD, 0xB8E7, prLVT}, // Lo [27] HANGUL SYLLABLE RYOG..HANGUL SYLLABLE RYOH + {0xB8E8, 0xB8E8, prLV}, // Lo HANGUL SYLLABLE RU + {0xB8E9, 0xB903, prLVT}, // Lo [27] HANGUL SYLLABLE RUG..HANGUL SYLLABLE RUH + {0xB904, 0xB904, prLV}, // Lo HANGUL SYLLABLE RWEO + {0xB905, 0xB91F, prLVT}, // Lo [27] HANGUL SYLLABLE RWEOG..HANGUL SYLLABLE RWEOH + {0xB920, 0xB920, prLV}, // Lo HANGUL SYLLABLE RWE + {0xB921, 0xB93B, prLVT}, // Lo [27] HANGUL SYLLABLE RWEG..HANGUL SYLLABLE RWEH + {0xB93C, 0xB93C, prLV}, // Lo HANGUL SYLLABLE RWI + {0xB93D, 0xB957, prLVT}, // Lo [27] HANGUL SYLLABLE RWIG..HANGUL SYLLABLE RWIH + {0xB958, 0xB958, prLV}, // Lo HANGUL SYLLABLE RYU + {0xB959, 0xB973, prLVT}, // Lo [27] HANGUL SYLLABLE RYUG..HANGUL SYLLABLE RYUH + {0xB974, 0xB974, prLV}, // Lo HANGUL SYLLABLE REU + {0xB975, 0xB98F, prLVT}, // Lo [27] HANGUL SYLLABLE REUG..HANGUL SYLLABLE REUH + {0xB990, 0xB990, prLV}, // Lo HANGUL SYLLABLE RYI + {0xB991, 0xB9AB, prLVT}, // Lo [27] HANGUL SYLLABLE RYIG..HANGUL SYLLABLE RYIH + {0xB9AC, 0xB9AC, prLV}, // Lo HANGUL SYLLABLE RI + {0xB9AD, 0xB9C7, prLVT}, // Lo [27] HANGUL SYLLABLE RIG..HANGUL SYLLABLE RIH + {0xB9C8, 0xB9C8, prLV}, // Lo HANGUL SYLLABLE MA + {0xB9C9, 0xB9E3, prLVT}, // Lo [27] HANGUL SYLLABLE MAG..HANGUL SYLLABLE MAH + {0xB9E4, 0xB9E4, prLV}, // Lo HANGUL SYLLABLE MAE + {0xB9E5, 0xB9FF, prLVT}, // Lo [27] HANGUL SYLLABLE MAEG..HANGUL SYLLABLE MAEH + {0xBA00, 0xBA00, prLV}, // Lo HANGUL SYLLABLE MYA + {0xBA01, 0xBA1B, prLVT}, // Lo [27] HANGUL SYLLABLE MYAG..HANGUL SYLLABLE MYAH + {0xBA1C, 0xBA1C, prLV}, // Lo HANGUL SYLLABLE MYAE + {0xBA1D, 0xBA37, prLVT}, // Lo [27] HANGUL SYLLABLE MYAEG..HANGUL SYLLABLE MYAEH + {0xBA38, 0xBA38, prLV}, // Lo HANGUL SYLLABLE MEO + {0xBA39, 0xBA53, prLVT}, // Lo [27] HANGUL SYLLABLE MEOG..HANGUL SYLLABLE MEOH + {0xBA54, 0xBA54, prLV}, // Lo HANGUL SYLLABLE ME + {0xBA55, 0xBA6F, prLVT}, // Lo [27] HANGUL SYLLABLE MEG..HANGUL SYLLABLE MEH + {0xBA70, 0xBA70, prLV}, // Lo HANGUL SYLLABLE MYEO + {0xBA71, 0xBA8B, prLVT}, // Lo [27] HANGUL SYLLABLE MYEOG..HANGUL SYLLABLE MYEOH + {0xBA8C, 0xBA8C, prLV}, // Lo HANGUL SYLLABLE MYE + {0xBA8D, 0xBAA7, prLVT}, // Lo [27] HANGUL SYLLABLE MYEG..HANGUL SYLLABLE MYEH + {0xBAA8, 0xBAA8, prLV}, // Lo HANGUL SYLLABLE MO + {0xBAA9, 0xBAC3, prLVT}, // Lo [27] HANGUL SYLLABLE MOG..HANGUL SYLLABLE MOH + {0xBAC4, 0xBAC4, prLV}, // Lo HANGUL SYLLABLE MWA + {0xBAC5, 0xBADF, prLVT}, // Lo [27] HANGUL SYLLABLE MWAG..HANGUL SYLLABLE MWAH + {0xBAE0, 0xBAE0, prLV}, // Lo HANGUL SYLLABLE MWAE + {0xBAE1, 0xBAFB, prLVT}, // Lo [27] HANGUL SYLLABLE MWAEG..HANGUL SYLLABLE MWAEH + {0xBAFC, 0xBAFC, prLV}, // Lo HANGUL SYLLABLE MOE + {0xBAFD, 0xBB17, prLVT}, // Lo [27] HANGUL SYLLABLE MOEG..HANGUL SYLLABLE MOEH + {0xBB18, 0xBB18, prLV}, // Lo HANGUL SYLLABLE MYO + {0xBB19, 0xBB33, prLVT}, // Lo [27] HANGUL SYLLABLE MYOG..HANGUL SYLLABLE MYOH + {0xBB34, 0xBB34, prLV}, // Lo HANGUL SYLLABLE MU + {0xBB35, 0xBB4F, prLVT}, // Lo [27] HANGUL SYLLABLE MUG..HANGUL SYLLABLE MUH + {0xBB50, 0xBB50, prLV}, // Lo HANGUL SYLLABLE MWEO + {0xBB51, 0xBB6B, prLVT}, // Lo [27] HANGUL SYLLABLE MWEOG..HANGUL SYLLABLE MWEOH + {0xBB6C, 0xBB6C, prLV}, // Lo HANGUL SYLLABLE MWE + {0xBB6D, 0xBB87, prLVT}, // Lo [27] HANGUL SYLLABLE MWEG..HANGUL SYLLABLE MWEH + {0xBB88, 0xBB88, prLV}, // Lo HANGUL SYLLABLE MWI + {0xBB89, 0xBBA3, prLVT}, // Lo [27] HANGUL SYLLABLE MWIG..HANGUL SYLLABLE MWIH + {0xBBA4, 0xBBA4, prLV}, // Lo HANGUL SYLLABLE MYU + {0xBBA5, 0xBBBF, prLVT}, // Lo [27] HANGUL SYLLABLE MYUG..HANGUL SYLLABLE MYUH + {0xBBC0, 0xBBC0, prLV}, // Lo HANGUL SYLLABLE MEU + {0xBBC1, 0xBBDB, prLVT}, // Lo [27] HANGUL SYLLABLE MEUG..HANGUL SYLLABLE MEUH + {0xBBDC, 0xBBDC, prLV}, // Lo HANGUL SYLLABLE MYI + {0xBBDD, 0xBBF7, prLVT}, // Lo [27] HANGUL SYLLABLE MYIG..HANGUL SYLLABLE MYIH + {0xBBF8, 0xBBF8, prLV}, // Lo HANGUL SYLLABLE MI + {0xBBF9, 0xBC13, prLVT}, // Lo [27] HANGUL SYLLABLE MIG..HANGUL SYLLABLE MIH + {0xBC14, 0xBC14, prLV}, // Lo HANGUL SYLLABLE BA + {0xBC15, 0xBC2F, prLVT}, // Lo [27] HANGUL SYLLABLE BAG..HANGUL SYLLABLE BAH + {0xBC30, 0xBC30, prLV}, // Lo HANGUL SYLLABLE BAE + {0xBC31, 0xBC4B, prLVT}, // Lo [27] HANGUL SYLLABLE BAEG..HANGUL SYLLABLE BAEH + {0xBC4C, 0xBC4C, prLV}, // Lo HANGUL SYLLABLE BYA + {0xBC4D, 0xBC67, prLVT}, // Lo [27] HANGUL SYLLABLE BYAG..HANGUL SYLLABLE BYAH + {0xBC68, 0xBC68, prLV}, // Lo HANGUL SYLLABLE BYAE + {0xBC69, 0xBC83, prLVT}, // Lo [27] HANGUL SYLLABLE BYAEG..HANGUL SYLLABLE BYAEH + {0xBC84, 0xBC84, prLV}, // Lo HANGUL SYLLABLE BEO + {0xBC85, 0xBC9F, prLVT}, // Lo [27] HANGUL SYLLABLE BEOG..HANGUL SYLLABLE BEOH + {0xBCA0, 0xBCA0, prLV}, // Lo HANGUL SYLLABLE BE + {0xBCA1, 0xBCBB, prLVT}, // Lo [27] HANGUL SYLLABLE BEG..HANGUL SYLLABLE BEH + {0xBCBC, 0xBCBC, prLV}, // Lo HANGUL SYLLABLE BYEO + {0xBCBD, 0xBCD7, prLVT}, // Lo [27] HANGUL SYLLABLE BYEOG..HANGUL SYLLABLE BYEOH + {0xBCD8, 0xBCD8, prLV}, // Lo HANGUL SYLLABLE BYE + {0xBCD9, 0xBCF3, prLVT}, // Lo [27] HANGUL SYLLABLE BYEG..HANGUL SYLLABLE BYEH + {0xBCF4, 0xBCF4, prLV}, // Lo HANGUL SYLLABLE BO + {0xBCF5, 0xBD0F, prLVT}, // Lo [27] HANGUL SYLLABLE BOG..HANGUL SYLLABLE BOH + {0xBD10, 0xBD10, prLV}, // Lo HANGUL SYLLABLE BWA + {0xBD11, 0xBD2B, prLVT}, // Lo [27] HANGUL SYLLABLE BWAG..HANGUL SYLLABLE BWAH + {0xBD2C, 0xBD2C, prLV}, // Lo HANGUL SYLLABLE BWAE + {0xBD2D, 0xBD47, prLVT}, // Lo [27] HANGUL SYLLABLE BWAEG..HANGUL SYLLABLE BWAEH + {0xBD48, 0xBD48, prLV}, // Lo HANGUL SYLLABLE BOE + {0xBD49, 0xBD63, prLVT}, // Lo [27] HANGUL SYLLABLE BOEG..HANGUL SYLLABLE BOEH + {0xBD64, 0xBD64, prLV}, // Lo HANGUL SYLLABLE BYO + {0xBD65, 0xBD7F, prLVT}, // Lo [27] HANGUL SYLLABLE BYOG..HANGUL SYLLABLE BYOH + {0xBD80, 0xBD80, prLV}, // Lo HANGUL SYLLABLE BU + {0xBD81, 0xBD9B, prLVT}, // Lo [27] HANGUL SYLLABLE BUG..HANGUL SYLLABLE BUH + {0xBD9C, 0xBD9C, prLV}, // Lo HANGUL SYLLABLE BWEO + {0xBD9D, 0xBDB7, prLVT}, // Lo [27] HANGUL SYLLABLE BWEOG..HANGUL SYLLABLE BWEOH + {0xBDB8, 0xBDB8, prLV}, // Lo HANGUL SYLLABLE BWE + {0xBDB9, 0xBDD3, prLVT}, // Lo [27] HANGUL SYLLABLE BWEG..HANGUL SYLLABLE BWEH + {0xBDD4, 0xBDD4, prLV}, // Lo HANGUL SYLLABLE BWI + {0xBDD5, 0xBDEF, prLVT}, // Lo [27] HANGUL SYLLABLE BWIG..HANGUL SYLLABLE BWIH + {0xBDF0, 0xBDF0, prLV}, // Lo HANGUL SYLLABLE BYU + {0xBDF1, 0xBE0B, prLVT}, // Lo [27] HANGUL SYLLABLE BYUG..HANGUL SYLLABLE BYUH + {0xBE0C, 0xBE0C, prLV}, // Lo HANGUL SYLLABLE BEU + {0xBE0D, 0xBE27, prLVT}, // Lo [27] HANGUL SYLLABLE BEUG..HANGUL SYLLABLE BEUH + {0xBE28, 0xBE28, prLV}, // Lo HANGUL SYLLABLE BYI + {0xBE29, 0xBE43, prLVT}, // Lo [27] HANGUL SYLLABLE BYIG..HANGUL SYLLABLE BYIH + {0xBE44, 0xBE44, prLV}, // Lo HANGUL SYLLABLE BI + {0xBE45, 0xBE5F, prLVT}, // Lo [27] HANGUL SYLLABLE BIG..HANGUL SYLLABLE BIH + {0xBE60, 0xBE60, prLV}, // Lo HANGUL SYLLABLE BBA + {0xBE61, 0xBE7B, prLVT}, // Lo [27] HANGUL SYLLABLE BBAG..HANGUL SYLLABLE BBAH + {0xBE7C, 0xBE7C, prLV}, // Lo HANGUL SYLLABLE BBAE + {0xBE7D, 0xBE97, prLVT}, // Lo [27] HANGUL SYLLABLE BBAEG..HANGUL SYLLABLE BBAEH + {0xBE98, 0xBE98, prLV}, // Lo HANGUL SYLLABLE BBYA + {0xBE99, 0xBEB3, prLVT}, // Lo [27] HANGUL SYLLABLE BBYAG..HANGUL SYLLABLE BBYAH + {0xBEB4, 0xBEB4, prLV}, // Lo HANGUL SYLLABLE BBYAE + {0xBEB5, 0xBECF, prLVT}, // Lo [27] HANGUL SYLLABLE BBYAEG..HANGUL SYLLABLE BBYAEH + {0xBED0, 0xBED0, prLV}, // Lo HANGUL SYLLABLE BBEO + {0xBED1, 0xBEEB, prLVT}, // Lo [27] HANGUL SYLLABLE BBEOG..HANGUL SYLLABLE BBEOH + {0xBEEC, 0xBEEC, prLV}, // Lo HANGUL SYLLABLE BBE + {0xBEED, 0xBF07, prLVT}, // Lo [27] HANGUL SYLLABLE BBEG..HANGUL SYLLABLE BBEH + {0xBF08, 0xBF08, prLV}, // Lo HANGUL SYLLABLE BBYEO + {0xBF09, 0xBF23, prLVT}, // Lo [27] HANGUL SYLLABLE BBYEOG..HANGUL SYLLABLE BBYEOH + {0xBF24, 0xBF24, prLV}, // Lo HANGUL SYLLABLE BBYE + {0xBF25, 0xBF3F, prLVT}, // Lo [27] HANGUL SYLLABLE BBYEG..HANGUL SYLLABLE BBYEH + {0xBF40, 0xBF40, prLV}, // Lo HANGUL SYLLABLE BBO + {0xBF41, 0xBF5B, prLVT}, // Lo [27] HANGUL SYLLABLE BBOG..HANGUL SYLLABLE BBOH + {0xBF5C, 0xBF5C, prLV}, // Lo HANGUL SYLLABLE BBWA + {0xBF5D, 0xBF77, prLVT}, // Lo [27] HANGUL SYLLABLE BBWAG..HANGUL SYLLABLE BBWAH + {0xBF78, 0xBF78, prLV}, // Lo HANGUL SYLLABLE BBWAE + {0xBF79, 0xBF93, prLVT}, // Lo [27] HANGUL SYLLABLE BBWAEG..HANGUL SYLLABLE BBWAEH + {0xBF94, 0xBF94, prLV}, // Lo HANGUL SYLLABLE BBOE + {0xBF95, 0xBFAF, prLVT}, // Lo [27] HANGUL SYLLABLE BBOEG..HANGUL SYLLABLE BBOEH + {0xBFB0, 0xBFB0, prLV}, // Lo HANGUL SYLLABLE BBYO + {0xBFB1, 0xBFCB, prLVT}, // Lo [27] HANGUL SYLLABLE BBYOG..HANGUL SYLLABLE BBYOH + {0xBFCC, 0xBFCC, prLV}, // Lo HANGUL SYLLABLE BBU + {0xBFCD, 0xBFE7, prLVT}, // Lo [27] HANGUL SYLLABLE BBUG..HANGUL SYLLABLE BBUH + {0xBFE8, 0xBFE8, prLV}, // Lo HANGUL SYLLABLE BBWEO + {0xBFE9, 0xC003, prLVT}, // Lo [27] HANGUL SYLLABLE BBWEOG..HANGUL SYLLABLE BBWEOH + {0xC004, 0xC004, prLV}, // Lo HANGUL SYLLABLE BBWE + {0xC005, 0xC01F, prLVT}, // Lo [27] HANGUL SYLLABLE BBWEG..HANGUL SYLLABLE BBWEH + {0xC020, 0xC020, prLV}, // Lo HANGUL SYLLABLE BBWI + {0xC021, 0xC03B, prLVT}, // Lo [27] HANGUL SYLLABLE BBWIG..HANGUL SYLLABLE BBWIH + {0xC03C, 0xC03C, prLV}, // Lo HANGUL SYLLABLE BBYU + {0xC03D, 0xC057, prLVT}, // Lo [27] HANGUL SYLLABLE BBYUG..HANGUL SYLLABLE BBYUH + {0xC058, 0xC058, prLV}, // Lo HANGUL SYLLABLE BBEU + {0xC059, 0xC073, prLVT}, // Lo [27] HANGUL SYLLABLE BBEUG..HANGUL SYLLABLE BBEUH + {0xC074, 0xC074, prLV}, // Lo HANGUL SYLLABLE BBYI + {0xC075, 0xC08F, prLVT}, // Lo [27] HANGUL SYLLABLE BBYIG..HANGUL SYLLABLE BBYIH + {0xC090, 0xC090, prLV}, // Lo HANGUL SYLLABLE BBI + {0xC091, 0xC0AB, prLVT}, // Lo [27] HANGUL SYLLABLE BBIG..HANGUL SYLLABLE BBIH + {0xC0AC, 0xC0AC, prLV}, // Lo HANGUL SYLLABLE SA + {0xC0AD, 0xC0C7, prLVT}, // Lo [27] HANGUL SYLLABLE SAG..HANGUL SYLLABLE SAH + {0xC0C8, 0xC0C8, prLV}, // Lo HANGUL SYLLABLE SAE + {0xC0C9, 0xC0E3, prLVT}, // Lo [27] HANGUL SYLLABLE SAEG..HANGUL SYLLABLE SAEH + {0xC0E4, 0xC0E4, prLV}, // Lo HANGUL SYLLABLE SYA + {0xC0E5, 0xC0FF, prLVT}, // Lo [27] HANGUL SYLLABLE SYAG..HANGUL SYLLABLE SYAH + {0xC100, 0xC100, prLV}, // Lo HANGUL SYLLABLE SYAE + {0xC101, 0xC11B, prLVT}, // Lo [27] HANGUL SYLLABLE SYAEG..HANGUL SYLLABLE SYAEH + {0xC11C, 0xC11C, prLV}, // Lo HANGUL SYLLABLE SEO + {0xC11D, 0xC137, prLVT}, // Lo [27] HANGUL SYLLABLE SEOG..HANGUL SYLLABLE SEOH + {0xC138, 0xC138, prLV}, // Lo HANGUL SYLLABLE SE + {0xC139, 0xC153, prLVT}, // Lo [27] HANGUL SYLLABLE SEG..HANGUL SYLLABLE SEH + {0xC154, 0xC154, prLV}, // Lo HANGUL SYLLABLE SYEO + {0xC155, 0xC16F, prLVT}, // Lo [27] HANGUL SYLLABLE SYEOG..HANGUL SYLLABLE SYEOH + {0xC170, 0xC170, prLV}, // Lo HANGUL SYLLABLE SYE + {0xC171, 0xC18B, prLVT}, // Lo [27] HANGUL SYLLABLE SYEG..HANGUL SYLLABLE SYEH + {0xC18C, 0xC18C, prLV}, // Lo HANGUL SYLLABLE SO + {0xC18D, 0xC1A7, prLVT}, // Lo [27] HANGUL SYLLABLE SOG..HANGUL SYLLABLE SOH + {0xC1A8, 0xC1A8, prLV}, // Lo HANGUL SYLLABLE SWA + {0xC1A9, 0xC1C3, prLVT}, // Lo [27] HANGUL SYLLABLE SWAG..HANGUL SYLLABLE SWAH + {0xC1C4, 0xC1C4, prLV}, // Lo HANGUL SYLLABLE SWAE + {0xC1C5, 0xC1DF, prLVT}, // Lo [27] HANGUL SYLLABLE SWAEG..HANGUL SYLLABLE SWAEH + {0xC1E0, 0xC1E0, prLV}, // Lo HANGUL SYLLABLE SOE + {0xC1E1, 0xC1FB, prLVT}, // Lo [27] HANGUL SYLLABLE SOEG..HANGUL SYLLABLE SOEH + {0xC1FC, 0xC1FC, prLV}, // Lo HANGUL SYLLABLE SYO + {0xC1FD, 0xC217, prLVT}, // Lo [27] HANGUL SYLLABLE SYOG..HANGUL SYLLABLE SYOH + {0xC218, 0xC218, prLV}, // Lo HANGUL SYLLABLE SU + {0xC219, 0xC233, prLVT}, // Lo [27] HANGUL SYLLABLE SUG..HANGUL SYLLABLE SUH + {0xC234, 0xC234, prLV}, // Lo HANGUL SYLLABLE SWEO + {0xC235, 0xC24F, prLVT}, // Lo [27] HANGUL SYLLABLE SWEOG..HANGUL SYLLABLE SWEOH + {0xC250, 0xC250, prLV}, // Lo HANGUL SYLLABLE SWE + {0xC251, 0xC26B, prLVT}, // Lo [27] HANGUL SYLLABLE SWEG..HANGUL SYLLABLE SWEH + {0xC26C, 0xC26C, prLV}, // Lo HANGUL SYLLABLE SWI + {0xC26D, 0xC287, prLVT}, // Lo [27] HANGUL SYLLABLE SWIG..HANGUL SYLLABLE SWIH + {0xC288, 0xC288, prLV}, // Lo HANGUL SYLLABLE SYU + {0xC289, 0xC2A3, prLVT}, // Lo [27] HANGUL SYLLABLE SYUG..HANGUL SYLLABLE SYUH + {0xC2A4, 0xC2A4, prLV}, // Lo HANGUL SYLLABLE SEU + {0xC2A5, 0xC2BF, prLVT}, // Lo [27] HANGUL SYLLABLE SEUG..HANGUL SYLLABLE SEUH + {0xC2C0, 0xC2C0, prLV}, // Lo HANGUL SYLLABLE SYI + {0xC2C1, 0xC2DB, prLVT}, // Lo [27] HANGUL SYLLABLE SYIG..HANGUL SYLLABLE SYIH + {0xC2DC, 0xC2DC, prLV}, // Lo HANGUL SYLLABLE SI + {0xC2DD, 0xC2F7, prLVT}, // Lo [27] HANGUL SYLLABLE SIG..HANGUL SYLLABLE SIH + {0xC2F8, 0xC2F8, prLV}, // Lo HANGUL SYLLABLE SSA + {0xC2F9, 0xC313, prLVT}, // Lo [27] HANGUL SYLLABLE SSAG..HANGUL SYLLABLE SSAH + {0xC314, 0xC314, prLV}, // Lo HANGUL SYLLABLE SSAE + {0xC315, 0xC32F, prLVT}, // Lo [27] HANGUL SYLLABLE SSAEG..HANGUL SYLLABLE SSAEH + {0xC330, 0xC330, prLV}, // Lo HANGUL SYLLABLE SSYA + {0xC331, 0xC34B, prLVT}, // Lo [27] HANGUL SYLLABLE SSYAG..HANGUL SYLLABLE SSYAH + {0xC34C, 0xC34C, prLV}, // Lo HANGUL SYLLABLE SSYAE + {0xC34D, 0xC367, prLVT}, // Lo [27] HANGUL SYLLABLE SSYAEG..HANGUL SYLLABLE SSYAEH + {0xC368, 0xC368, prLV}, // Lo HANGUL SYLLABLE SSEO + {0xC369, 0xC383, prLVT}, // Lo [27] HANGUL SYLLABLE SSEOG..HANGUL SYLLABLE SSEOH + {0xC384, 0xC384, prLV}, // Lo HANGUL SYLLABLE SSE + {0xC385, 0xC39F, prLVT}, // Lo [27] HANGUL SYLLABLE SSEG..HANGUL SYLLABLE SSEH + {0xC3A0, 0xC3A0, prLV}, // Lo HANGUL SYLLABLE SSYEO + {0xC3A1, 0xC3BB, prLVT}, // Lo [27] HANGUL SYLLABLE SSYEOG..HANGUL SYLLABLE SSYEOH + {0xC3BC, 0xC3BC, prLV}, // Lo HANGUL SYLLABLE SSYE + {0xC3BD, 0xC3D7, prLVT}, // Lo [27] HANGUL SYLLABLE SSYEG..HANGUL SYLLABLE SSYEH + {0xC3D8, 0xC3D8, prLV}, // Lo HANGUL SYLLABLE SSO + {0xC3D9, 0xC3F3, prLVT}, // Lo [27] HANGUL SYLLABLE SSOG..HANGUL SYLLABLE SSOH + {0xC3F4, 0xC3F4, prLV}, // Lo HANGUL SYLLABLE SSWA + {0xC3F5, 0xC40F, prLVT}, // Lo [27] HANGUL SYLLABLE SSWAG..HANGUL SYLLABLE SSWAH + {0xC410, 0xC410, prLV}, // Lo HANGUL SYLLABLE SSWAE + {0xC411, 0xC42B, prLVT}, // Lo [27] HANGUL SYLLABLE SSWAEG..HANGUL SYLLABLE SSWAEH + {0xC42C, 0xC42C, prLV}, // Lo HANGUL SYLLABLE SSOE + {0xC42D, 0xC447, prLVT}, // Lo [27] HANGUL SYLLABLE SSOEG..HANGUL SYLLABLE SSOEH + {0xC448, 0xC448, prLV}, // Lo HANGUL SYLLABLE SSYO + {0xC449, 0xC463, prLVT}, // Lo [27] HANGUL SYLLABLE SSYOG..HANGUL SYLLABLE SSYOH + {0xC464, 0xC464, prLV}, // Lo HANGUL SYLLABLE SSU + {0xC465, 0xC47F, prLVT}, // Lo [27] HANGUL SYLLABLE SSUG..HANGUL SYLLABLE SSUH + {0xC480, 0xC480, prLV}, // Lo HANGUL SYLLABLE SSWEO + {0xC481, 0xC49B, prLVT}, // Lo [27] HANGUL SYLLABLE SSWEOG..HANGUL SYLLABLE SSWEOH + {0xC49C, 0xC49C, prLV}, // Lo HANGUL SYLLABLE SSWE + {0xC49D, 0xC4B7, prLVT}, // Lo [27] HANGUL SYLLABLE SSWEG..HANGUL SYLLABLE SSWEH + {0xC4B8, 0xC4B8, prLV}, // Lo HANGUL SYLLABLE SSWI + {0xC4B9, 0xC4D3, prLVT}, // Lo [27] HANGUL SYLLABLE SSWIG..HANGUL SYLLABLE SSWIH + {0xC4D4, 0xC4D4, prLV}, // Lo HANGUL SYLLABLE SSYU + {0xC4D5, 0xC4EF, prLVT}, // Lo [27] HANGUL SYLLABLE SSYUG..HANGUL SYLLABLE SSYUH + {0xC4F0, 0xC4F0, prLV}, // Lo HANGUL SYLLABLE SSEU + {0xC4F1, 0xC50B, prLVT}, // Lo [27] HANGUL SYLLABLE SSEUG..HANGUL SYLLABLE SSEUH + {0xC50C, 0xC50C, prLV}, // Lo HANGUL SYLLABLE SSYI + {0xC50D, 0xC527, prLVT}, // Lo [27] HANGUL SYLLABLE SSYIG..HANGUL SYLLABLE SSYIH + {0xC528, 0xC528, prLV}, // Lo HANGUL SYLLABLE SSI + {0xC529, 0xC543, prLVT}, // Lo [27] HANGUL SYLLABLE SSIG..HANGUL SYLLABLE SSIH + {0xC544, 0xC544, prLV}, // Lo HANGUL SYLLABLE A + {0xC545, 0xC55F, prLVT}, // Lo [27] HANGUL SYLLABLE AG..HANGUL SYLLABLE AH + {0xC560, 0xC560, prLV}, // Lo HANGUL SYLLABLE AE + {0xC561, 0xC57B, prLVT}, // Lo [27] HANGUL SYLLABLE AEG..HANGUL SYLLABLE AEH + {0xC57C, 0xC57C, prLV}, // Lo HANGUL SYLLABLE YA + {0xC57D, 0xC597, prLVT}, // Lo [27] HANGUL SYLLABLE YAG..HANGUL SYLLABLE YAH + {0xC598, 0xC598, prLV}, // Lo HANGUL SYLLABLE YAE + {0xC599, 0xC5B3, prLVT}, // Lo [27] HANGUL SYLLABLE YAEG..HANGUL SYLLABLE YAEH + {0xC5B4, 0xC5B4, prLV}, // Lo HANGUL SYLLABLE EO + {0xC5B5, 0xC5CF, prLVT}, // Lo [27] HANGUL SYLLABLE EOG..HANGUL SYLLABLE EOH + {0xC5D0, 0xC5D0, prLV}, // Lo HANGUL SYLLABLE E + {0xC5D1, 0xC5EB, prLVT}, // Lo [27] HANGUL SYLLABLE EG..HANGUL SYLLABLE EH + {0xC5EC, 0xC5EC, prLV}, // Lo HANGUL SYLLABLE YEO + {0xC5ED, 0xC607, prLVT}, // Lo [27] HANGUL SYLLABLE YEOG..HANGUL SYLLABLE YEOH + {0xC608, 0xC608, prLV}, // Lo HANGUL SYLLABLE YE + {0xC609, 0xC623, prLVT}, // Lo [27] HANGUL SYLLABLE YEG..HANGUL SYLLABLE YEH + {0xC624, 0xC624, prLV}, // Lo HANGUL SYLLABLE O + {0xC625, 0xC63F, prLVT}, // Lo [27] HANGUL SYLLABLE OG..HANGUL SYLLABLE OH + {0xC640, 0xC640, prLV}, // Lo HANGUL SYLLABLE WA + {0xC641, 0xC65B, prLVT}, // Lo [27] HANGUL SYLLABLE WAG..HANGUL SYLLABLE WAH + {0xC65C, 0xC65C, prLV}, // Lo HANGUL SYLLABLE WAE + {0xC65D, 0xC677, prLVT}, // Lo [27] HANGUL SYLLABLE WAEG..HANGUL SYLLABLE WAEH + {0xC678, 0xC678, prLV}, // Lo HANGUL SYLLABLE OE + {0xC679, 0xC693, prLVT}, // Lo [27] HANGUL SYLLABLE OEG..HANGUL SYLLABLE OEH + {0xC694, 0xC694, prLV}, // Lo HANGUL SYLLABLE YO + {0xC695, 0xC6AF, prLVT}, // Lo [27] HANGUL SYLLABLE YOG..HANGUL SYLLABLE YOH + {0xC6B0, 0xC6B0, prLV}, // Lo HANGUL SYLLABLE U + {0xC6B1, 0xC6CB, prLVT}, // Lo [27] HANGUL SYLLABLE UG..HANGUL SYLLABLE UH + {0xC6CC, 0xC6CC, prLV}, // Lo HANGUL SYLLABLE WEO + {0xC6CD, 0xC6E7, prLVT}, // Lo [27] HANGUL SYLLABLE WEOG..HANGUL SYLLABLE WEOH + {0xC6E8, 0xC6E8, prLV}, // Lo HANGUL SYLLABLE WE + {0xC6E9, 0xC703, prLVT}, // Lo [27] HANGUL SYLLABLE WEG..HANGUL SYLLABLE WEH + {0xC704, 0xC704, prLV}, // Lo HANGUL SYLLABLE WI + {0xC705, 0xC71F, prLVT}, // Lo [27] HANGUL SYLLABLE WIG..HANGUL SYLLABLE WIH + {0xC720, 0xC720, prLV}, // Lo HANGUL SYLLABLE YU + {0xC721, 0xC73B, prLVT}, // Lo [27] HANGUL SYLLABLE YUG..HANGUL SYLLABLE YUH + {0xC73C, 0xC73C, prLV}, // Lo HANGUL SYLLABLE EU + {0xC73D, 0xC757, prLVT}, // Lo [27] HANGUL SYLLABLE EUG..HANGUL SYLLABLE EUH + {0xC758, 0xC758, prLV}, // Lo HANGUL SYLLABLE YI + {0xC759, 0xC773, prLVT}, // Lo [27] HANGUL SYLLABLE YIG..HANGUL SYLLABLE YIH + {0xC774, 0xC774, prLV}, // Lo HANGUL SYLLABLE I + {0xC775, 0xC78F, prLVT}, // Lo [27] HANGUL SYLLABLE IG..HANGUL SYLLABLE IH + {0xC790, 0xC790, prLV}, // Lo HANGUL SYLLABLE JA + {0xC791, 0xC7AB, prLVT}, // Lo [27] HANGUL SYLLABLE JAG..HANGUL SYLLABLE JAH + {0xC7AC, 0xC7AC, prLV}, // Lo HANGUL SYLLABLE JAE + {0xC7AD, 0xC7C7, prLVT}, // Lo [27] HANGUL SYLLABLE JAEG..HANGUL SYLLABLE JAEH + {0xC7C8, 0xC7C8, prLV}, // Lo HANGUL SYLLABLE JYA + {0xC7C9, 0xC7E3, prLVT}, // Lo [27] HANGUL SYLLABLE JYAG..HANGUL SYLLABLE JYAH + {0xC7E4, 0xC7E4, prLV}, // Lo HANGUL SYLLABLE JYAE + {0xC7E5, 0xC7FF, prLVT}, // Lo [27] HANGUL SYLLABLE JYAEG..HANGUL SYLLABLE JYAEH + {0xC800, 0xC800, prLV}, // Lo HANGUL SYLLABLE JEO + {0xC801, 0xC81B, prLVT}, // Lo [27] HANGUL SYLLABLE JEOG..HANGUL SYLLABLE JEOH + {0xC81C, 0xC81C, prLV}, // Lo HANGUL SYLLABLE JE + {0xC81D, 0xC837, prLVT}, // Lo [27] HANGUL SYLLABLE JEG..HANGUL SYLLABLE JEH + {0xC838, 0xC838, prLV}, // Lo HANGUL SYLLABLE JYEO + {0xC839, 0xC853, prLVT}, // Lo [27] HANGUL SYLLABLE JYEOG..HANGUL SYLLABLE JYEOH + {0xC854, 0xC854, prLV}, // Lo HANGUL SYLLABLE JYE + {0xC855, 0xC86F, prLVT}, // Lo [27] HANGUL SYLLABLE JYEG..HANGUL SYLLABLE JYEH + {0xC870, 0xC870, prLV}, // Lo HANGUL SYLLABLE JO + {0xC871, 0xC88B, prLVT}, // Lo [27] HANGUL SYLLABLE JOG..HANGUL SYLLABLE JOH + {0xC88C, 0xC88C, prLV}, // Lo HANGUL SYLLABLE JWA + {0xC88D, 0xC8A7, prLVT}, // Lo [27] HANGUL SYLLABLE JWAG..HANGUL SYLLABLE JWAH + {0xC8A8, 0xC8A8, prLV}, // Lo HANGUL SYLLABLE JWAE + {0xC8A9, 0xC8C3, prLVT}, // Lo [27] HANGUL SYLLABLE JWAEG..HANGUL SYLLABLE JWAEH + {0xC8C4, 0xC8C4, prLV}, // Lo HANGUL SYLLABLE JOE + {0xC8C5, 0xC8DF, prLVT}, // Lo [27] HANGUL SYLLABLE JOEG..HANGUL SYLLABLE JOEH + {0xC8E0, 0xC8E0, prLV}, // Lo HANGUL SYLLABLE JYO + {0xC8E1, 0xC8FB, prLVT}, // Lo [27] HANGUL SYLLABLE JYOG..HANGUL SYLLABLE JYOH + {0xC8FC, 0xC8FC, prLV}, // Lo HANGUL SYLLABLE JU + {0xC8FD, 0xC917, prLVT}, // Lo [27] HANGUL SYLLABLE JUG..HANGUL SYLLABLE JUH + {0xC918, 0xC918, prLV}, // Lo HANGUL SYLLABLE JWEO + {0xC919, 0xC933, prLVT}, // Lo [27] HANGUL SYLLABLE JWEOG..HANGUL SYLLABLE JWEOH + {0xC934, 0xC934, prLV}, // Lo HANGUL SYLLABLE JWE + {0xC935, 0xC94F, prLVT}, // Lo [27] HANGUL SYLLABLE JWEG..HANGUL SYLLABLE JWEH + {0xC950, 0xC950, prLV}, // Lo HANGUL SYLLABLE JWI + {0xC951, 0xC96B, prLVT}, // Lo [27] HANGUL SYLLABLE JWIG..HANGUL SYLLABLE JWIH + {0xC96C, 0xC96C, prLV}, // Lo HANGUL SYLLABLE JYU + {0xC96D, 0xC987, prLVT}, // Lo [27] HANGUL SYLLABLE JYUG..HANGUL SYLLABLE JYUH + {0xC988, 0xC988, prLV}, // Lo HANGUL SYLLABLE JEU + {0xC989, 0xC9A3, prLVT}, // Lo [27] HANGUL SYLLABLE JEUG..HANGUL SYLLABLE JEUH + {0xC9A4, 0xC9A4, prLV}, // Lo HANGUL SYLLABLE JYI + {0xC9A5, 0xC9BF, prLVT}, // Lo [27] HANGUL SYLLABLE JYIG..HANGUL SYLLABLE JYIH + {0xC9C0, 0xC9C0, prLV}, // Lo HANGUL SYLLABLE JI + {0xC9C1, 0xC9DB, prLVT}, // Lo [27] HANGUL SYLLABLE JIG..HANGUL SYLLABLE JIH + {0xC9DC, 0xC9DC, prLV}, // Lo HANGUL SYLLABLE JJA + {0xC9DD, 0xC9F7, prLVT}, // Lo [27] HANGUL SYLLABLE JJAG..HANGUL SYLLABLE JJAH + {0xC9F8, 0xC9F8, prLV}, // Lo HANGUL SYLLABLE JJAE + {0xC9F9, 0xCA13, prLVT}, // Lo [27] HANGUL SYLLABLE JJAEG..HANGUL SYLLABLE JJAEH + {0xCA14, 0xCA14, prLV}, // Lo HANGUL SYLLABLE JJYA + {0xCA15, 0xCA2F, prLVT}, // Lo [27] HANGUL SYLLABLE JJYAG..HANGUL SYLLABLE JJYAH + {0xCA30, 0xCA30, prLV}, // Lo HANGUL SYLLABLE JJYAE + {0xCA31, 0xCA4B, prLVT}, // Lo [27] HANGUL SYLLABLE JJYAEG..HANGUL SYLLABLE JJYAEH + {0xCA4C, 0xCA4C, prLV}, // Lo HANGUL SYLLABLE JJEO + {0xCA4D, 0xCA67, prLVT}, // Lo [27] HANGUL SYLLABLE JJEOG..HANGUL SYLLABLE JJEOH + {0xCA68, 0xCA68, prLV}, // Lo HANGUL SYLLABLE JJE + {0xCA69, 0xCA83, prLVT}, // Lo [27] HANGUL SYLLABLE JJEG..HANGUL SYLLABLE JJEH + {0xCA84, 0xCA84, prLV}, // Lo HANGUL SYLLABLE JJYEO + {0xCA85, 0xCA9F, prLVT}, // Lo [27] HANGUL SYLLABLE JJYEOG..HANGUL SYLLABLE JJYEOH + {0xCAA0, 0xCAA0, prLV}, // Lo HANGUL SYLLABLE JJYE + {0xCAA1, 0xCABB, prLVT}, // Lo [27] HANGUL SYLLABLE JJYEG..HANGUL SYLLABLE JJYEH + {0xCABC, 0xCABC, prLV}, // Lo HANGUL SYLLABLE JJO + {0xCABD, 0xCAD7, prLVT}, // Lo [27] HANGUL SYLLABLE JJOG..HANGUL SYLLABLE JJOH + {0xCAD8, 0xCAD8, prLV}, // Lo HANGUL SYLLABLE JJWA + {0xCAD9, 0xCAF3, prLVT}, // Lo [27] HANGUL SYLLABLE JJWAG..HANGUL SYLLABLE JJWAH + {0xCAF4, 0xCAF4, prLV}, // Lo HANGUL SYLLABLE JJWAE + {0xCAF5, 0xCB0F, prLVT}, // Lo [27] HANGUL SYLLABLE JJWAEG..HANGUL SYLLABLE JJWAEH + {0xCB10, 0xCB10, prLV}, // Lo HANGUL SYLLABLE JJOE + {0xCB11, 0xCB2B, prLVT}, // Lo [27] HANGUL SYLLABLE JJOEG..HANGUL SYLLABLE JJOEH + {0xCB2C, 0xCB2C, prLV}, // Lo HANGUL SYLLABLE JJYO + {0xCB2D, 0xCB47, prLVT}, // Lo [27] HANGUL SYLLABLE JJYOG..HANGUL SYLLABLE JJYOH + {0xCB48, 0xCB48, prLV}, // Lo HANGUL SYLLABLE JJU + {0xCB49, 0xCB63, prLVT}, // Lo [27] HANGUL SYLLABLE JJUG..HANGUL SYLLABLE JJUH + {0xCB64, 0xCB64, prLV}, // Lo HANGUL SYLLABLE JJWEO + {0xCB65, 0xCB7F, prLVT}, // Lo [27] HANGUL SYLLABLE JJWEOG..HANGUL SYLLABLE JJWEOH + {0xCB80, 0xCB80, prLV}, // Lo HANGUL SYLLABLE JJWE + {0xCB81, 0xCB9B, prLVT}, // Lo [27] HANGUL SYLLABLE JJWEG..HANGUL SYLLABLE JJWEH + {0xCB9C, 0xCB9C, prLV}, // Lo HANGUL SYLLABLE JJWI + {0xCB9D, 0xCBB7, prLVT}, // Lo [27] HANGUL SYLLABLE JJWIG..HANGUL SYLLABLE JJWIH + {0xCBB8, 0xCBB8, prLV}, // Lo HANGUL SYLLABLE JJYU + {0xCBB9, 0xCBD3, prLVT}, // Lo [27] HANGUL SYLLABLE JJYUG..HANGUL SYLLABLE JJYUH + {0xCBD4, 0xCBD4, prLV}, // Lo HANGUL SYLLABLE JJEU + {0xCBD5, 0xCBEF, prLVT}, // Lo [27] HANGUL SYLLABLE JJEUG..HANGUL SYLLABLE JJEUH + {0xCBF0, 0xCBF0, prLV}, // Lo HANGUL SYLLABLE JJYI + {0xCBF1, 0xCC0B, prLVT}, // Lo [27] HANGUL SYLLABLE JJYIG..HANGUL SYLLABLE JJYIH + {0xCC0C, 0xCC0C, prLV}, // Lo HANGUL SYLLABLE JJI + {0xCC0D, 0xCC27, prLVT}, // Lo [27] HANGUL SYLLABLE JJIG..HANGUL SYLLABLE JJIH + {0xCC28, 0xCC28, prLV}, // Lo HANGUL SYLLABLE CA + {0xCC29, 0xCC43, prLVT}, // Lo [27] HANGUL SYLLABLE CAG..HANGUL SYLLABLE CAH + {0xCC44, 0xCC44, prLV}, // Lo HANGUL SYLLABLE CAE + {0xCC45, 0xCC5F, prLVT}, // Lo [27] HANGUL SYLLABLE CAEG..HANGUL SYLLABLE CAEH + {0xCC60, 0xCC60, prLV}, // Lo HANGUL SYLLABLE CYA + {0xCC61, 0xCC7B, prLVT}, // Lo [27] HANGUL SYLLABLE CYAG..HANGUL SYLLABLE CYAH + {0xCC7C, 0xCC7C, prLV}, // Lo HANGUL SYLLABLE CYAE + {0xCC7D, 0xCC97, prLVT}, // Lo [27] HANGUL SYLLABLE CYAEG..HANGUL SYLLABLE CYAEH + {0xCC98, 0xCC98, prLV}, // Lo HANGUL SYLLABLE CEO + {0xCC99, 0xCCB3, prLVT}, // Lo [27] HANGUL SYLLABLE CEOG..HANGUL SYLLABLE CEOH + {0xCCB4, 0xCCB4, prLV}, // Lo HANGUL SYLLABLE CE + {0xCCB5, 0xCCCF, prLVT}, // Lo [27] HANGUL SYLLABLE CEG..HANGUL SYLLABLE CEH + {0xCCD0, 0xCCD0, prLV}, // Lo HANGUL SYLLABLE CYEO + {0xCCD1, 0xCCEB, prLVT}, // Lo [27] HANGUL SYLLABLE CYEOG..HANGUL SYLLABLE CYEOH + {0xCCEC, 0xCCEC, prLV}, // Lo HANGUL SYLLABLE CYE + {0xCCED, 0xCD07, prLVT}, // Lo [27] HANGUL SYLLABLE CYEG..HANGUL SYLLABLE CYEH + {0xCD08, 0xCD08, prLV}, // Lo HANGUL SYLLABLE CO + {0xCD09, 0xCD23, prLVT}, // Lo [27] HANGUL SYLLABLE COG..HANGUL SYLLABLE COH + {0xCD24, 0xCD24, prLV}, // Lo HANGUL SYLLABLE CWA + {0xCD25, 0xCD3F, prLVT}, // Lo [27] HANGUL SYLLABLE CWAG..HANGUL SYLLABLE CWAH + {0xCD40, 0xCD40, prLV}, // Lo HANGUL SYLLABLE CWAE + {0xCD41, 0xCD5B, prLVT}, // Lo [27] HANGUL SYLLABLE CWAEG..HANGUL SYLLABLE CWAEH + {0xCD5C, 0xCD5C, prLV}, // Lo HANGUL SYLLABLE COE + {0xCD5D, 0xCD77, prLVT}, // Lo [27] HANGUL SYLLABLE COEG..HANGUL SYLLABLE COEH + {0xCD78, 0xCD78, prLV}, // Lo HANGUL SYLLABLE CYO + {0xCD79, 0xCD93, prLVT}, // Lo [27] HANGUL SYLLABLE CYOG..HANGUL SYLLABLE CYOH + {0xCD94, 0xCD94, prLV}, // Lo HANGUL SYLLABLE CU + {0xCD95, 0xCDAF, prLVT}, // Lo [27] HANGUL SYLLABLE CUG..HANGUL SYLLABLE CUH + {0xCDB0, 0xCDB0, prLV}, // Lo HANGUL SYLLABLE CWEO + {0xCDB1, 0xCDCB, prLVT}, // Lo [27] HANGUL SYLLABLE CWEOG..HANGUL SYLLABLE CWEOH + {0xCDCC, 0xCDCC, prLV}, // Lo HANGUL SYLLABLE CWE + {0xCDCD, 0xCDE7, prLVT}, // Lo [27] HANGUL SYLLABLE CWEG..HANGUL SYLLABLE CWEH + {0xCDE8, 0xCDE8, prLV}, // Lo HANGUL SYLLABLE CWI + {0xCDE9, 0xCE03, prLVT}, // Lo [27] HANGUL SYLLABLE CWIG..HANGUL SYLLABLE CWIH + {0xCE04, 0xCE04, prLV}, // Lo HANGUL SYLLABLE CYU + {0xCE05, 0xCE1F, prLVT}, // Lo [27] HANGUL SYLLABLE CYUG..HANGUL SYLLABLE CYUH + {0xCE20, 0xCE20, prLV}, // Lo HANGUL SYLLABLE CEU + {0xCE21, 0xCE3B, prLVT}, // Lo [27] HANGUL SYLLABLE CEUG..HANGUL SYLLABLE CEUH + {0xCE3C, 0xCE3C, prLV}, // Lo HANGUL SYLLABLE CYI + {0xCE3D, 0xCE57, prLVT}, // Lo [27] HANGUL SYLLABLE CYIG..HANGUL SYLLABLE CYIH + {0xCE58, 0xCE58, prLV}, // Lo HANGUL SYLLABLE CI + {0xCE59, 0xCE73, prLVT}, // Lo [27] HANGUL SYLLABLE CIG..HANGUL SYLLABLE CIH + {0xCE74, 0xCE74, prLV}, // Lo HANGUL SYLLABLE KA + {0xCE75, 0xCE8F, prLVT}, // Lo [27] HANGUL SYLLABLE KAG..HANGUL SYLLABLE KAH + {0xCE90, 0xCE90, prLV}, // Lo HANGUL SYLLABLE KAE + {0xCE91, 0xCEAB, prLVT}, // Lo [27] HANGUL SYLLABLE KAEG..HANGUL SYLLABLE KAEH + {0xCEAC, 0xCEAC, prLV}, // Lo HANGUL SYLLABLE KYA + {0xCEAD, 0xCEC7, prLVT}, // Lo [27] HANGUL SYLLABLE KYAG..HANGUL SYLLABLE KYAH + {0xCEC8, 0xCEC8, prLV}, // Lo HANGUL SYLLABLE KYAE + {0xCEC9, 0xCEE3, prLVT}, // Lo [27] HANGUL SYLLABLE KYAEG..HANGUL SYLLABLE KYAEH + {0xCEE4, 0xCEE4, prLV}, // Lo HANGUL SYLLABLE KEO + {0xCEE5, 0xCEFF, prLVT}, // Lo [27] HANGUL SYLLABLE KEOG..HANGUL SYLLABLE KEOH + {0xCF00, 0xCF00, prLV}, // Lo HANGUL SYLLABLE KE + {0xCF01, 0xCF1B, prLVT}, // Lo [27] HANGUL SYLLABLE KEG..HANGUL SYLLABLE KEH + {0xCF1C, 0xCF1C, prLV}, // Lo HANGUL SYLLABLE KYEO + {0xCF1D, 0xCF37, prLVT}, // Lo [27] HANGUL SYLLABLE KYEOG..HANGUL SYLLABLE KYEOH + {0xCF38, 0xCF38, prLV}, // Lo HANGUL SYLLABLE KYE + {0xCF39, 0xCF53, prLVT}, // Lo [27] HANGUL SYLLABLE KYEG..HANGUL SYLLABLE KYEH + {0xCF54, 0xCF54, prLV}, // Lo HANGUL SYLLABLE KO + {0xCF55, 0xCF6F, prLVT}, // Lo [27] HANGUL SYLLABLE KOG..HANGUL SYLLABLE KOH + {0xCF70, 0xCF70, prLV}, // Lo HANGUL SYLLABLE KWA + {0xCF71, 0xCF8B, prLVT}, // Lo [27] HANGUL SYLLABLE KWAG..HANGUL SYLLABLE KWAH + {0xCF8C, 0xCF8C, prLV}, // Lo HANGUL SYLLABLE KWAE + {0xCF8D, 0xCFA7, prLVT}, // Lo [27] HANGUL SYLLABLE KWAEG..HANGUL SYLLABLE KWAEH + {0xCFA8, 0xCFA8, prLV}, // Lo HANGUL SYLLABLE KOE + {0xCFA9, 0xCFC3, prLVT}, // Lo [27] HANGUL SYLLABLE KOEG..HANGUL SYLLABLE KOEH + {0xCFC4, 0xCFC4, prLV}, // Lo HANGUL SYLLABLE KYO + {0xCFC5, 0xCFDF, prLVT}, // Lo [27] HANGUL SYLLABLE KYOG..HANGUL SYLLABLE KYOH + {0xCFE0, 0xCFE0, prLV}, // Lo HANGUL SYLLABLE KU + {0xCFE1, 0xCFFB, prLVT}, // Lo [27] HANGUL SYLLABLE KUG..HANGUL SYLLABLE KUH + {0xCFFC, 0xCFFC, prLV}, // Lo HANGUL SYLLABLE KWEO + {0xCFFD, 0xD017, prLVT}, // Lo [27] HANGUL SYLLABLE KWEOG..HANGUL SYLLABLE KWEOH + {0xD018, 0xD018, prLV}, // Lo HANGUL SYLLABLE KWE + {0xD019, 0xD033, prLVT}, // Lo [27] HANGUL SYLLABLE KWEG..HANGUL SYLLABLE KWEH + {0xD034, 0xD034, prLV}, // Lo HANGUL SYLLABLE KWI + {0xD035, 0xD04F, prLVT}, // Lo [27] HANGUL SYLLABLE KWIG..HANGUL SYLLABLE KWIH + {0xD050, 0xD050, prLV}, // Lo HANGUL SYLLABLE KYU + {0xD051, 0xD06B, prLVT}, // Lo [27] HANGUL SYLLABLE KYUG..HANGUL SYLLABLE KYUH + {0xD06C, 0xD06C, prLV}, // Lo HANGUL SYLLABLE KEU + {0xD06D, 0xD087, prLVT}, // Lo [27] HANGUL SYLLABLE KEUG..HANGUL SYLLABLE KEUH + {0xD088, 0xD088, prLV}, // Lo HANGUL SYLLABLE KYI + {0xD089, 0xD0A3, prLVT}, // Lo [27] HANGUL SYLLABLE KYIG..HANGUL SYLLABLE KYIH + {0xD0A4, 0xD0A4, prLV}, // Lo HANGUL SYLLABLE KI + {0xD0A5, 0xD0BF, prLVT}, // Lo [27] HANGUL SYLLABLE KIG..HANGUL SYLLABLE KIH + {0xD0C0, 0xD0C0, prLV}, // Lo HANGUL SYLLABLE TA + {0xD0C1, 0xD0DB, prLVT}, // Lo [27] HANGUL SYLLABLE TAG..HANGUL SYLLABLE TAH + {0xD0DC, 0xD0DC, prLV}, // Lo HANGUL SYLLABLE TAE + {0xD0DD, 0xD0F7, prLVT}, // Lo [27] HANGUL SYLLABLE TAEG..HANGUL SYLLABLE TAEH + {0xD0F8, 0xD0F8, prLV}, // Lo HANGUL SYLLABLE TYA + {0xD0F9, 0xD113, prLVT}, // Lo [27] HANGUL SYLLABLE TYAG..HANGUL SYLLABLE TYAH + {0xD114, 0xD114, prLV}, // Lo HANGUL SYLLABLE TYAE + {0xD115, 0xD12F, prLVT}, // Lo [27] HANGUL SYLLABLE TYAEG..HANGUL SYLLABLE TYAEH + {0xD130, 0xD130, prLV}, // Lo HANGUL SYLLABLE TEO + {0xD131, 0xD14B, prLVT}, // Lo [27] HANGUL SYLLABLE TEOG..HANGUL SYLLABLE TEOH + {0xD14C, 0xD14C, prLV}, // Lo HANGUL SYLLABLE TE + {0xD14D, 0xD167, prLVT}, // Lo [27] HANGUL SYLLABLE TEG..HANGUL SYLLABLE TEH + {0xD168, 0xD168, prLV}, // Lo HANGUL SYLLABLE TYEO + {0xD169, 0xD183, prLVT}, // Lo [27] HANGUL SYLLABLE TYEOG..HANGUL SYLLABLE TYEOH + {0xD184, 0xD184, prLV}, // Lo HANGUL SYLLABLE TYE + {0xD185, 0xD19F, prLVT}, // Lo [27] HANGUL SYLLABLE TYEG..HANGUL SYLLABLE TYEH + {0xD1A0, 0xD1A0, prLV}, // Lo HANGUL SYLLABLE TO + {0xD1A1, 0xD1BB, prLVT}, // Lo [27] HANGUL SYLLABLE TOG..HANGUL SYLLABLE TOH + {0xD1BC, 0xD1BC, prLV}, // Lo HANGUL SYLLABLE TWA + {0xD1BD, 0xD1D7, prLVT}, // Lo [27] HANGUL SYLLABLE TWAG..HANGUL SYLLABLE TWAH + {0xD1D8, 0xD1D8, prLV}, // Lo HANGUL SYLLABLE TWAE + {0xD1D9, 0xD1F3, prLVT}, // Lo [27] HANGUL SYLLABLE TWAEG..HANGUL SYLLABLE TWAEH + {0xD1F4, 0xD1F4, prLV}, // Lo HANGUL SYLLABLE TOE + {0xD1F5, 0xD20F, prLVT}, // Lo [27] HANGUL SYLLABLE TOEG..HANGUL SYLLABLE TOEH + {0xD210, 0xD210, prLV}, // Lo HANGUL SYLLABLE TYO + {0xD211, 0xD22B, prLVT}, // Lo [27] HANGUL SYLLABLE TYOG..HANGUL SYLLABLE TYOH + {0xD22C, 0xD22C, prLV}, // Lo HANGUL SYLLABLE TU + {0xD22D, 0xD247, prLVT}, // Lo [27] HANGUL SYLLABLE TUG..HANGUL SYLLABLE TUH + {0xD248, 0xD248, prLV}, // Lo HANGUL SYLLABLE TWEO + {0xD249, 0xD263, prLVT}, // Lo [27] HANGUL SYLLABLE TWEOG..HANGUL SYLLABLE TWEOH + {0xD264, 0xD264, prLV}, // Lo HANGUL SYLLABLE TWE + {0xD265, 0xD27F, prLVT}, // Lo [27] HANGUL SYLLABLE TWEG..HANGUL SYLLABLE TWEH + {0xD280, 0xD280, prLV}, // Lo HANGUL SYLLABLE TWI + {0xD281, 0xD29B, prLVT}, // Lo [27] HANGUL SYLLABLE TWIG..HANGUL SYLLABLE TWIH + {0xD29C, 0xD29C, prLV}, // Lo HANGUL SYLLABLE TYU + {0xD29D, 0xD2B7, prLVT}, // Lo [27] HANGUL SYLLABLE TYUG..HANGUL SYLLABLE TYUH + {0xD2B8, 0xD2B8, prLV}, // Lo HANGUL SYLLABLE TEU + {0xD2B9, 0xD2D3, prLVT}, // Lo [27] HANGUL SYLLABLE TEUG..HANGUL SYLLABLE TEUH + {0xD2D4, 0xD2D4, prLV}, // Lo HANGUL SYLLABLE TYI + {0xD2D5, 0xD2EF, prLVT}, // Lo [27] HANGUL SYLLABLE TYIG..HANGUL SYLLABLE TYIH + {0xD2F0, 0xD2F0, prLV}, // Lo HANGUL SYLLABLE TI + {0xD2F1, 0xD30B, prLVT}, // Lo [27] HANGUL SYLLABLE TIG..HANGUL SYLLABLE TIH + {0xD30C, 0xD30C, prLV}, // Lo HANGUL SYLLABLE PA + {0xD30D, 0xD327, prLVT}, // Lo [27] HANGUL SYLLABLE PAG..HANGUL SYLLABLE PAH + {0xD328, 0xD328, prLV}, // Lo HANGUL SYLLABLE PAE + {0xD329, 0xD343, prLVT}, // Lo [27] HANGUL SYLLABLE PAEG..HANGUL SYLLABLE PAEH + {0xD344, 0xD344, prLV}, // Lo HANGUL SYLLABLE PYA + {0xD345, 0xD35F, prLVT}, // Lo [27] HANGUL SYLLABLE PYAG..HANGUL SYLLABLE PYAH + {0xD360, 0xD360, prLV}, // Lo HANGUL SYLLABLE PYAE + {0xD361, 0xD37B, prLVT}, // Lo [27] HANGUL SYLLABLE PYAEG..HANGUL SYLLABLE PYAEH + {0xD37C, 0xD37C, prLV}, // Lo HANGUL SYLLABLE PEO + {0xD37D, 0xD397, prLVT}, // Lo [27] HANGUL SYLLABLE PEOG..HANGUL SYLLABLE PEOH + {0xD398, 0xD398, prLV}, // Lo HANGUL SYLLABLE PE + {0xD399, 0xD3B3, prLVT}, // Lo [27] HANGUL SYLLABLE PEG..HANGUL SYLLABLE PEH + {0xD3B4, 0xD3B4, prLV}, // Lo HANGUL SYLLABLE PYEO + {0xD3B5, 0xD3CF, prLVT}, // Lo [27] HANGUL SYLLABLE PYEOG..HANGUL SYLLABLE PYEOH + {0xD3D0, 0xD3D0, prLV}, // Lo HANGUL SYLLABLE PYE + {0xD3D1, 0xD3EB, prLVT}, // Lo [27] HANGUL SYLLABLE PYEG..HANGUL SYLLABLE PYEH + {0xD3EC, 0xD3EC, prLV}, // Lo HANGUL SYLLABLE PO + {0xD3ED, 0xD407, prLVT}, // Lo [27] HANGUL SYLLABLE POG..HANGUL SYLLABLE POH + {0xD408, 0xD408, prLV}, // Lo HANGUL SYLLABLE PWA + {0xD409, 0xD423, prLVT}, // Lo [27] HANGUL SYLLABLE PWAG..HANGUL SYLLABLE PWAH + {0xD424, 0xD424, prLV}, // Lo HANGUL SYLLABLE PWAE + {0xD425, 0xD43F, prLVT}, // Lo [27] HANGUL SYLLABLE PWAEG..HANGUL SYLLABLE PWAEH + {0xD440, 0xD440, prLV}, // Lo HANGUL SYLLABLE POE + {0xD441, 0xD45B, prLVT}, // Lo [27] HANGUL SYLLABLE POEG..HANGUL SYLLABLE POEH + {0xD45C, 0xD45C, prLV}, // Lo HANGUL SYLLABLE PYO + {0xD45D, 0xD477, prLVT}, // Lo [27] HANGUL SYLLABLE PYOG..HANGUL SYLLABLE PYOH + {0xD478, 0xD478, prLV}, // Lo HANGUL SYLLABLE PU + {0xD479, 0xD493, prLVT}, // Lo [27] HANGUL SYLLABLE PUG..HANGUL SYLLABLE PUH + {0xD494, 0xD494, prLV}, // Lo HANGUL SYLLABLE PWEO + {0xD495, 0xD4AF, prLVT}, // Lo [27] HANGUL SYLLABLE PWEOG..HANGUL SYLLABLE PWEOH + {0xD4B0, 0xD4B0, prLV}, // Lo HANGUL SYLLABLE PWE + {0xD4B1, 0xD4CB, prLVT}, // Lo [27] HANGUL SYLLABLE PWEG..HANGUL SYLLABLE PWEH + {0xD4CC, 0xD4CC, prLV}, // Lo HANGUL SYLLABLE PWI + {0xD4CD, 0xD4E7, prLVT}, // Lo [27] HANGUL SYLLABLE PWIG..HANGUL SYLLABLE PWIH + {0xD4E8, 0xD4E8, prLV}, // Lo HANGUL SYLLABLE PYU + {0xD4E9, 0xD503, prLVT}, // Lo [27] HANGUL SYLLABLE PYUG..HANGUL SYLLABLE PYUH + {0xD504, 0xD504, prLV}, // Lo HANGUL SYLLABLE PEU + {0xD505, 0xD51F, prLVT}, // Lo [27] HANGUL SYLLABLE PEUG..HANGUL SYLLABLE PEUH + {0xD520, 0xD520, prLV}, // Lo HANGUL SYLLABLE PYI + {0xD521, 0xD53B, prLVT}, // Lo [27] HANGUL SYLLABLE PYIG..HANGUL SYLLABLE PYIH + {0xD53C, 0xD53C, prLV}, // Lo HANGUL SYLLABLE PI + {0xD53D, 0xD557, prLVT}, // Lo [27] HANGUL SYLLABLE PIG..HANGUL SYLLABLE PIH + {0xD558, 0xD558, prLV}, // Lo HANGUL SYLLABLE HA + {0xD559, 0xD573, prLVT}, // Lo [27] HANGUL SYLLABLE HAG..HANGUL SYLLABLE HAH + {0xD574, 0xD574, prLV}, // Lo HANGUL SYLLABLE HAE + {0xD575, 0xD58F, prLVT}, // Lo [27] HANGUL SYLLABLE HAEG..HANGUL SYLLABLE HAEH + {0xD590, 0xD590, prLV}, // Lo HANGUL SYLLABLE HYA + {0xD591, 0xD5AB, prLVT}, // Lo [27] HANGUL SYLLABLE HYAG..HANGUL SYLLABLE HYAH + {0xD5AC, 0xD5AC, prLV}, // Lo HANGUL SYLLABLE HYAE + {0xD5AD, 0xD5C7, prLVT}, // Lo [27] HANGUL SYLLABLE HYAEG..HANGUL SYLLABLE HYAEH + {0xD5C8, 0xD5C8, prLV}, // Lo HANGUL SYLLABLE HEO + {0xD5C9, 0xD5E3, prLVT}, // Lo [27] HANGUL SYLLABLE HEOG..HANGUL SYLLABLE HEOH + {0xD5E4, 0xD5E4, prLV}, // Lo HANGUL SYLLABLE HE + {0xD5E5, 0xD5FF, prLVT}, // Lo [27] HANGUL SYLLABLE HEG..HANGUL SYLLABLE HEH + {0xD600, 0xD600, prLV}, // Lo HANGUL SYLLABLE HYEO + {0xD601, 0xD61B, prLVT}, // Lo [27] HANGUL SYLLABLE HYEOG..HANGUL SYLLABLE HYEOH + {0xD61C, 0xD61C, prLV}, // Lo HANGUL SYLLABLE HYE + {0xD61D, 0xD637, prLVT}, // Lo [27] HANGUL SYLLABLE HYEG..HANGUL SYLLABLE HYEH + {0xD638, 0xD638, prLV}, // Lo HANGUL SYLLABLE HO + {0xD639, 0xD653, prLVT}, // Lo [27] HANGUL SYLLABLE HOG..HANGUL SYLLABLE HOH + {0xD654, 0xD654, prLV}, // Lo HANGUL SYLLABLE HWA + {0xD655, 0xD66F, prLVT}, // Lo [27] HANGUL SYLLABLE HWAG..HANGUL SYLLABLE HWAH + {0xD670, 0xD670, prLV}, // Lo HANGUL SYLLABLE HWAE + {0xD671, 0xD68B, prLVT}, // Lo [27] HANGUL SYLLABLE HWAEG..HANGUL SYLLABLE HWAEH + {0xD68C, 0xD68C, prLV}, // Lo HANGUL SYLLABLE HOE + {0xD68D, 0xD6A7, prLVT}, // Lo [27] HANGUL SYLLABLE HOEG..HANGUL SYLLABLE HOEH + {0xD6A8, 0xD6A8, prLV}, // Lo HANGUL SYLLABLE HYO + {0xD6A9, 0xD6C3, prLVT}, // Lo [27] HANGUL SYLLABLE HYOG..HANGUL SYLLABLE HYOH + {0xD6C4, 0xD6C4, prLV}, // Lo HANGUL SYLLABLE HU + {0xD6C5, 0xD6DF, prLVT}, // Lo [27] HANGUL SYLLABLE HUG..HANGUL SYLLABLE HUH + {0xD6E0, 0xD6E0, prLV}, // Lo HANGUL SYLLABLE HWEO + {0xD6E1, 0xD6FB, prLVT}, // Lo [27] HANGUL SYLLABLE HWEOG..HANGUL SYLLABLE HWEOH + {0xD6FC, 0xD6FC, prLV}, // Lo HANGUL SYLLABLE HWE + {0xD6FD, 0xD717, prLVT}, // Lo [27] HANGUL SYLLABLE HWEG..HANGUL SYLLABLE HWEH + {0xD718, 0xD718, prLV}, // Lo HANGUL SYLLABLE HWI + {0xD719, 0xD733, prLVT}, // Lo [27] HANGUL SYLLABLE HWIG..HANGUL SYLLABLE HWIH + {0xD734, 0xD734, prLV}, // Lo HANGUL SYLLABLE HYU + {0xD735, 0xD74F, prLVT}, // Lo [27] HANGUL SYLLABLE HYUG..HANGUL SYLLABLE HYUH + {0xD750, 0xD750, prLV}, // Lo HANGUL SYLLABLE HEU + {0xD751, 0xD76B, prLVT}, // Lo [27] HANGUL SYLLABLE HEUG..HANGUL SYLLABLE HEUH + {0xD76C, 0xD76C, prLV}, // Lo HANGUL SYLLABLE HYI + {0xD76D, 0xD787, prLVT}, // Lo [27] HANGUL SYLLABLE HYIG..HANGUL SYLLABLE HYIH + {0xD788, 0xD788, prLV}, // Lo HANGUL SYLLABLE HI + {0xD789, 0xD7A3, prLVT}, // Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH + {0xD7B0, 0xD7C6, prV}, // Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E + {0xD7CB, 0xD7FB, prT}, // Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH + {0xFB1E, 0xFB1E, prExtend}, // Mn HEBREW POINT JUDEO-SPANISH VARIKA + {0xFE00, 0xFE0F, prExtend}, // Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 + {0xFE20, 0xFE2F, prExtend}, // Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + {0xFEFF, 0xFEFF, prControl}, // Cf ZERO WIDTH NO-BREAK SPACE + {0xFF9E, 0xFF9F, prExtend}, // Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK + {0xFFF0, 0xFFF8, prControl}, // Cn [9] .. + {0xFFF9, 0xFFFB, prControl}, // Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR + {0x101FD, 0x101FD, prExtend}, // Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE + {0x102E0, 0x102E0, prExtend}, // Mn COPTIC EPACT THOUSANDS MARK + {0x10376, 0x1037A, prExtend}, // Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII + {0x10A01, 0x10A03, prExtend}, // Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R + {0x10A05, 0x10A06, prExtend}, // Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O + {0x10A0C, 0x10A0F, prExtend}, // Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA + {0x10A38, 0x10A3A, prExtend}, // Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW + {0x10A3F, 0x10A3F, prExtend}, // Mn KHAROSHTHI VIRAMA + {0x10AE5, 0x10AE6, prExtend}, // Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW + {0x10D24, 0x10D27, prExtend}, // Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI + {0x10EAB, 0x10EAC, prExtend}, // Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK + {0x10EFD, 0x10EFF, prExtend}, // Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA + {0x10F46, 0x10F50, prExtend}, // Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW + {0x10F82, 0x10F85, prExtend}, // Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW + {0x11000, 0x11000, prSpacingMark}, // Mc BRAHMI SIGN CANDRABINDU + {0x11001, 0x11001, prExtend}, // Mn BRAHMI SIGN ANUSVARA + {0x11002, 0x11002, prSpacingMark}, // Mc BRAHMI SIGN VISARGA + {0x11038, 0x11046, prExtend}, // Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA + {0x11070, 0x11070, prExtend}, // Mn BRAHMI SIGN OLD TAMIL VIRAMA + {0x11073, 0x11074, prExtend}, // Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O + {0x1107F, 0x11081, prExtend}, // Mn [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA + {0x11082, 0x11082, prSpacingMark}, // Mc KAITHI SIGN VISARGA + {0x110B0, 0x110B2, prSpacingMark}, // Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II + {0x110B3, 0x110B6, prExtend}, // Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI + {0x110B7, 0x110B8, prSpacingMark}, // Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU + {0x110B9, 0x110BA, prExtend}, // Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA + {0x110BD, 0x110BD, prPrepend}, // Cf KAITHI NUMBER SIGN + {0x110C2, 0x110C2, prExtend}, // Mn KAITHI VOWEL SIGN VOCALIC R + {0x110CD, 0x110CD, prPrepend}, // Cf KAITHI NUMBER SIGN ABOVE + {0x11100, 0x11102, prExtend}, // Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA + {0x11127, 0x1112B, prExtend}, // Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU + {0x1112C, 0x1112C, prSpacingMark}, // Mc CHAKMA VOWEL SIGN E + {0x1112D, 0x11134, prExtend}, // Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA + {0x11145, 0x11146, prSpacingMark}, // Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI + {0x11173, 0x11173, prExtend}, // Mn MAHAJANI SIGN NUKTA + {0x11180, 0x11181, prExtend}, // Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA + {0x11182, 0x11182, prSpacingMark}, // Mc SHARADA SIGN VISARGA + {0x111B3, 0x111B5, prSpacingMark}, // Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II + {0x111B6, 0x111BE, prExtend}, // Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O + {0x111BF, 0x111C0, prSpacingMark}, // Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA + {0x111C2, 0x111C3, prPrepend}, // Lo [2] SHARADA SIGN JIHVAMULIYA..SHARADA SIGN UPADHMANIYA + {0x111C9, 0x111CC, prExtend}, // Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK + {0x111CE, 0x111CE, prSpacingMark}, // Mc SHARADA VOWEL SIGN PRISHTHAMATRA E + {0x111CF, 0x111CF, prExtend}, // Mn SHARADA SIGN INVERTED CANDRABINDU + {0x1122C, 0x1122E, prSpacingMark}, // Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II + {0x1122F, 0x11231, prExtend}, // Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI + {0x11232, 0x11233, prSpacingMark}, // Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU + {0x11234, 0x11234, prExtend}, // Mn KHOJKI SIGN ANUSVARA + {0x11235, 0x11235, prSpacingMark}, // Mc KHOJKI SIGN VIRAMA + {0x11236, 0x11237, prExtend}, // Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA + {0x1123E, 0x1123E, prExtend}, // Mn KHOJKI SIGN SUKUN + {0x11241, 0x11241, prExtend}, // Mn KHOJKI VOWEL SIGN VOCALIC R + {0x112DF, 0x112DF, prExtend}, // Mn KHUDAWADI SIGN ANUSVARA + {0x112E0, 0x112E2, prSpacingMark}, // Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II + {0x112E3, 0x112EA, prExtend}, // Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA + {0x11300, 0x11301, prExtend}, // Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU + {0x11302, 0x11303, prSpacingMark}, // Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA + {0x1133B, 0x1133C, prExtend}, // Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA + {0x1133E, 0x1133E, prExtend}, // Mc GRANTHA VOWEL SIGN AA + {0x1133F, 0x1133F, prSpacingMark}, // Mc GRANTHA VOWEL SIGN I + {0x11340, 0x11340, prExtend}, // Mn GRANTHA VOWEL SIGN II + {0x11341, 0x11344, prSpacingMark}, // Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR + {0x11347, 0x11348, prSpacingMark}, // Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI + {0x1134B, 0x1134D, prSpacingMark}, // Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA + {0x11357, 0x11357, prExtend}, // Mc GRANTHA AU LENGTH MARK + {0x11362, 0x11363, prSpacingMark}, // Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL + {0x11366, 0x1136C, prExtend}, // Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX + {0x11370, 0x11374, prExtend}, // Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA + {0x11435, 0x11437, prSpacingMark}, // Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II + {0x11438, 0x1143F, prExtend}, // Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI + {0x11440, 0x11441, prSpacingMark}, // Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU + {0x11442, 0x11444, prExtend}, // Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA + {0x11445, 0x11445, prSpacingMark}, // Mc NEWA SIGN VISARGA + {0x11446, 0x11446, prExtend}, // Mn NEWA SIGN NUKTA + {0x1145E, 0x1145E, prExtend}, // Mn NEWA SANDHI MARK + {0x114B0, 0x114B0, prExtend}, // Mc TIRHUTA VOWEL SIGN AA + {0x114B1, 0x114B2, prSpacingMark}, // Mc [2] TIRHUTA VOWEL SIGN I..TIRHUTA VOWEL SIGN II + {0x114B3, 0x114B8, prExtend}, // Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL + {0x114B9, 0x114B9, prSpacingMark}, // Mc TIRHUTA VOWEL SIGN E + {0x114BA, 0x114BA, prExtend}, // Mn TIRHUTA VOWEL SIGN SHORT E + {0x114BB, 0x114BC, prSpacingMark}, // Mc [2] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN O + {0x114BD, 0x114BD, prExtend}, // Mc TIRHUTA VOWEL SIGN SHORT O + {0x114BE, 0x114BE, prSpacingMark}, // Mc TIRHUTA VOWEL SIGN AU + {0x114BF, 0x114C0, prExtend}, // Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA + {0x114C1, 0x114C1, prSpacingMark}, // Mc TIRHUTA SIGN VISARGA + {0x114C2, 0x114C3, prExtend}, // Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA + {0x115AF, 0x115AF, prExtend}, // Mc SIDDHAM VOWEL SIGN AA + {0x115B0, 0x115B1, prSpacingMark}, // Mc [2] SIDDHAM VOWEL SIGN I..SIDDHAM VOWEL SIGN II + {0x115B2, 0x115B5, prExtend}, // Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR + {0x115B8, 0x115BB, prSpacingMark}, // Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU + {0x115BC, 0x115BD, prExtend}, // Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA + {0x115BE, 0x115BE, prSpacingMark}, // Mc SIDDHAM SIGN VISARGA + {0x115BF, 0x115C0, prExtend}, // Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA + {0x115DC, 0x115DD, prExtend}, // Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU + {0x11630, 0x11632, prSpacingMark}, // Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II + {0x11633, 0x1163A, prExtend}, // Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI + {0x1163B, 0x1163C, prSpacingMark}, // Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU + {0x1163D, 0x1163D, prExtend}, // Mn MODI SIGN ANUSVARA + {0x1163E, 0x1163E, prSpacingMark}, // Mc MODI SIGN VISARGA + {0x1163F, 0x11640, prExtend}, // Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA + {0x116AB, 0x116AB, prExtend}, // Mn TAKRI SIGN ANUSVARA + {0x116AC, 0x116AC, prSpacingMark}, // Mc TAKRI SIGN VISARGA + {0x116AD, 0x116AD, prExtend}, // Mn TAKRI VOWEL SIGN AA + {0x116AE, 0x116AF, prSpacingMark}, // Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II + {0x116B0, 0x116B5, prExtend}, // Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU + {0x116B6, 0x116B6, prSpacingMark}, // Mc TAKRI SIGN VIRAMA + {0x116B7, 0x116B7, prExtend}, // Mn TAKRI SIGN NUKTA + {0x1171D, 0x1171F, prExtend}, // Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA + {0x11722, 0x11725, prExtend}, // Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU + {0x11726, 0x11726, prSpacingMark}, // Mc AHOM VOWEL SIGN E + {0x11727, 0x1172B, prExtend}, // Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER + {0x1182C, 0x1182E, prSpacingMark}, // Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II + {0x1182F, 0x11837, prExtend}, // Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA + {0x11838, 0x11838, prSpacingMark}, // Mc DOGRA SIGN VISARGA + {0x11839, 0x1183A, prExtend}, // Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA + {0x11930, 0x11930, prExtend}, // Mc DIVES AKURU VOWEL SIGN AA + {0x11931, 0x11935, prSpacingMark}, // Mc [5] DIVES AKURU VOWEL SIGN I..DIVES AKURU VOWEL SIGN E + {0x11937, 0x11938, prSpacingMark}, // Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O + {0x1193B, 0x1193C, prExtend}, // Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU + {0x1193D, 0x1193D, prSpacingMark}, // Mc DIVES AKURU SIGN HALANTA + {0x1193E, 0x1193E, prExtend}, // Mn DIVES AKURU VIRAMA + {0x1193F, 0x1193F, prPrepend}, // Lo DIVES AKURU PREFIXED NASAL SIGN + {0x11940, 0x11940, prSpacingMark}, // Mc DIVES AKURU MEDIAL YA + {0x11941, 0x11941, prPrepend}, // Lo DIVES AKURU INITIAL RA + {0x11942, 0x11942, prSpacingMark}, // Mc DIVES AKURU MEDIAL RA + {0x11943, 0x11943, prExtend}, // Mn DIVES AKURU SIGN NUKTA + {0x119D1, 0x119D3, prSpacingMark}, // Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II + {0x119D4, 0x119D7, prExtend}, // Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR + {0x119DA, 0x119DB, prExtend}, // Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI + {0x119DC, 0x119DF, prSpacingMark}, // Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA + {0x119E0, 0x119E0, prExtend}, // Mn NANDINAGARI SIGN VIRAMA + {0x119E4, 0x119E4, prSpacingMark}, // Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E + {0x11A01, 0x11A0A, prExtend}, // Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK + {0x11A33, 0x11A38, prExtend}, // Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA + {0x11A39, 0x11A39, prSpacingMark}, // Mc ZANABAZAR SQUARE SIGN VISARGA + {0x11A3A, 0x11A3A, prPrepend}, // Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA + {0x11A3B, 0x11A3E, prExtend}, // Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA + {0x11A47, 0x11A47, prExtend}, // Mn ZANABAZAR SQUARE SUBJOINER + {0x11A51, 0x11A56, prExtend}, // Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE + {0x11A57, 0x11A58, prSpacingMark}, // Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU + {0x11A59, 0x11A5B, prExtend}, // Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK + {0x11A84, 0x11A89, prPrepend}, // Lo [6] SOYOMBO SIGN JIHVAMULIYA..SOYOMBO CLUSTER-INITIAL LETTER SA + {0x11A8A, 0x11A96, prExtend}, // Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA + {0x11A97, 0x11A97, prSpacingMark}, // Mc SOYOMBO SIGN VISARGA + {0x11A98, 0x11A99, prExtend}, // Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER + {0x11C2F, 0x11C2F, prSpacingMark}, // Mc BHAIKSUKI VOWEL SIGN AA + {0x11C30, 0x11C36, prExtend}, // Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L + {0x11C38, 0x11C3D, prExtend}, // Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA + {0x11C3E, 0x11C3E, prSpacingMark}, // Mc BHAIKSUKI SIGN VISARGA + {0x11C3F, 0x11C3F, prExtend}, // Mn BHAIKSUKI SIGN VIRAMA + {0x11C92, 0x11CA7, prExtend}, // Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA + {0x11CA9, 0x11CA9, prSpacingMark}, // Mc MARCHEN SUBJOINED LETTER YA + {0x11CAA, 0x11CB0, prExtend}, // Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA + {0x11CB1, 0x11CB1, prSpacingMark}, // Mc MARCHEN VOWEL SIGN I + {0x11CB2, 0x11CB3, prExtend}, // Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E + {0x11CB4, 0x11CB4, prSpacingMark}, // Mc MARCHEN VOWEL SIGN O + {0x11CB5, 0x11CB6, prExtend}, // Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU + {0x11D31, 0x11D36, prExtend}, // Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R + {0x11D3A, 0x11D3A, prExtend}, // Mn MASARAM GONDI VOWEL SIGN E + {0x11D3C, 0x11D3D, prExtend}, // Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O + {0x11D3F, 0x11D45, prExtend}, // Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA + {0x11D46, 0x11D46, prPrepend}, // Lo MASARAM GONDI REPHA + {0x11D47, 0x11D47, prExtend}, // Mn MASARAM GONDI RA-KARA + {0x11D8A, 0x11D8E, prSpacingMark}, // Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU + {0x11D90, 0x11D91, prExtend}, // Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI + {0x11D93, 0x11D94, prSpacingMark}, // Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU + {0x11D95, 0x11D95, prExtend}, // Mn GUNJALA GONDI SIGN ANUSVARA + {0x11D96, 0x11D96, prSpacingMark}, // Mc GUNJALA GONDI SIGN VISARGA + {0x11D97, 0x11D97, prExtend}, // Mn GUNJALA GONDI VIRAMA + {0x11EF3, 0x11EF4, prExtend}, // Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U + {0x11EF5, 0x11EF6, prSpacingMark}, // Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O + {0x11F00, 0x11F01, prExtend}, // Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA + {0x11F02, 0x11F02, prPrepend}, // Lo KAWI SIGN REPHA + {0x11F03, 0x11F03, prSpacingMark}, // Mc KAWI SIGN VISARGA + {0x11F34, 0x11F35, prSpacingMark}, // Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA + {0x11F36, 0x11F3A, prExtend}, // Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R + {0x11F3E, 0x11F3F, prSpacingMark}, // Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI + {0x11F40, 0x11F40, prExtend}, // Mn KAWI VOWEL SIGN EU + {0x11F41, 0x11F41, prSpacingMark}, // Mc KAWI SIGN KILLER + {0x11F42, 0x11F42, prExtend}, // Mn KAWI CONJOINER + {0x13430, 0x1343F, prControl}, // Cf [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE + {0x13440, 0x13440, prExtend}, // Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY + {0x13447, 0x13455, prExtend}, // Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED + {0x16AF0, 0x16AF4, prExtend}, // Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE + {0x16B30, 0x16B36, prExtend}, // Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM + {0x16F4F, 0x16F4F, prExtend}, // Mn MIAO SIGN CONSONANT MODIFIER BAR + {0x16F51, 0x16F87, prSpacingMark}, // Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI + {0x16F8F, 0x16F92, prExtend}, // Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW + {0x16FE4, 0x16FE4, prExtend}, // Mn KHITAN SMALL SCRIPT FILLER + {0x16FF0, 0x16FF1, prSpacingMark}, // Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY + {0x1BC9D, 0x1BC9E, prExtend}, // Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK + {0x1BCA0, 0x1BCA3, prControl}, // Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP + {0x1CF00, 0x1CF2D, prExtend}, // Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT + {0x1CF30, 0x1CF46, prExtend}, // Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG + {0x1D165, 0x1D165, prExtend}, // Mc MUSICAL SYMBOL COMBINING STEM + {0x1D166, 0x1D166, prSpacingMark}, // Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM + {0x1D167, 0x1D169, prExtend}, // Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 + {0x1D16D, 0x1D16D, prSpacingMark}, // Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT + {0x1D16E, 0x1D172, prExtend}, // Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5 + {0x1D173, 0x1D17A, prControl}, // Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE + {0x1D17B, 0x1D182, prExtend}, // Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE + {0x1D185, 0x1D18B, prExtend}, // Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE + {0x1D1AA, 0x1D1AD, prExtend}, // Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO + {0x1D242, 0x1D244, prExtend}, // Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + {0x1DA00, 0x1DA36, prExtend}, // Mn [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN + {0x1DA3B, 0x1DA6C, prExtend}, // Mn [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT + {0x1DA75, 0x1DA75, prExtend}, // Mn SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS + {0x1DA84, 0x1DA84, prExtend}, // Mn SIGNWRITING LOCATION HEAD NECK + {0x1DA9B, 0x1DA9F, prExtend}, // Mn [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 + {0x1DAA1, 0x1DAAF, prExtend}, // Mn [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 + {0x1E000, 0x1E006, prExtend}, // Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE + {0x1E008, 0x1E018, prExtend}, // Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU + {0x1E01B, 0x1E021, prExtend}, // Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI + {0x1E023, 0x1E024, prExtend}, // Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS + {0x1E026, 0x1E02A, prExtend}, // Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA + {0x1E08F, 0x1E08F, prExtend}, // Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + {0x1E130, 0x1E136, prExtend}, // Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D + {0x1E2AE, 0x1E2AE, prExtend}, // Mn TOTO SIGN RISING TONE + {0x1E2EC, 0x1E2EF, prExtend}, // Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI + {0x1E4EC, 0x1E4EF, prExtend}, // Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH + {0x1E8D0, 0x1E8D6, prExtend}, // Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + {0x1E944, 0x1E94A, prExtend}, // Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA + {0x1F000, 0x1F003, prExtendedPictographic}, // E0.0 [4] (๐Ÿ€€..๐Ÿ€ƒ) MAHJONG TILE EAST WIND..MAHJONG TILE NORTH WIND + {0x1F004, 0x1F004, prExtendedPictographic}, // E0.6 [1] (๐Ÿ€„) mahjong red dragon + {0x1F005, 0x1F0CE, prExtendedPictographic}, // E0.0 [202] (๐Ÿ€…..๐ŸƒŽ) MAHJONG TILE GREEN DRAGON..PLAYING CARD KING OF DIAMONDS + {0x1F0CF, 0x1F0CF, prExtendedPictographic}, // E0.6 [1] (๐Ÿƒ) joker + {0x1F0D0, 0x1F0FF, prExtendedPictographic}, // E0.0 [48] (๐Ÿƒ..๐Ÿƒฟ) .. + {0x1F10D, 0x1F10F, prExtendedPictographic}, // E0.0 [3] (๐Ÿ„..๐Ÿ„) CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH + {0x1F12F, 0x1F12F, prExtendedPictographic}, // E0.0 [1] (๐Ÿ„ฏ) COPYLEFT SYMBOL + {0x1F16C, 0x1F16F, prExtendedPictographic}, // E0.0 [4] (๐Ÿ…ฌ..๐Ÿ…ฏ) RAISED MR SIGN..CIRCLED HUMAN FIGURE + {0x1F170, 0x1F171, prExtendedPictographic}, // E0.6 [2] (๐Ÿ…ฐ๏ธ..๐Ÿ…ฑ๏ธ) A button (blood type)..B button (blood type) + {0x1F17E, 0x1F17F, prExtendedPictographic}, // E0.6 [2] (๐Ÿ…พ๏ธ..๐Ÿ…ฟ๏ธ) O button (blood type)..P button + {0x1F18E, 0x1F18E, prExtendedPictographic}, // E0.6 [1] (๐Ÿ†Ž) AB button (blood type) + {0x1F191, 0x1F19A, prExtendedPictographic}, // E0.6 [10] (๐Ÿ†‘..๐Ÿ†š) CL button..VS button + {0x1F1AD, 0x1F1E5, prExtendedPictographic}, // E0.0 [57] (๐Ÿ†ญ..๐Ÿ‡ฅ) MASK WORK SYMBOL.. + {0x1F1E6, 0x1F1FF, prRegionalIndicator}, // So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + {0x1F201, 0x1F202, prExtendedPictographic}, // E0.6 [2] (๐Ÿˆ..๐Ÿˆ‚๏ธ) Japanese โ€œhereโ€ button..Japanese โ€œservice chargeโ€ button + {0x1F203, 0x1F20F, prExtendedPictographic}, // E0.0 [13] (๐Ÿˆƒ..๐Ÿˆ) .. + {0x1F21A, 0x1F21A, prExtendedPictographic}, // E0.6 [1] (๐Ÿˆš) Japanese โ€œfree of chargeโ€ button + {0x1F22F, 0x1F22F, prExtendedPictographic}, // E0.6 [1] (๐Ÿˆฏ) Japanese โ€œreservedโ€ button + {0x1F232, 0x1F23A, prExtendedPictographic}, // E0.6 [9] (๐Ÿˆฒ..๐Ÿˆบ) Japanese โ€œprohibitedโ€ button..Japanese โ€œopen for businessโ€ button + {0x1F23C, 0x1F23F, prExtendedPictographic}, // E0.0 [4] (๐Ÿˆผ..๐Ÿˆฟ) .. + {0x1F249, 0x1F24F, prExtendedPictographic}, // E0.0 [7] (๐Ÿ‰‰..๐Ÿ‰) .. + {0x1F250, 0x1F251, prExtendedPictographic}, // E0.6 [2] (๐Ÿ‰..๐Ÿ‰‘) Japanese โ€œbargainโ€ button..Japanese โ€œacceptableโ€ button + {0x1F252, 0x1F2FF, prExtendedPictographic}, // E0.0 [174] (๐Ÿ‰’..๐Ÿ‹ฟ) .. + {0x1F300, 0x1F30C, prExtendedPictographic}, // E0.6 [13] (๐ŸŒ€..๐ŸŒŒ) cyclone..milky way + {0x1F30D, 0x1F30E, prExtendedPictographic}, // E0.7 [2] (๐ŸŒ..๐ŸŒŽ) globe showing Europe-Africa..globe showing Americas + {0x1F30F, 0x1F30F, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ) globe showing Asia-Australia + {0x1F310, 0x1F310, prExtendedPictographic}, // E1.0 [1] (๐ŸŒ) globe with meridians + {0x1F311, 0x1F311, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ‘) new moon + {0x1F312, 0x1F312, prExtendedPictographic}, // E1.0 [1] (๐ŸŒ’) waxing crescent moon + {0x1F313, 0x1F315, prExtendedPictographic}, // E0.6 [3] (๐ŸŒ“..๐ŸŒ•) first quarter moon..full moon + {0x1F316, 0x1F318, prExtendedPictographic}, // E1.0 [3] (๐ŸŒ–..๐ŸŒ˜) waning gibbous moon..waning crescent moon + {0x1F319, 0x1F319, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ™) crescent moon + {0x1F31A, 0x1F31A, prExtendedPictographic}, // E1.0 [1] (๐ŸŒš) new moon face + {0x1F31B, 0x1F31B, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ›) first quarter moon face + {0x1F31C, 0x1F31C, prExtendedPictographic}, // E0.7 [1] (๐ŸŒœ) last quarter moon face + {0x1F31D, 0x1F31E, prExtendedPictographic}, // E1.0 [2] (๐ŸŒ..๐ŸŒž) full moon face..sun with face + {0x1F31F, 0x1F320, prExtendedPictographic}, // E0.6 [2] (๐ŸŒŸ..๐ŸŒ ) glowing star..shooting star + {0x1F321, 0x1F321, prExtendedPictographic}, // E0.7 [1] (๐ŸŒก๏ธ) thermometer + {0x1F322, 0x1F323, prExtendedPictographic}, // E0.0 [2] (๐ŸŒข..๐ŸŒฃ) BLACK DROPLET..WHITE SUN + {0x1F324, 0x1F32C, prExtendedPictographic}, // E0.7 [9] (๐ŸŒค๏ธ..๐ŸŒฌ๏ธ) sun behind small cloud..wind face + {0x1F32D, 0x1F32F, prExtendedPictographic}, // E1.0 [3] (๐ŸŒญ..๐ŸŒฏ) hot dog..burrito + {0x1F330, 0x1F331, prExtendedPictographic}, // E0.6 [2] (๐ŸŒฐ..๐ŸŒฑ) chestnut..seedling + {0x1F332, 0x1F333, prExtendedPictographic}, // E1.0 [2] (๐ŸŒฒ..๐ŸŒณ) evergreen tree..deciduous tree + {0x1F334, 0x1F335, prExtendedPictographic}, // E0.6 [2] (๐ŸŒด..๐ŸŒต) palm tree..cactus + {0x1F336, 0x1F336, prExtendedPictographic}, // E0.7 [1] (๐ŸŒถ๏ธ) hot pepper + {0x1F337, 0x1F34A, prExtendedPictographic}, // E0.6 [20] (๐ŸŒท..๐ŸŠ) tulip..tangerine + {0x1F34B, 0x1F34B, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‹) lemon + {0x1F34C, 0x1F34F, prExtendedPictographic}, // E0.6 [4] (๐ŸŒ..๐Ÿ) banana..green apple + {0x1F350, 0x1F350, prExtendedPictographic}, // E1.0 [1] (๐Ÿ) pear + {0x1F351, 0x1F37B, prExtendedPictographic}, // E0.6 [43] (๐Ÿ‘..๐Ÿป) peach..clinking beer mugs + {0x1F37C, 0x1F37C, prExtendedPictographic}, // E1.0 [1] (๐Ÿผ) baby bottle + {0x1F37D, 0x1F37D, prExtendedPictographic}, // E0.7 [1] (๐Ÿฝ๏ธ) fork and knife with plate + {0x1F37E, 0x1F37F, prExtendedPictographic}, // E1.0 [2] (๐Ÿพ..๐Ÿฟ) bottle with popping cork..popcorn + {0x1F380, 0x1F393, prExtendedPictographic}, // E0.6 [20] (๐ŸŽ€..๐ŸŽ“) ribbon..graduation cap + {0x1F394, 0x1F395, prExtendedPictographic}, // E0.0 [2] (๐ŸŽ”..๐ŸŽ•) HEART WITH TIP ON THE LEFT..BOUQUET OF FLOWERS + {0x1F396, 0x1F397, prExtendedPictographic}, // E0.7 [2] (๐ŸŽ–๏ธ..๐ŸŽ—๏ธ) military medal..reminder ribbon + {0x1F398, 0x1F398, prExtendedPictographic}, // E0.0 [1] (๐ŸŽ˜) MUSICAL KEYBOARD WITH JACKS + {0x1F399, 0x1F39B, prExtendedPictographic}, // E0.7 [3] (๐ŸŽ™๏ธ..๐ŸŽ›๏ธ) studio microphone..control knobs + {0x1F39C, 0x1F39D, prExtendedPictographic}, // E0.0 [2] (๐ŸŽœ..๐ŸŽ) BEAMED ASCENDING MUSICAL NOTES..BEAMED DESCENDING MUSICAL NOTES + {0x1F39E, 0x1F39F, prExtendedPictographic}, // E0.7 [2] (๐ŸŽž๏ธ..๐ŸŽŸ๏ธ) film frames..admission tickets + {0x1F3A0, 0x1F3C4, prExtendedPictographic}, // E0.6 [37] (๐ŸŽ ..๐Ÿ„) carousel horse..person surfing + {0x1F3C5, 0x1F3C5, prExtendedPictographic}, // E1.0 [1] (๐Ÿ…) sports medal + {0x1F3C6, 0x1F3C6, prExtendedPictographic}, // E0.6 [1] (๐Ÿ†) trophy + {0x1F3C7, 0x1F3C7, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‡) horse racing + {0x1F3C8, 0x1F3C8, prExtendedPictographic}, // E0.6 [1] (๐Ÿˆ) american football + {0x1F3C9, 0x1F3C9, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‰) rugby football + {0x1F3CA, 0x1F3CA, prExtendedPictographic}, // E0.6 [1] (๐ŸŠ) person swimming + {0x1F3CB, 0x1F3CE, prExtendedPictographic}, // E0.7 [4] (๐Ÿ‹๏ธ..๐ŸŽ๏ธ) person lifting weights..racing car + {0x1F3CF, 0x1F3D3, prExtendedPictographic}, // E1.0 [5] (๐Ÿ..๐Ÿ“) cricket game..ping pong + {0x1F3D4, 0x1F3DF, prExtendedPictographic}, // E0.7 [12] (๐Ÿ”๏ธ..๐ŸŸ๏ธ) snow-capped mountain..stadium + {0x1F3E0, 0x1F3E3, prExtendedPictographic}, // E0.6 [4] (๐Ÿ ..๐Ÿฃ) house..Japanese post office + {0x1F3E4, 0x1F3E4, prExtendedPictographic}, // E1.0 [1] (๐Ÿค) post office + {0x1F3E5, 0x1F3F0, prExtendedPictographic}, // E0.6 [12] (๐Ÿฅ..๐Ÿฐ) hospital..castle + {0x1F3F1, 0x1F3F2, prExtendedPictographic}, // E0.0 [2] (๐Ÿฑ..๐Ÿฒ) WHITE PENNANT..BLACK PENNANT + {0x1F3F3, 0x1F3F3, prExtendedPictographic}, // E0.7 [1] (๐Ÿณ๏ธ) white flag + {0x1F3F4, 0x1F3F4, prExtendedPictographic}, // E1.0 [1] (๐Ÿด) black flag + {0x1F3F5, 0x1F3F5, prExtendedPictographic}, // E0.7 [1] (๐Ÿต๏ธ) rosette + {0x1F3F6, 0x1F3F6, prExtendedPictographic}, // E0.0 [1] (๐Ÿถ) BLACK ROSETTE + {0x1F3F7, 0x1F3F7, prExtendedPictographic}, // E0.7 [1] (๐Ÿท๏ธ) label + {0x1F3F8, 0x1F3FA, prExtendedPictographic}, // E1.0 [3] (๐Ÿธ..๐Ÿบ) badminton..amphora + {0x1F3FB, 0x1F3FF, prExtend}, // Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 + {0x1F400, 0x1F407, prExtendedPictographic}, // E1.0 [8] (๐Ÿ€..๐Ÿ‡) rat..rabbit + {0x1F408, 0x1F408, prExtendedPictographic}, // E0.7 [1] (๐Ÿˆ) cat + {0x1F409, 0x1F40B, prExtendedPictographic}, // E1.0 [3] (๐Ÿ‰..๐Ÿ‹) dragon..whale + {0x1F40C, 0x1F40E, prExtendedPictographic}, // E0.6 [3] (๐ŸŒ..๐ŸŽ) snail..horse + {0x1F40F, 0x1F410, prExtendedPictographic}, // E1.0 [2] (๐Ÿ..๐Ÿ) ram..goat + {0x1F411, 0x1F412, prExtendedPictographic}, // E0.6 [2] (๐Ÿ‘..๐Ÿ’) ewe..monkey + {0x1F413, 0x1F413, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“) rooster + {0x1F414, 0x1F414, prExtendedPictographic}, // E0.6 [1] (๐Ÿ”) chicken + {0x1F415, 0x1F415, prExtendedPictographic}, // E0.7 [1] (๐Ÿ•) dog + {0x1F416, 0x1F416, prExtendedPictographic}, // E1.0 [1] (๐Ÿ–) pig + {0x1F417, 0x1F429, prExtendedPictographic}, // E0.6 [19] (๐Ÿ—..๐Ÿฉ) boar..poodle + {0x1F42A, 0x1F42A, prExtendedPictographic}, // E1.0 [1] (๐Ÿช) camel + {0x1F42B, 0x1F43E, prExtendedPictographic}, // E0.6 [20] (๐Ÿซ..๐Ÿพ) two-hump camel..paw prints + {0x1F43F, 0x1F43F, prExtendedPictographic}, // E0.7 [1] (๐Ÿฟ๏ธ) chipmunk + {0x1F440, 0x1F440, prExtendedPictographic}, // E0.6 [1] (๐Ÿ‘€) eyes + {0x1F441, 0x1F441, prExtendedPictographic}, // E0.7 [1] (๐Ÿ‘๏ธ) eye + {0x1F442, 0x1F464, prExtendedPictographic}, // E0.6 [35] (๐Ÿ‘‚..๐Ÿ‘ค) ear..bust in silhouette + {0x1F465, 0x1F465, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‘ฅ) busts in silhouette + {0x1F466, 0x1F46B, prExtendedPictographic}, // E0.6 [6] (๐Ÿ‘ฆ..๐Ÿ‘ซ) boy..woman and man holding hands + {0x1F46C, 0x1F46D, prExtendedPictographic}, // E1.0 [2] (๐Ÿ‘ฌ..๐Ÿ‘ญ) men holding hands..women holding hands + {0x1F46E, 0x1F4AC, prExtendedPictographic}, // E0.6 [63] (๐Ÿ‘ฎ..๐Ÿ’ฌ) police officer..speech balloon + {0x1F4AD, 0x1F4AD, prExtendedPictographic}, // E1.0 [1] (๐Ÿ’ญ) thought balloon + {0x1F4AE, 0x1F4B5, prExtendedPictographic}, // E0.6 [8] (๐Ÿ’ฎ..๐Ÿ’ต) white flower..dollar banknote + {0x1F4B6, 0x1F4B7, prExtendedPictographic}, // E1.0 [2] (๐Ÿ’ถ..๐Ÿ’ท) euro banknote..pound banknote + {0x1F4B8, 0x1F4EB, prExtendedPictographic}, // E0.6 [52] (๐Ÿ’ธ..๐Ÿ“ซ) money with wings..closed mailbox with raised flag + {0x1F4EC, 0x1F4ED, prExtendedPictographic}, // E0.7 [2] (๐Ÿ“ฌ..๐Ÿ“ญ) open mailbox with raised flag..open mailbox with lowered flag + {0x1F4EE, 0x1F4EE, prExtendedPictographic}, // E0.6 [1] (๐Ÿ“ฎ) postbox + {0x1F4EF, 0x1F4EF, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“ฏ) postal horn + {0x1F4F0, 0x1F4F4, prExtendedPictographic}, // E0.6 [5] (๐Ÿ“ฐ..๐Ÿ“ด) newspaper..mobile phone off + {0x1F4F5, 0x1F4F5, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“ต) no mobile phones + {0x1F4F6, 0x1F4F7, prExtendedPictographic}, // E0.6 [2] (๐Ÿ“ถ..๐Ÿ“ท) antenna bars..camera + {0x1F4F8, 0x1F4F8, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“ธ) camera with flash + {0x1F4F9, 0x1F4FC, prExtendedPictographic}, // E0.6 [4] (๐Ÿ“น..๐Ÿ“ผ) video camera..videocassette + {0x1F4FD, 0x1F4FD, prExtendedPictographic}, // E0.7 [1] (๐Ÿ“ฝ๏ธ) film projector + {0x1F4FE, 0x1F4FE, prExtendedPictographic}, // E0.0 [1] (๐Ÿ“พ) PORTABLE STEREO + {0x1F4FF, 0x1F502, prExtendedPictographic}, // E1.0 [4] (๐Ÿ“ฟ..๐Ÿ”‚) prayer beads..repeat single button + {0x1F503, 0x1F503, prExtendedPictographic}, // E0.6 [1] (๐Ÿ”ƒ) clockwise vertical arrows + {0x1F504, 0x1F507, prExtendedPictographic}, // E1.0 [4] (๐Ÿ”„..๐Ÿ”‡) counterclockwise arrows button..muted speaker + {0x1F508, 0x1F508, prExtendedPictographic}, // E0.7 [1] (๐Ÿ”ˆ) speaker low volume + {0x1F509, 0x1F509, prExtendedPictographic}, // E1.0 [1] (๐Ÿ”‰) speaker medium volume + {0x1F50A, 0x1F514, prExtendedPictographic}, // E0.6 [11] (๐Ÿ”Š..๐Ÿ””) speaker high volume..bell + {0x1F515, 0x1F515, prExtendedPictographic}, // E1.0 [1] (๐Ÿ”•) bell with slash + {0x1F516, 0x1F52B, prExtendedPictographic}, // E0.6 [22] (๐Ÿ”–..๐Ÿ”ซ) bookmark..water pistol + {0x1F52C, 0x1F52D, prExtendedPictographic}, // E1.0 [2] (๐Ÿ”ฌ..๐Ÿ”ญ) microscope..telescope + {0x1F52E, 0x1F53D, prExtendedPictographic}, // E0.6 [16] (๐Ÿ”ฎ..๐Ÿ”ฝ) crystal ball..downwards button + {0x1F546, 0x1F548, prExtendedPictographic}, // E0.0 [3] (๐Ÿ•†..๐Ÿ•ˆ) WHITE LATIN CROSS..CELTIC CROSS + {0x1F549, 0x1F54A, prExtendedPictographic}, // E0.7 [2] (๐Ÿ•‰๏ธ..๐Ÿ•Š๏ธ) om..dove + {0x1F54B, 0x1F54E, prExtendedPictographic}, // E1.0 [4] (๐Ÿ•‹..๐Ÿ•Ž) kaaba..menorah + {0x1F54F, 0x1F54F, prExtendedPictographic}, // E0.0 [1] (๐Ÿ•) BOWL OF HYGIEIA + {0x1F550, 0x1F55B, prExtendedPictographic}, // E0.6 [12] (๐Ÿ•..๐Ÿ•›) one oโ€™clock..twelve oโ€™clock + {0x1F55C, 0x1F567, prExtendedPictographic}, // E0.7 [12] (๐Ÿ•œ..๐Ÿ•ง) one-thirty..twelve-thirty + {0x1F568, 0x1F56E, prExtendedPictographic}, // E0.0 [7] (๐Ÿ•จ..๐Ÿ•ฎ) RIGHT SPEAKER..BOOK + {0x1F56F, 0x1F570, prExtendedPictographic}, // E0.7 [2] (๐Ÿ•ฏ๏ธ..๐Ÿ•ฐ๏ธ) candle..mantelpiece clock + {0x1F571, 0x1F572, prExtendedPictographic}, // E0.0 [2] (๐Ÿ•ฑ..๐Ÿ•ฒ) BLACK SKULL AND CROSSBONES..NO PIRACY + {0x1F573, 0x1F579, prExtendedPictographic}, // E0.7 [7] (๐Ÿ•ณ๏ธ..๐Ÿ•น๏ธ) hole..joystick + {0x1F57A, 0x1F57A, prExtendedPictographic}, // E3.0 [1] (๐Ÿ•บ) man dancing + {0x1F57B, 0x1F586, prExtendedPictographic}, // E0.0 [12] (๐Ÿ•ป..๐Ÿ–†) LEFT HAND TELEPHONE RECEIVER..PEN OVER STAMPED ENVELOPE + {0x1F587, 0x1F587, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–‡๏ธ) linked paperclips + {0x1F588, 0x1F589, prExtendedPictographic}, // E0.0 [2] (๐Ÿ–ˆ..๐Ÿ–‰) BLACK PUSHPIN..LOWER LEFT PENCIL + {0x1F58A, 0x1F58D, prExtendedPictographic}, // E0.7 [4] (๐Ÿ–Š๏ธ..๐Ÿ–๏ธ) pen..crayon + {0x1F58E, 0x1F58F, prExtendedPictographic}, // E0.0 [2] (๐Ÿ–Ž..๐Ÿ–) LEFT WRITING HAND..TURNED OK HAND SIGN + {0x1F590, 0x1F590, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–๏ธ) hand with fingers splayed + {0x1F591, 0x1F594, prExtendedPictographic}, // E0.0 [4] (๐Ÿ–‘..๐Ÿ–”) REVERSED RAISED HAND WITH FINGERS SPLAYED..REVERSED VICTORY HAND + {0x1F595, 0x1F596, prExtendedPictographic}, // E1.0 [2] (๐Ÿ–•..๐Ÿ––) middle finger..vulcan salute + {0x1F597, 0x1F5A3, prExtendedPictographic}, // E0.0 [13] (๐Ÿ–—..๐Ÿ–ฃ) WHITE DOWN POINTING LEFT HAND INDEX..BLACK DOWN POINTING BACKHAND INDEX + {0x1F5A4, 0x1F5A4, prExtendedPictographic}, // E3.0 [1] (๐Ÿ–ค) black heart + {0x1F5A5, 0x1F5A5, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–ฅ๏ธ) desktop computer + {0x1F5A6, 0x1F5A7, prExtendedPictographic}, // E0.0 [2] (๐Ÿ–ฆ..๐Ÿ–ง) KEYBOARD AND MOUSE..THREE NETWORKED COMPUTERS + {0x1F5A8, 0x1F5A8, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–จ๏ธ) printer + {0x1F5A9, 0x1F5B0, prExtendedPictographic}, // E0.0 [8] (๐Ÿ–ฉ..๐Ÿ–ฐ) POCKET CALCULATOR..TWO BUTTON MOUSE + {0x1F5B1, 0x1F5B2, prExtendedPictographic}, // E0.7 [2] (๐Ÿ–ฑ๏ธ..๐Ÿ–ฒ๏ธ) computer mouse..trackball + {0x1F5B3, 0x1F5BB, prExtendedPictographic}, // E0.0 [9] (๐Ÿ–ณ..๐Ÿ–ป) OLD PERSONAL COMPUTER..DOCUMENT WITH PICTURE + {0x1F5BC, 0x1F5BC, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–ผ๏ธ) framed picture + {0x1F5BD, 0x1F5C1, prExtendedPictographic}, // E0.0 [5] (๐Ÿ–ฝ..๐Ÿ—) FRAME WITH TILES..OPEN FOLDER + {0x1F5C2, 0x1F5C4, prExtendedPictographic}, // E0.7 [3] (๐Ÿ—‚๏ธ..๐Ÿ—„๏ธ) card index dividers..file cabinet + {0x1F5C5, 0x1F5D0, prExtendedPictographic}, // E0.0 [12] (๐Ÿ—…..๐Ÿ—) EMPTY NOTE..PAGES + {0x1F5D1, 0x1F5D3, prExtendedPictographic}, // E0.7 [3] (๐Ÿ—‘๏ธ..๐Ÿ—“๏ธ) wastebasket..spiral calendar + {0x1F5D4, 0x1F5DB, prExtendedPictographic}, // E0.0 [8] (๐Ÿ—”..๐Ÿ—›) DESKTOP WINDOW..DECREASE FONT SIZE SYMBOL + {0x1F5DC, 0x1F5DE, prExtendedPictographic}, // E0.7 [3] (๐Ÿ—œ๏ธ..๐Ÿ—ž๏ธ) clamp..rolled-up newspaper + {0x1F5DF, 0x1F5E0, prExtendedPictographic}, // E0.0 [2] (๐Ÿ—Ÿ..๐Ÿ— ) PAGE WITH CIRCLED TEXT..STOCK CHART + {0x1F5E1, 0x1F5E1, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ก๏ธ) dagger + {0x1F5E2, 0x1F5E2, prExtendedPictographic}, // E0.0 [1] (๐Ÿ—ข) LIPS + {0x1F5E3, 0x1F5E3, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ฃ๏ธ) speaking head + {0x1F5E4, 0x1F5E7, prExtendedPictographic}, // E0.0 [4] (๐Ÿ—ค..๐Ÿ—ง) THREE RAYS ABOVE..THREE RAYS RIGHT + {0x1F5E8, 0x1F5E8, prExtendedPictographic}, // E2.0 [1] (๐Ÿ—จ๏ธ) left speech bubble + {0x1F5E9, 0x1F5EE, prExtendedPictographic}, // E0.0 [6] (๐Ÿ—ฉ..๐Ÿ—ฎ) RIGHT SPEECH BUBBLE..LEFT ANGER BUBBLE + {0x1F5EF, 0x1F5EF, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ฏ๏ธ) right anger bubble + {0x1F5F0, 0x1F5F2, prExtendedPictographic}, // E0.0 [3] (๐Ÿ—ฐ..๐Ÿ—ฒ) MOOD BUBBLE..LIGHTNING MOOD + {0x1F5F3, 0x1F5F3, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ณ๏ธ) ballot box with ballot + {0x1F5F4, 0x1F5F9, prExtendedPictographic}, // E0.0 [6] (๐Ÿ—ด..๐Ÿ—น) BALLOT SCRIPT X..BALLOT BOX WITH BOLD CHECK + {0x1F5FA, 0x1F5FA, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—บ๏ธ) world map + {0x1F5FB, 0x1F5FF, prExtendedPictographic}, // E0.6 [5] (๐Ÿ—ป..๐Ÿ—ฟ) mount fuji..moai + {0x1F600, 0x1F600, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜€) grinning face + {0x1F601, 0x1F606, prExtendedPictographic}, // E0.6 [6] (๐Ÿ˜..๐Ÿ˜†) beaming face with smiling eyes..grinning squinting face + {0x1F607, 0x1F608, prExtendedPictographic}, // E1.0 [2] (๐Ÿ˜‡..๐Ÿ˜ˆ) smiling face with halo..smiling face with horns + {0x1F609, 0x1F60D, prExtendedPictographic}, // E0.6 [5] (๐Ÿ˜‰..๐Ÿ˜) winking face..smiling face with heart-eyes + {0x1F60E, 0x1F60E, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜Ž) smiling face with sunglasses + {0x1F60F, 0x1F60F, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜) smirking face + {0x1F610, 0x1F610, prExtendedPictographic}, // E0.7 [1] (๐Ÿ˜) neutral face + {0x1F611, 0x1F611, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜‘) expressionless face + {0x1F612, 0x1F614, prExtendedPictographic}, // E0.6 [3] (๐Ÿ˜’..๐Ÿ˜”) unamused face..pensive face + {0x1F615, 0x1F615, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜•) confused face + {0x1F616, 0x1F616, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜–) confounded face + {0x1F617, 0x1F617, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜—) kissing face + {0x1F618, 0x1F618, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜˜) face blowing a kiss + {0x1F619, 0x1F619, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜™) kissing face with smiling eyes + {0x1F61A, 0x1F61A, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜š) kissing face with closed eyes + {0x1F61B, 0x1F61B, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜›) face with tongue + {0x1F61C, 0x1F61E, prExtendedPictographic}, // E0.6 [3] (๐Ÿ˜œ..๐Ÿ˜ž) winking face with tongue..disappointed face + {0x1F61F, 0x1F61F, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜Ÿ) worried face + {0x1F620, 0x1F625, prExtendedPictographic}, // E0.6 [6] (๐Ÿ˜ ..๐Ÿ˜ฅ) angry face..sad but relieved face + {0x1F626, 0x1F627, prExtendedPictographic}, // E1.0 [2] (๐Ÿ˜ฆ..๐Ÿ˜ง) frowning face with open mouth..anguished face + {0x1F628, 0x1F62B, prExtendedPictographic}, // E0.6 [4] (๐Ÿ˜จ..๐Ÿ˜ซ) fearful face..tired face + {0x1F62C, 0x1F62C, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜ฌ) grimacing face + {0x1F62D, 0x1F62D, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜ญ) loudly crying face + {0x1F62E, 0x1F62F, prExtendedPictographic}, // E1.0 [2] (๐Ÿ˜ฎ..๐Ÿ˜ฏ) face with open mouth..hushed face + {0x1F630, 0x1F633, prExtendedPictographic}, // E0.6 [4] (๐Ÿ˜ฐ..๐Ÿ˜ณ) anxious face with sweat..flushed face + {0x1F634, 0x1F634, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜ด) sleeping face + {0x1F635, 0x1F635, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜ต) face with crossed-out eyes + {0x1F636, 0x1F636, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜ถ) face without mouth + {0x1F637, 0x1F640, prExtendedPictographic}, // E0.6 [10] (๐Ÿ˜ท..๐Ÿ™€) face with medical mask..weary cat + {0x1F641, 0x1F644, prExtendedPictographic}, // E1.0 [4] (๐Ÿ™..๐Ÿ™„) slightly frowning face..face with rolling eyes + {0x1F645, 0x1F64F, prExtendedPictographic}, // E0.6 [11] (๐Ÿ™…..๐Ÿ™) person gesturing NO..folded hands + {0x1F680, 0x1F680, prExtendedPictographic}, // E0.6 [1] (๐Ÿš€) rocket + {0x1F681, 0x1F682, prExtendedPictographic}, // E1.0 [2] (๐Ÿš..๐Ÿš‚) helicopter..locomotive + {0x1F683, 0x1F685, prExtendedPictographic}, // E0.6 [3] (๐Ÿšƒ..๐Ÿš…) railway car..bullet train + {0x1F686, 0x1F686, prExtendedPictographic}, // E1.0 [1] (๐Ÿš†) train + {0x1F687, 0x1F687, prExtendedPictographic}, // E0.6 [1] (๐Ÿš‡) metro + {0x1F688, 0x1F688, prExtendedPictographic}, // E1.0 [1] (๐Ÿšˆ) light rail + {0x1F689, 0x1F689, prExtendedPictographic}, // E0.6 [1] (๐Ÿš‰) station + {0x1F68A, 0x1F68B, prExtendedPictographic}, // E1.0 [2] (๐ŸšŠ..๐Ÿš‹) tram..tram car + {0x1F68C, 0x1F68C, prExtendedPictographic}, // E0.6 [1] (๐ŸšŒ) bus + {0x1F68D, 0x1F68D, prExtendedPictographic}, // E0.7 [1] (๐Ÿš) oncoming bus + {0x1F68E, 0x1F68E, prExtendedPictographic}, // E1.0 [1] (๐ŸšŽ) trolleybus + {0x1F68F, 0x1F68F, prExtendedPictographic}, // E0.6 [1] (๐Ÿš) bus stop + {0x1F690, 0x1F690, prExtendedPictographic}, // E1.0 [1] (๐Ÿš) minibus + {0x1F691, 0x1F693, prExtendedPictographic}, // E0.6 [3] (๐Ÿš‘..๐Ÿš“) ambulance..police car + {0x1F694, 0x1F694, prExtendedPictographic}, // E0.7 [1] (๐Ÿš”) oncoming police car + {0x1F695, 0x1F695, prExtendedPictographic}, // E0.6 [1] (๐Ÿš•) taxi + {0x1F696, 0x1F696, prExtendedPictographic}, // E1.0 [1] (๐Ÿš–) oncoming taxi + {0x1F697, 0x1F697, prExtendedPictographic}, // E0.6 [1] (๐Ÿš—) automobile + {0x1F698, 0x1F698, prExtendedPictographic}, // E0.7 [1] (๐Ÿš˜) oncoming automobile + {0x1F699, 0x1F69A, prExtendedPictographic}, // E0.6 [2] (๐Ÿš™..๐Ÿšš) sport utility vehicle..delivery truck + {0x1F69B, 0x1F6A1, prExtendedPictographic}, // E1.0 [7] (๐Ÿš›..๐Ÿšก) articulated lorry..aerial tramway + {0x1F6A2, 0x1F6A2, prExtendedPictographic}, // E0.6 [1] (๐Ÿšข) ship + {0x1F6A3, 0x1F6A3, prExtendedPictographic}, // E1.0 [1] (๐Ÿšฃ) person rowing boat + {0x1F6A4, 0x1F6A5, prExtendedPictographic}, // E0.6 [2] (๐Ÿšค..๐Ÿšฅ) speedboat..horizontal traffic light + {0x1F6A6, 0x1F6A6, prExtendedPictographic}, // E1.0 [1] (๐Ÿšฆ) vertical traffic light + {0x1F6A7, 0x1F6AD, prExtendedPictographic}, // E0.6 [7] (๐Ÿšง..๐Ÿšญ) construction..no smoking + {0x1F6AE, 0x1F6B1, prExtendedPictographic}, // E1.0 [4] (๐Ÿšฎ..๐Ÿšฑ) litter in bin sign..non-potable water + {0x1F6B2, 0x1F6B2, prExtendedPictographic}, // E0.6 [1] (๐Ÿšฒ) bicycle + {0x1F6B3, 0x1F6B5, prExtendedPictographic}, // E1.0 [3] (๐Ÿšณ..๐Ÿšต) no bicycles..person mountain biking + {0x1F6B6, 0x1F6B6, prExtendedPictographic}, // E0.6 [1] (๐Ÿšถ) person walking + {0x1F6B7, 0x1F6B8, prExtendedPictographic}, // E1.0 [2] (๐Ÿšท..๐Ÿšธ) no pedestrians..children crossing + {0x1F6B9, 0x1F6BE, prExtendedPictographic}, // E0.6 [6] (๐Ÿšน..๐Ÿšพ) menโ€™s room..water closet + {0x1F6BF, 0x1F6BF, prExtendedPictographic}, // E1.0 [1] (๐Ÿšฟ) shower + {0x1F6C0, 0x1F6C0, prExtendedPictographic}, // E0.6 [1] (๐Ÿ›€) person taking bath + {0x1F6C1, 0x1F6C5, prExtendedPictographic}, // E1.0 [5] (๐Ÿ›..๐Ÿ›…) bathtub..left luggage + {0x1F6C6, 0x1F6CA, prExtendedPictographic}, // E0.0 [5] (๐Ÿ›†..๐Ÿ›Š) TRIANGLE WITH ROUNDED CORNERS..GIRLS SYMBOL + {0x1F6CB, 0x1F6CB, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›‹๏ธ) couch and lamp + {0x1F6CC, 0x1F6CC, prExtendedPictographic}, // E1.0 [1] (๐Ÿ›Œ) person in bed + {0x1F6CD, 0x1F6CF, prExtendedPictographic}, // E0.7 [3] (๐Ÿ›๏ธ..๐Ÿ›๏ธ) shopping bags..bed + {0x1F6D0, 0x1F6D0, prExtendedPictographic}, // E1.0 [1] (๐Ÿ›) place of worship + {0x1F6D1, 0x1F6D2, prExtendedPictographic}, // E3.0 [2] (๐Ÿ›‘..๐Ÿ›’) stop sign..shopping cart + {0x1F6D3, 0x1F6D4, prExtendedPictographic}, // E0.0 [2] (๐Ÿ›“..๐Ÿ›”) STUPA..PAGODA + {0x1F6D5, 0x1F6D5, prExtendedPictographic}, // E12.0 [1] (๐Ÿ›•) hindu temple + {0x1F6D6, 0x1F6D7, prExtendedPictographic}, // E13.0 [2] (๐Ÿ›–..๐Ÿ›—) hut..elevator + {0x1F6D8, 0x1F6DB, prExtendedPictographic}, // E0.0 [4] (๐Ÿ›˜..๐Ÿ››) .. + {0x1F6DC, 0x1F6DC, prExtendedPictographic}, // E15.0 [1] (๐Ÿ›œ) wireless + {0x1F6DD, 0x1F6DF, prExtendedPictographic}, // E14.0 [3] (๐Ÿ›..๐Ÿ›Ÿ) playground slide..ring buoy + {0x1F6E0, 0x1F6E5, prExtendedPictographic}, // E0.7 [6] (๐Ÿ› ๏ธ..๐Ÿ›ฅ๏ธ) hammer and wrench..motor boat + {0x1F6E6, 0x1F6E8, prExtendedPictographic}, // E0.0 [3] (๐Ÿ›ฆ..๐Ÿ›จ) UP-POINTING MILITARY AIRPLANE..UP-POINTING SMALL AIRPLANE + {0x1F6E9, 0x1F6E9, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›ฉ๏ธ) small airplane + {0x1F6EA, 0x1F6EA, prExtendedPictographic}, // E0.0 [1] (๐Ÿ›ช) NORTHEAST-POINTING AIRPLANE + {0x1F6EB, 0x1F6EC, prExtendedPictographic}, // E1.0 [2] (๐Ÿ›ซ..๐Ÿ›ฌ) airplane departure..airplane arrival + {0x1F6ED, 0x1F6EF, prExtendedPictographic}, // E0.0 [3] (๐Ÿ›ญ..๐Ÿ›ฏ) .. + {0x1F6F0, 0x1F6F0, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›ฐ๏ธ) satellite + {0x1F6F1, 0x1F6F2, prExtendedPictographic}, // E0.0 [2] (๐Ÿ›ฑ..๐Ÿ›ฒ) ONCOMING FIRE ENGINE..DIESEL LOCOMOTIVE + {0x1F6F3, 0x1F6F3, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›ณ๏ธ) passenger ship + {0x1F6F4, 0x1F6F6, prExtendedPictographic}, // E3.0 [3] (๐Ÿ›ด..๐Ÿ›ถ) kick scooter..canoe + {0x1F6F7, 0x1F6F8, prExtendedPictographic}, // E5.0 [2] (๐Ÿ›ท..๐Ÿ›ธ) sled..flying saucer + {0x1F6F9, 0x1F6F9, prExtendedPictographic}, // E11.0 [1] (๐Ÿ›น) skateboard + {0x1F6FA, 0x1F6FA, prExtendedPictographic}, // E12.0 [1] (๐Ÿ›บ) auto rickshaw + {0x1F6FB, 0x1F6FC, prExtendedPictographic}, // E13.0 [2] (๐Ÿ›ป..๐Ÿ›ผ) pickup truck..roller skate + {0x1F6FD, 0x1F6FF, prExtendedPictographic}, // E0.0 [3] (๐Ÿ›ฝ..๐Ÿ›ฟ) .. + {0x1F774, 0x1F77F, prExtendedPictographic}, // E0.0 [12] (๐Ÿด..๐Ÿฟ) LOT OF FORTUNE..ORCUS + {0x1F7D5, 0x1F7DF, prExtendedPictographic}, // E0.0 [11] (๐ŸŸ•..๐ŸŸŸ) CIRCLED TRIANGLE.. + {0x1F7E0, 0x1F7EB, prExtendedPictographic}, // E12.0 [12] (๐ŸŸ ..๐ŸŸซ) orange circle..brown square + {0x1F7EC, 0x1F7EF, prExtendedPictographic}, // E0.0 [4] (๐ŸŸฌ..๐ŸŸฏ) .. + {0x1F7F0, 0x1F7F0, prExtendedPictographic}, // E14.0 [1] (๐ŸŸฐ) heavy equals sign + {0x1F7F1, 0x1F7FF, prExtendedPictographic}, // E0.0 [15] (๐ŸŸฑ..๐ŸŸฟ) .. + {0x1F80C, 0x1F80F, prExtendedPictographic}, // E0.0 [4] (๐Ÿ Œ..๐Ÿ ) .. + {0x1F848, 0x1F84F, prExtendedPictographic}, // E0.0 [8] (๐Ÿกˆ..๐Ÿก) .. + {0x1F85A, 0x1F85F, prExtendedPictographic}, // E0.0 [6] (๐Ÿกš..๐ŸกŸ) .. + {0x1F888, 0x1F88F, prExtendedPictographic}, // E0.0 [8] (๐Ÿขˆ..๐Ÿข) .. + {0x1F8AE, 0x1F8FF, prExtendedPictographic}, // E0.0 [82] (๐Ÿขฎ..๐Ÿฃฟ) .. + {0x1F90C, 0x1F90C, prExtendedPictographic}, // E13.0 [1] (๐ŸคŒ) pinched fingers + {0x1F90D, 0x1F90F, prExtendedPictographic}, // E12.0 [3] (๐Ÿค..๐Ÿค) white heart..pinching hand + {0x1F910, 0x1F918, prExtendedPictographic}, // E1.0 [9] (๐Ÿค..๐Ÿค˜) zipper-mouth face..sign of the horns + {0x1F919, 0x1F91E, prExtendedPictographic}, // E3.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers + {0x1F91F, 0x1F91F, prExtendedPictographic}, // E5.0 [1] (๐ŸคŸ) love-you gesture + {0x1F920, 0x1F927, prExtendedPictographic}, // E3.0 [8] (๐Ÿค ..๐Ÿคง) cowboy hat face..sneezing face + {0x1F928, 0x1F92F, prExtendedPictographic}, // E5.0 [8] (๐Ÿคจ..๐Ÿคฏ) face with raised eyebrow..exploding head + {0x1F930, 0x1F930, prExtendedPictographic}, // E3.0 [1] (๐Ÿคฐ) pregnant woman + {0x1F931, 0x1F932, prExtendedPictographic}, // E5.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together + {0x1F933, 0x1F93A, prExtendedPictographic}, // E3.0 [8] (๐Ÿคณ..๐Ÿคบ) selfie..person fencing + {0x1F93C, 0x1F93E, prExtendedPictographic}, // E3.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball + {0x1F93F, 0x1F93F, prExtendedPictographic}, // E12.0 [1] (๐Ÿคฟ) diving mask + {0x1F940, 0x1F945, prExtendedPictographic}, // E3.0 [6] (๐Ÿฅ€..๐Ÿฅ…) wilted flower..goal net + {0x1F947, 0x1F94B, prExtendedPictographic}, // E3.0 [5] (๐Ÿฅ‡..๐Ÿฅ‹) 1st place medal..martial arts uniform + {0x1F94C, 0x1F94C, prExtendedPictographic}, // E5.0 [1] (๐ŸฅŒ) curling stone + {0x1F94D, 0x1F94F, prExtendedPictographic}, // E11.0 [3] (๐Ÿฅ..๐Ÿฅ) lacrosse..flying disc + {0x1F950, 0x1F95E, prExtendedPictographic}, // E3.0 [15] (๐Ÿฅ..๐Ÿฅž) croissant..pancakes + {0x1F95F, 0x1F96B, prExtendedPictographic}, // E5.0 [13] (๐ŸฅŸ..๐Ÿฅซ) dumpling..canned food + {0x1F96C, 0x1F970, prExtendedPictographic}, // E11.0 [5] (๐Ÿฅฌ..๐Ÿฅฐ) leafy green..smiling face with hearts + {0x1F971, 0x1F971, prExtendedPictographic}, // E12.0 [1] (๐Ÿฅฑ) yawning face + {0x1F972, 0x1F972, prExtendedPictographic}, // E13.0 [1] (๐Ÿฅฒ) smiling face with tear + {0x1F973, 0x1F976, prExtendedPictographic}, // E11.0 [4] (๐Ÿฅณ..๐Ÿฅถ) partying face..cold face + {0x1F977, 0x1F978, prExtendedPictographic}, // E13.0 [2] (๐Ÿฅท..๐Ÿฅธ) ninja..disguised face + {0x1F979, 0x1F979, prExtendedPictographic}, // E14.0 [1] (๐Ÿฅน) face holding back tears + {0x1F97A, 0x1F97A, prExtendedPictographic}, // E11.0 [1] (๐Ÿฅบ) pleading face + {0x1F97B, 0x1F97B, prExtendedPictographic}, // E12.0 [1] (๐Ÿฅป) sari + {0x1F97C, 0x1F97F, prExtendedPictographic}, // E11.0 [4] (๐Ÿฅผ..๐Ÿฅฟ) lab coat..flat shoe + {0x1F980, 0x1F984, prExtendedPictographic}, // E1.0 [5] (๐Ÿฆ€..๐Ÿฆ„) crab..unicorn + {0x1F985, 0x1F991, prExtendedPictographic}, // E3.0 [13] (๐Ÿฆ…..๐Ÿฆ‘) eagle..squid + {0x1F992, 0x1F997, prExtendedPictographic}, // E5.0 [6] (๐Ÿฆ’..๐Ÿฆ—) giraffe..cricket + {0x1F998, 0x1F9A2, prExtendedPictographic}, // E11.0 [11] (๐Ÿฆ˜..๐Ÿฆข) kangaroo..swan + {0x1F9A3, 0x1F9A4, prExtendedPictographic}, // E13.0 [2] (๐Ÿฆฃ..๐Ÿฆค) mammoth..dodo + {0x1F9A5, 0x1F9AA, prExtendedPictographic}, // E12.0 [6] (๐Ÿฆฅ..๐Ÿฆช) sloth..oyster + {0x1F9AB, 0x1F9AD, prExtendedPictographic}, // E13.0 [3] (๐Ÿฆซ..๐Ÿฆญ) beaver..seal + {0x1F9AE, 0x1F9AF, prExtendedPictographic}, // E12.0 [2] (๐Ÿฆฎ..๐Ÿฆฏ) guide dog..white cane + {0x1F9B0, 0x1F9B9, prExtendedPictographic}, // E11.0 [10] (๐Ÿฆฐ..๐Ÿฆน) red hair..supervillain + {0x1F9BA, 0x1F9BF, prExtendedPictographic}, // E12.0 [6] (๐Ÿฆบ..๐Ÿฆฟ) safety vest..mechanical leg + {0x1F9C0, 0x1F9C0, prExtendedPictographic}, // E1.0 [1] (๐Ÿง€) cheese wedge + {0x1F9C1, 0x1F9C2, prExtendedPictographic}, // E11.0 [2] (๐Ÿง..๐Ÿง‚) cupcake..salt + {0x1F9C3, 0x1F9CA, prExtendedPictographic}, // E12.0 [8] (๐Ÿงƒ..๐ŸงŠ) beverage box..ice + {0x1F9CB, 0x1F9CB, prExtendedPictographic}, // E13.0 [1] (๐Ÿง‹) bubble tea + {0x1F9CC, 0x1F9CC, prExtendedPictographic}, // E14.0 [1] (๐ŸงŒ) troll + {0x1F9CD, 0x1F9CF, prExtendedPictographic}, // E12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person + {0x1F9D0, 0x1F9E6, prExtendedPictographic}, // E5.0 [23] (๐Ÿง..๐Ÿงฆ) face with monocle..socks + {0x1F9E7, 0x1F9FF, prExtendedPictographic}, // E11.0 [25] (๐Ÿงง..๐Ÿงฟ) red envelope..nazar amulet + {0x1FA00, 0x1FA6F, prExtendedPictographic}, // E0.0 [112] (๐Ÿจ€..๐Ÿฉฏ) NEUTRAL CHESS KING.. + {0x1FA70, 0x1FA73, prExtendedPictographic}, // E12.0 [4] (๐Ÿฉฐ..๐Ÿฉณ) ballet shoes..shorts + {0x1FA74, 0x1FA74, prExtendedPictographic}, // E13.0 [1] (๐Ÿฉด) thong sandal + {0x1FA75, 0x1FA77, prExtendedPictographic}, // E15.0 [3] (๐Ÿฉต..๐Ÿฉท) light blue heart..pink heart + {0x1FA78, 0x1FA7A, prExtendedPictographic}, // E12.0 [3] (๐Ÿฉธ..๐Ÿฉบ) drop of blood..stethoscope + {0x1FA7B, 0x1FA7C, prExtendedPictographic}, // E14.0 [2] (๐Ÿฉป..๐Ÿฉผ) x-ray..crutch + {0x1FA7D, 0x1FA7F, prExtendedPictographic}, // E0.0 [3] (๐Ÿฉฝ..๐Ÿฉฟ) .. + {0x1FA80, 0x1FA82, prExtendedPictographic}, // E12.0 [3] (๐Ÿช€..๐Ÿช‚) yo-yo..parachute + {0x1FA83, 0x1FA86, prExtendedPictographic}, // E13.0 [4] (๐Ÿชƒ..๐Ÿช†) boomerang..nesting dolls + {0x1FA87, 0x1FA88, prExtendedPictographic}, // E15.0 [2] (๐Ÿช‡..๐Ÿชˆ) maracas..flute + {0x1FA89, 0x1FA8F, prExtendedPictographic}, // E0.0 [7] (๐Ÿช‰..๐Ÿช) .. + {0x1FA90, 0x1FA95, prExtendedPictographic}, // E12.0 [6] (๐Ÿช..๐Ÿช•) ringed planet..banjo + {0x1FA96, 0x1FAA8, prExtendedPictographic}, // E13.0 [19] (๐Ÿช–..๐Ÿชจ) military helmet..rock + {0x1FAA9, 0x1FAAC, prExtendedPictographic}, // E14.0 [4] (๐Ÿชฉ..๐Ÿชฌ) mirror ball..hamsa + {0x1FAAD, 0x1FAAF, prExtendedPictographic}, // E15.0 [3] (๐Ÿชญ..๐Ÿชฏ) folding hand fan..khanda + {0x1FAB0, 0x1FAB6, prExtendedPictographic}, // E13.0 [7] (๐Ÿชฐ..๐Ÿชถ) fly..feather + {0x1FAB7, 0x1FABA, prExtendedPictographic}, // E14.0 [4] (๐Ÿชท..๐Ÿชบ) lotus..nest with eggs + {0x1FABB, 0x1FABD, prExtendedPictographic}, // E15.0 [3] (๐Ÿชป..๐Ÿชฝ) hyacinth..wing + {0x1FABE, 0x1FABE, prExtendedPictographic}, // E0.0 [1] (๐Ÿชพ) + {0x1FABF, 0x1FABF, prExtendedPictographic}, // E15.0 [1] (๐Ÿชฟ) goose + {0x1FAC0, 0x1FAC2, prExtendedPictographic}, // E13.0 [3] (๐Ÿซ€..๐Ÿซ‚) anatomical heart..people hugging + {0x1FAC3, 0x1FAC5, prExtendedPictographic}, // E14.0 [3] (๐Ÿซƒ..๐Ÿซ…) pregnant man..person with crown + {0x1FAC6, 0x1FACD, prExtendedPictographic}, // E0.0 [8] (๐Ÿซ†..๐Ÿซ) .. + {0x1FACE, 0x1FACF, prExtendedPictographic}, // E15.0 [2] (๐ŸซŽ..๐Ÿซ) moose..donkey + {0x1FAD0, 0x1FAD6, prExtendedPictographic}, // E13.0 [7] (๐Ÿซ..๐Ÿซ–) blueberries..teapot + {0x1FAD7, 0x1FAD9, prExtendedPictographic}, // E14.0 [3] (๐Ÿซ—..๐Ÿซ™) pouring liquid..jar + {0x1FADA, 0x1FADB, prExtendedPictographic}, // E15.0 [2] (๐Ÿซš..๐Ÿซ›) ginger root..pea pod + {0x1FADC, 0x1FADF, prExtendedPictographic}, // E0.0 [4] (๐Ÿซœ..๐ŸซŸ) .. + {0x1FAE0, 0x1FAE7, prExtendedPictographic}, // E14.0 [8] (๐Ÿซ ..๐Ÿซง) melting face..bubbles + {0x1FAE8, 0x1FAE8, prExtendedPictographic}, // E15.0 [1] (๐Ÿซจ) shaking face + {0x1FAE9, 0x1FAEF, prExtendedPictographic}, // E0.0 [7] (๐Ÿซฉ..๐Ÿซฏ) .. + {0x1FAF0, 0x1FAF6, prExtendedPictographic}, // E14.0 [7] (๐Ÿซฐ..๐Ÿซถ) hand with index finger and thumb crossed..heart hands + {0x1FAF7, 0x1FAF8, prExtendedPictographic}, // E15.0 [2] (๐Ÿซท..๐Ÿซธ) leftwards pushing hand..rightwards pushing hand + {0x1FAF9, 0x1FAFF, prExtendedPictographic}, // E0.0 [7] (๐Ÿซน..๐Ÿซฟ) .. + {0x1FC00, 0x1FFFD, prExtendedPictographic}, // E0.0[1022] (๐Ÿฐ€..๐Ÿฟฝ) .. + {0xE0000, 0xE0000, prControl}, // Cn + {0xE0001, 0xE0001, prControl}, // Cf LANGUAGE TAG + {0xE0002, 0xE001F, prControl}, // Cn [30] .. + {0xE0020, 0xE007F, prExtend}, // Cf [96] TAG SPACE..CANCEL TAG + {0xE0080, 0xE00FF, prControl}, // Cn [128] .. + {0xE0100, 0xE01EF, prExtend}, // Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + {0xE01F0, 0xE0FFF, prControl}, // Cn [3600] .. +} diff --git a/vendor/github.com/rivo/uniseg/graphemerules.go b/vendor/github.com/rivo/uniseg/graphemerules.go new file mode 100644 index 0000000000..5d399d29c8 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/graphemerules.go @@ -0,0 +1,176 @@ +package uniseg + +// The states of the grapheme cluster parser. +const ( + grAny = iota + grCR + grControlLF + grL + grLVV + grLVTT + grPrepend + grExtendedPictographic + grExtendedPictographicZWJ + grRIOdd + grRIEven +) + +// The grapheme cluster parser's breaking instructions. +const ( + grNoBoundary = iota + grBoundary +) + +// grTransitions implements the grapheme cluster parser's state transitions. +// Maps state and property to a new state, a breaking instruction, and rule +// number. The breaking instruction always refers to the boundary between the +// last and next code point. Returns negative values if no transition is found. +// +// This function is used as follows: +// +// 1. Find specific state + specific property. Stop if found. +// 2. Find specific state + any property. +// 3. Find any state + specific property. +// 4. If only (2) or (3) (but not both) was found, stop. +// 5. If both (2) and (3) were found, use state from (3) and breaking instruction +// from the transition with the lower rule number, prefer (3) if rule numbers +// are equal. Stop. +// 6. Assume grAny and grBoundary. +// +// Unicode version 15.0.0. +func grTransitions(state, prop int) (newState int, newProp int, boundary int) { + // It turns out that using a big switch statement is much faster than using + // a map. + + switch uint64(state) | uint64(prop)<<32 { + // GB5 + case grAny | prCR<<32: + return grCR, grBoundary, 50 + case grAny | prLF<<32: + return grControlLF, grBoundary, 50 + case grAny | prControl<<32: + return grControlLF, grBoundary, 50 + + // GB4 + case grCR | prAny<<32: + return grAny, grBoundary, 40 + case grControlLF | prAny<<32: + return grAny, grBoundary, 40 + + // GB3 + case grCR | prLF<<32: + return grControlLF, grNoBoundary, 30 + + // GB6 + case grAny | prL<<32: + return grL, grBoundary, 9990 + case grL | prL<<32: + return grL, grNoBoundary, 60 + case grL | prV<<32: + return grLVV, grNoBoundary, 60 + case grL | prLV<<32: + return grLVV, grNoBoundary, 60 + case grL | prLVT<<32: + return grLVTT, grNoBoundary, 60 + + // GB7 + case grAny | prLV<<32: + return grLVV, grBoundary, 9990 + case grAny | prV<<32: + return grLVV, grBoundary, 9990 + case grLVV | prV<<32: + return grLVV, grNoBoundary, 70 + case grLVV | prT<<32: + return grLVTT, grNoBoundary, 70 + + // GB8 + case grAny | prLVT<<32: + return grLVTT, grBoundary, 9990 + case grAny | prT<<32: + return grLVTT, grBoundary, 9990 + case grLVTT | prT<<32: + return grLVTT, grNoBoundary, 80 + + // GB9 + case grAny | prExtend<<32: + return grAny, grNoBoundary, 90 + case grAny | prZWJ<<32: + return grAny, grNoBoundary, 90 + + // GB9a + case grAny | prSpacingMark<<32: + return grAny, grNoBoundary, 91 + + // GB9b + case grAny | prPrepend<<32: + return grPrepend, grBoundary, 9990 + case grPrepend | prAny<<32: + return grAny, grNoBoundary, 92 + + // GB11 + case grAny | prExtendedPictographic<<32: + return grExtendedPictographic, grBoundary, 9990 + case grExtendedPictographic | prExtend<<32: + return grExtendedPictographic, grNoBoundary, 110 + case grExtendedPictographic | prZWJ<<32: + return grExtendedPictographicZWJ, grNoBoundary, 110 + case grExtendedPictographicZWJ | prExtendedPictographic<<32: + return grExtendedPictographic, grNoBoundary, 110 + + // GB12 / GB13 + case grAny | prRegionalIndicator<<32: + return grRIOdd, grBoundary, 9990 + case grRIOdd | prRegionalIndicator<<32: + return grRIEven, grNoBoundary, 120 + case grRIEven | prRegionalIndicator<<32: + return grRIOdd, grBoundary, 120 + default: + return -1, -1, -1 + } +} + +// transitionGraphemeState determines the new state of the grapheme cluster +// parser given the current state and the next code point. It also returns the +// code point's grapheme property (the value mapped by the [graphemeCodePoints] +// table) and whether a cluster boundary was detected. +func transitionGraphemeState(state int, r rune) (newState, prop int, boundary bool) { + // Determine the property of the next character. + prop = propertyGraphemes(r) + + // Find the applicable transition. + nextState, nextProp, _ := grTransitions(state, prop) + if nextState >= 0 { + // We have a specific transition. We'll use it. + return nextState, prop, nextProp == grBoundary + } + + // No specific transition found. Try the less specific ones. + anyPropState, anyPropProp, anyPropRule := grTransitions(state, prAny) + anyStateState, anyStateProp, anyStateRule := grTransitions(grAny, prop) + if anyPropState >= 0 && anyStateState >= 0 { + // Both apply. We'll use a mix (see comments for grTransitions). + newState = anyStateState + boundary = anyStateProp == grBoundary + if anyPropRule < anyStateRule { + boundary = anyPropProp == grBoundary + } + return + } + + if anyPropState >= 0 { + // We only have a specific state. + return anyPropState, prop, anyPropProp == grBoundary + // This branch will probably never be reached because okAnyState will + // always be true given the current transition map. But we keep it here + // for future modifications to the transition map where this may not be + // true anymore. + } + + if anyStateState >= 0 { + // We only have a specific property. + return anyStateState, prop, anyStateProp == grBoundary + } + + // No known transition. GB999: Any รท Any. + return grAny, prop, true +} diff --git a/vendor/github.com/rivo/uniseg/line.go b/vendor/github.com/rivo/uniseg/line.go new file mode 100644 index 0000000000..7a46318d93 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/line.go @@ -0,0 +1,134 @@ +package uniseg + +import "unicode/utf8" + +// FirstLineSegment returns the prefix of the given byte slice after which a +// decision to break the string over to the next line can or must be made, +// according to the rules of [Unicode Standard Annex #14]. This is used to +// implement line breaking. +// +// Line breaking, also known as word wrapping, is the process of breaking a +// section of text into lines such that it will fit in the available width of a +// page, window or other display area. +// +// The returned "segment" may not be broken into smaller parts, unless no other +// breaking opportunities present themselves, in which case you may break by +// grapheme clusters (using the [FirstGraphemeCluster] function to determine the +// grapheme clusters). +// +// The "mustBreak" flag indicates whether you MUST break the line after the +// given segment (true), for example after newline characters, or you MAY break +// the line after the given segment (false). +// +// This function can be called continuously to extract all non-breaking sub-sets +// from a byte slice, as illustrated in the example below. +// +// If you don't know the current state, for example when calling the function +// for the first time, you must pass -1. For consecutive calls, pass the state +// and rest slice returned by the previous call. +// +// The "rest" slice is the sub-slice of the original byte slice "b" starting +// after the last byte of the identified line segment. If the length of the +// "rest" slice is 0, the entire byte slice "b" has been processed. The +// "segment" byte slice is the sub-slice of the input slice containing the +// identified line segment. +// +// Given an empty byte slice "b", the function returns nil values. +// +// Note that in accordance with [UAX #14 LB3], the final segment will end with +// "mustBreak" set to true. You can choose to ignore this by checking if the +// length of the "rest" slice is 0 and calling [HasTrailingLineBreak] or +// [HasTrailingLineBreakInString] on the last rune. +// +// Note also that this algorithm may break within grapheme clusters. This is +// addressed in Section 8.2 Example 6 of UAX #14. To avoid this, you can use +// the [Step] function instead. +// +// [Unicode Standard Annex #14]: https://www.unicode.org/reports/tr14/ +// [UAX #14 LB3]: https://www.unicode.org/reports/tr14/#Algorithm +func FirstLineSegment(b []byte, state int) (segment, rest []byte, mustBreak bool, newState int) { + // An empty byte slice returns nothing. + if len(b) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRune(b) + if len(b) <= length { // If we're already past the end, there is nothing else to parse. + return b, nil, true, lbAny // LB3. + } + + // If we don't know the state, determine it now. + if state < 0 { + state, _ = transitionLineBreakState(state, r, b[length:], "") + } + + // Transition until we find a boundary. + var boundary int + for { + r, l := utf8.DecodeRune(b[length:]) + state, boundary = transitionLineBreakState(state, r, b[length+l:], "") + + if boundary != LineDontBreak { + return b[:length], b[length:], boundary == LineMustBreak, state + } + + length += l + if len(b) <= length { + return b, nil, true, lbAny // LB3 + } + } +} + +// FirstLineSegmentInString is like [FirstLineSegment] but its input and outputs +// are strings. +func FirstLineSegmentInString(str string, state int) (segment, rest string, mustBreak bool, newState int) { + // An empty byte slice returns nothing. + if len(str) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRuneInString(str) + if len(str) <= length { // If we're already past the end, there is nothing else to parse. + return str, "", true, lbAny // LB3. + } + + // If we don't know the state, determine it now. + if state < 0 { + state, _ = transitionLineBreakState(state, r, nil, str[length:]) + } + + // Transition until we find a boundary. + var boundary int + for { + r, l := utf8.DecodeRuneInString(str[length:]) + state, boundary = transitionLineBreakState(state, r, nil, str[length+l:]) + + if boundary != LineDontBreak { + return str[:length], str[length:], boundary == LineMustBreak, state + } + + length += l + if len(str) <= length { + return str, "", true, lbAny // LB3. + } + } +} + +// HasTrailingLineBreak returns true if the last rune in the given byte slice is +// one of the hard line break code points defined in LB4 and LB5 of [UAX #14]. +// +// [UAX #14]: https://www.unicode.org/reports/tr14/#Algorithm +func HasTrailingLineBreak(b []byte) bool { + r, _ := utf8.DecodeLastRune(b) + property, _ := propertyLineBreak(r) + return property == prBK || property == prCR || property == prLF || property == prNL +} + +// HasTrailingLineBreakInString is like [HasTrailingLineBreak] but for a string. +func HasTrailingLineBreakInString(str string) bool { + r, _ := utf8.DecodeLastRuneInString(str) + property, _ := propertyLineBreak(r) + return property == prBK || property == prCR || property == prLF || property == prNL +} diff --git a/vendor/github.com/rivo/uniseg/lineproperties.go b/vendor/github.com/rivo/uniseg/lineproperties.go new file mode 100644 index 0000000000..ac7fac4c05 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/lineproperties.go @@ -0,0 +1,3554 @@ +// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// lineBreakCodePoints are taken from +// https://www.unicode.org/Public/15.0.0/ucd/LineBreak.txt +// and +// https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt +// ("Extended_Pictographic" only) +// on September 5, 2023. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var lineBreakCodePoints = [][4]int{ + {0x0000, 0x0008, prCM, gcCc}, // [9] .. + {0x0009, 0x0009, prBA, gcCc}, // + {0x000A, 0x000A, prLF, gcCc}, // + {0x000B, 0x000C, prBK, gcCc}, // [2] .. + {0x000D, 0x000D, prCR, gcCc}, // + {0x000E, 0x001F, prCM, gcCc}, // [18] .. + {0x0020, 0x0020, prSP, gcZs}, // SPACE + {0x0021, 0x0021, prEX, gcPo}, // EXCLAMATION MARK + {0x0022, 0x0022, prQU, gcPo}, // QUOTATION MARK + {0x0023, 0x0023, prAL, gcPo}, // NUMBER SIGN + {0x0024, 0x0024, prPR, gcSc}, // DOLLAR SIGN + {0x0025, 0x0025, prPO, gcPo}, // PERCENT SIGN + {0x0026, 0x0026, prAL, gcPo}, // AMPERSAND + {0x0027, 0x0027, prQU, gcPo}, // APOSTROPHE + {0x0028, 0x0028, prOP, gcPs}, // LEFT PARENTHESIS + {0x0029, 0x0029, prCP, gcPe}, // RIGHT PARENTHESIS + {0x002A, 0x002A, prAL, gcPo}, // ASTERISK + {0x002B, 0x002B, prPR, gcSm}, // PLUS SIGN + {0x002C, 0x002C, prIS, gcPo}, // COMMA + {0x002D, 0x002D, prHY, gcPd}, // HYPHEN-MINUS + {0x002E, 0x002E, prIS, gcPo}, // FULL STOP + {0x002F, 0x002F, prSY, gcPo}, // SOLIDUS + {0x0030, 0x0039, prNU, gcNd}, // [10] DIGIT ZERO..DIGIT NINE + {0x003A, 0x003B, prIS, gcPo}, // [2] COLON..SEMICOLON + {0x003C, 0x003E, prAL, gcSm}, // [3] LESS-THAN SIGN..GREATER-THAN SIGN + {0x003F, 0x003F, prEX, gcPo}, // QUESTION MARK + {0x0040, 0x0040, prAL, gcPo}, // COMMERCIAL AT + {0x0041, 0x005A, prAL, gcLu}, // [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z + {0x005B, 0x005B, prOP, gcPs}, // LEFT SQUARE BRACKET + {0x005C, 0x005C, prPR, gcPo}, // REVERSE SOLIDUS + {0x005D, 0x005D, prCP, gcPe}, // RIGHT SQUARE BRACKET + {0x005E, 0x005E, prAL, gcSk}, // CIRCUMFLEX ACCENT + {0x005F, 0x005F, prAL, gcPc}, // LOW LINE + {0x0060, 0x0060, prAL, gcSk}, // GRAVE ACCENT + {0x0061, 0x007A, prAL, gcLl}, // [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z + {0x007B, 0x007B, prOP, gcPs}, // LEFT CURLY BRACKET + {0x007C, 0x007C, prBA, gcSm}, // VERTICAL LINE + {0x007D, 0x007D, prCL, gcPe}, // RIGHT CURLY BRACKET + {0x007E, 0x007E, prAL, gcSm}, // TILDE + {0x007F, 0x007F, prCM, gcCc}, // + {0x0080, 0x0084, prCM, gcCc}, // [5] .. + {0x0085, 0x0085, prNL, gcCc}, // + {0x0086, 0x009F, prCM, gcCc}, // [26] .. + {0x00A0, 0x00A0, prGL, gcZs}, // NO-BREAK SPACE + {0x00A1, 0x00A1, prOP, gcPo}, // INVERTED EXCLAMATION MARK + {0x00A2, 0x00A2, prPO, gcSc}, // CENT SIGN + {0x00A3, 0x00A5, prPR, gcSc}, // [3] POUND SIGN..YEN SIGN + {0x00A6, 0x00A6, prAL, gcSo}, // BROKEN BAR + {0x00A7, 0x00A7, prAI, gcPo}, // SECTION SIGN + {0x00A8, 0x00A8, prAI, gcSk}, // DIAERESIS + {0x00A9, 0x00A9, prAL, gcSo}, // COPYRIGHT SIGN + {0x00AA, 0x00AA, prAI, gcLo}, // FEMININE ORDINAL INDICATOR + {0x00AB, 0x00AB, prQU, gcPi}, // LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00AC, 0x00AC, prAL, gcSm}, // NOT SIGN + {0x00AD, 0x00AD, prBA, gcCf}, // SOFT HYPHEN + {0x00AE, 0x00AE, prAL, gcSo}, // REGISTERED SIGN + {0x00AF, 0x00AF, prAL, gcSk}, // MACRON + {0x00B0, 0x00B0, prPO, gcSo}, // DEGREE SIGN + {0x00B1, 0x00B1, prPR, gcSm}, // PLUS-MINUS SIGN + {0x00B2, 0x00B3, prAI, gcNo}, // [2] SUPERSCRIPT TWO..SUPERSCRIPT THREE + {0x00B4, 0x00B4, prBB, gcSk}, // ACUTE ACCENT + {0x00B5, 0x00B5, prAL, gcLl}, // MICRO SIGN + {0x00B6, 0x00B7, prAI, gcPo}, // [2] PILCROW SIGN..MIDDLE DOT + {0x00B8, 0x00B8, prAI, gcSk}, // CEDILLA + {0x00B9, 0x00B9, prAI, gcNo}, // SUPERSCRIPT ONE + {0x00BA, 0x00BA, prAI, gcLo}, // MASCULINE ORDINAL INDICATOR + {0x00BB, 0x00BB, prQU, gcPf}, // RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00BC, 0x00BE, prAI, gcNo}, // [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS + {0x00BF, 0x00BF, prOP, gcPo}, // INVERTED QUESTION MARK + {0x00C0, 0x00D6, prAL, gcLu}, // [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS + {0x00D7, 0x00D7, prAI, gcSm}, // MULTIPLICATION SIGN + {0x00D8, 0x00F6, prAL, gcLC}, // [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS + {0x00F7, 0x00F7, prAI, gcSm}, // DIVISION SIGN + {0x00F8, 0x00FF, prAL, gcLl}, // [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS + {0x0100, 0x017F, prAL, gcLC}, // [128] LATIN CAPITAL LETTER A WITH MACRON..LATIN SMALL LETTER LONG S + {0x0180, 0x01BA, prAL, gcLC}, // [59] LATIN SMALL LETTER B WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL + {0x01BB, 0x01BB, prAL, gcLo}, // LATIN LETTER TWO WITH STROKE + {0x01BC, 0x01BF, prAL, gcLC}, // [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN + {0x01C0, 0x01C3, prAL, gcLo}, // [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK + {0x01C4, 0x024F, prAL, gcLC}, // [140] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER Y WITH STROKE + {0x0250, 0x0293, prAL, gcLl}, // [68] LATIN SMALL LETTER TURNED A..LATIN SMALL LETTER EZH WITH CURL + {0x0294, 0x0294, prAL, gcLo}, // LATIN LETTER GLOTTAL STOP + {0x0295, 0x02AF, prAL, gcLl}, // [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL + {0x02B0, 0x02C1, prAL, gcLm}, // [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP + {0x02C2, 0x02C5, prAL, gcSk}, // [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD + {0x02C6, 0x02C6, prAL, gcLm}, // MODIFIER LETTER CIRCUMFLEX ACCENT + {0x02C7, 0x02C7, prAI, gcLm}, // CARON + {0x02C8, 0x02C8, prBB, gcLm}, // MODIFIER LETTER VERTICAL LINE + {0x02C9, 0x02CB, prAI, gcLm}, // [3] MODIFIER LETTER MACRON..MODIFIER LETTER GRAVE ACCENT + {0x02CC, 0x02CC, prBB, gcLm}, // MODIFIER LETTER LOW VERTICAL LINE + {0x02CD, 0x02CD, prAI, gcLm}, // MODIFIER LETTER LOW MACRON + {0x02CE, 0x02CF, prAL, gcLm}, // [2] MODIFIER LETTER LOW GRAVE ACCENT..MODIFIER LETTER LOW ACUTE ACCENT + {0x02D0, 0x02D0, prAI, gcLm}, // MODIFIER LETTER TRIANGULAR COLON + {0x02D1, 0x02D1, prAL, gcLm}, // MODIFIER LETTER HALF TRIANGULAR COLON + {0x02D2, 0x02D7, prAL, gcSk}, // [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN + {0x02D8, 0x02DB, prAI, gcSk}, // [4] BREVE..OGONEK + {0x02DC, 0x02DC, prAL, gcSk}, // SMALL TILDE + {0x02DD, 0x02DD, prAI, gcSk}, // DOUBLE ACUTE ACCENT + {0x02DE, 0x02DE, prAL, gcSk}, // MODIFIER LETTER RHOTIC HOOK + {0x02DF, 0x02DF, prBB, gcSk}, // MODIFIER LETTER CROSS ACCENT + {0x02E0, 0x02E4, prAL, gcLm}, // [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP + {0x02E5, 0x02EB, prAL, gcSk}, // [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK + {0x02EC, 0x02EC, prAL, gcLm}, // MODIFIER LETTER VOICING + {0x02ED, 0x02ED, prAL, gcSk}, // MODIFIER LETTER UNASPIRATED + {0x02EE, 0x02EE, prAL, gcLm}, // MODIFIER LETTER DOUBLE APOSTROPHE + {0x02EF, 0x02FF, prAL, gcSk}, // [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW + {0x0300, 0x034E, prCM, gcMn}, // [79] COMBINING GRAVE ACCENT..COMBINING UPWARDS ARROW BELOW + {0x034F, 0x034F, prGL, gcMn}, // COMBINING GRAPHEME JOINER + {0x0350, 0x035B, prCM, gcMn}, // [12] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING ZIGZAG ABOVE + {0x035C, 0x0362, prGL, gcMn}, // [7] COMBINING DOUBLE BREVE BELOW..COMBINING DOUBLE RIGHTWARDS ARROW BELOW + {0x0363, 0x036F, prCM, gcMn}, // [13] COMBINING LATIN SMALL LETTER A..COMBINING LATIN SMALL LETTER X + {0x0370, 0x0373, prAL, gcLC}, // [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI + {0x0374, 0x0374, prAL, gcLm}, // GREEK NUMERAL SIGN + {0x0375, 0x0375, prAL, gcSk}, // GREEK LOWER NUMERAL SIGN + {0x0376, 0x0377, prAL, gcLC}, // [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA + {0x037A, 0x037A, prAL, gcLm}, // GREEK YPOGEGRAMMENI + {0x037B, 0x037D, prAL, gcLl}, // [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL + {0x037E, 0x037E, prIS, gcPo}, // GREEK QUESTION MARK + {0x037F, 0x037F, prAL, gcLu}, // GREEK CAPITAL LETTER YOT + {0x0384, 0x0385, prAL, gcSk}, // [2] GREEK TONOS..GREEK DIALYTIKA TONOS + {0x0386, 0x0386, prAL, gcLu}, // GREEK CAPITAL LETTER ALPHA WITH TONOS + {0x0387, 0x0387, prAL, gcPo}, // GREEK ANO TELEIA + {0x0388, 0x038A, prAL, gcLu}, // [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS + {0x038C, 0x038C, prAL, gcLu}, // GREEK CAPITAL LETTER OMICRON WITH TONOS + {0x038E, 0x03A1, prAL, gcLC}, // [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO + {0x03A3, 0x03F5, prAL, gcLC}, // [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL + {0x03F6, 0x03F6, prAL, gcSm}, // GREEK REVERSED LUNATE EPSILON SYMBOL + {0x03F7, 0x03FF, prAL, gcLC}, // [9] GREEK CAPITAL LETTER SHO..GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL + {0x0400, 0x0481, prAL, gcLC}, // [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA + {0x0482, 0x0482, prAL, gcSo}, // CYRILLIC THOUSANDS SIGN + {0x0483, 0x0487, prCM, gcMn}, // [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE + {0x0488, 0x0489, prCM, gcMe}, // [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN + {0x048A, 0x04FF, prAL, gcLC}, // [118] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER HA WITH STROKE + {0x0500, 0x052F, prAL, gcLC}, // [48] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER EL WITH DESCENDER + {0x0531, 0x0556, prAL, gcLu}, // [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH + {0x0559, 0x0559, prAL, gcLm}, // ARMENIAN MODIFIER LETTER LEFT HALF RING + {0x055A, 0x055F, prAL, gcPo}, // [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK + {0x0560, 0x0588, prAL, gcLl}, // [41] ARMENIAN SMALL LETTER TURNED AYB..ARMENIAN SMALL LETTER YI WITH STROKE + {0x0589, 0x0589, prIS, gcPo}, // ARMENIAN FULL STOP + {0x058A, 0x058A, prBA, gcPd}, // ARMENIAN HYPHEN + {0x058D, 0x058E, prAL, gcSo}, // [2] RIGHT-FACING ARMENIAN ETERNITY SIGN..LEFT-FACING ARMENIAN ETERNITY SIGN + {0x058F, 0x058F, prPR, gcSc}, // ARMENIAN DRAM SIGN + {0x0591, 0x05BD, prCM, gcMn}, // [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG + {0x05BE, 0x05BE, prBA, gcPd}, // HEBREW PUNCTUATION MAQAF + {0x05BF, 0x05BF, prCM, gcMn}, // HEBREW POINT RAFE + {0x05C0, 0x05C0, prAL, gcPo}, // HEBREW PUNCTUATION PASEQ + {0x05C1, 0x05C2, prCM, gcMn}, // [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT + {0x05C3, 0x05C3, prAL, gcPo}, // HEBREW PUNCTUATION SOF PASUQ + {0x05C4, 0x05C5, prCM, gcMn}, // [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT + {0x05C6, 0x05C6, prEX, gcPo}, // HEBREW PUNCTUATION NUN HAFUKHA + {0x05C7, 0x05C7, prCM, gcMn}, // HEBREW POINT QAMATS QATAN + {0x05D0, 0x05EA, prHL, gcLo}, // [27] HEBREW LETTER ALEF..HEBREW LETTER TAV + {0x05EF, 0x05F2, prHL, gcLo}, // [4] HEBREW YOD TRIANGLE..HEBREW LIGATURE YIDDISH DOUBLE YOD + {0x05F3, 0x05F4, prAL, gcPo}, // [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM + {0x0600, 0x0605, prAL, gcCf}, // [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE + {0x0606, 0x0608, prAL, gcSm}, // [3] ARABIC-INDIC CUBE ROOT..ARABIC RAY + {0x0609, 0x060A, prPO, gcPo}, // [2] ARABIC-INDIC PER MILLE SIGN..ARABIC-INDIC PER TEN THOUSAND SIGN + {0x060B, 0x060B, prPO, gcSc}, // AFGHANI SIGN + {0x060C, 0x060D, prIS, gcPo}, // [2] ARABIC COMMA..ARABIC DATE SEPARATOR + {0x060E, 0x060F, prAL, gcSo}, // [2] ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA + {0x0610, 0x061A, prCM, gcMn}, // [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA + {0x061B, 0x061B, prEX, gcPo}, // ARABIC SEMICOLON + {0x061C, 0x061C, prCM, gcCf}, // ARABIC LETTER MARK + {0x061D, 0x061F, prEX, gcPo}, // [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK + {0x0620, 0x063F, prAL, gcLo}, // [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE + {0x0640, 0x0640, prAL, gcLm}, // ARABIC TATWEEL + {0x0641, 0x064A, prAL, gcLo}, // [10] ARABIC LETTER FEH..ARABIC LETTER YEH + {0x064B, 0x065F, prCM, gcMn}, // [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW + {0x0660, 0x0669, prNU, gcNd}, // [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE + {0x066A, 0x066A, prPO, gcPo}, // ARABIC PERCENT SIGN + {0x066B, 0x066C, prNU, gcPo}, // [2] ARABIC DECIMAL SEPARATOR..ARABIC THOUSANDS SEPARATOR + {0x066D, 0x066D, prAL, gcPo}, // ARABIC FIVE POINTED STAR + {0x066E, 0x066F, prAL, gcLo}, // [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF + {0x0670, 0x0670, prCM, gcMn}, // ARABIC LETTER SUPERSCRIPT ALEF + {0x0671, 0x06D3, prAL, gcLo}, // [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE + {0x06D4, 0x06D4, prEX, gcPo}, // ARABIC FULL STOP + {0x06D5, 0x06D5, prAL, gcLo}, // ARABIC LETTER AE + {0x06D6, 0x06DC, prCM, gcMn}, // [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN + {0x06DD, 0x06DD, prAL, gcCf}, // ARABIC END OF AYAH + {0x06DE, 0x06DE, prAL, gcSo}, // ARABIC START OF RUB EL HIZB + {0x06DF, 0x06E4, prCM, gcMn}, // [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA + {0x06E5, 0x06E6, prAL, gcLm}, // [2] ARABIC SMALL WAW..ARABIC SMALL YEH + {0x06E7, 0x06E8, prCM, gcMn}, // [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON + {0x06E9, 0x06E9, prAL, gcSo}, // ARABIC PLACE OF SAJDAH + {0x06EA, 0x06ED, prCM, gcMn}, // [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM + {0x06EE, 0x06EF, prAL, gcLo}, // [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V + {0x06F0, 0x06F9, prNU, gcNd}, // [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE + {0x06FA, 0x06FC, prAL, gcLo}, // [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW + {0x06FD, 0x06FE, prAL, gcSo}, // [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN + {0x06FF, 0x06FF, prAL, gcLo}, // ARABIC LETTER HEH WITH INVERTED V + {0x0700, 0x070D, prAL, gcPo}, // [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS + {0x070F, 0x070F, prAL, gcCf}, // SYRIAC ABBREVIATION MARK + {0x0710, 0x0710, prAL, gcLo}, // SYRIAC LETTER ALAPH + {0x0711, 0x0711, prCM, gcMn}, // SYRIAC LETTER SUPERSCRIPT ALAPH + {0x0712, 0x072F, prAL, gcLo}, // [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH + {0x0730, 0x074A, prCM, gcMn}, // [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH + {0x074D, 0x074F, prAL, gcLo}, // [3] SYRIAC LETTER SOGDIAN ZHAIN..SYRIAC LETTER SOGDIAN FE + {0x0750, 0x077F, prAL, gcLo}, // [48] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE + {0x0780, 0x07A5, prAL, gcLo}, // [38] THAANA LETTER HAA..THAANA LETTER WAAVU + {0x07A6, 0x07B0, prCM, gcMn}, // [11] THAANA ABAFILI..THAANA SUKUN + {0x07B1, 0x07B1, prAL, gcLo}, // THAANA LETTER NAA + {0x07C0, 0x07C9, prNU, gcNd}, // [10] NKO DIGIT ZERO..NKO DIGIT NINE + {0x07CA, 0x07EA, prAL, gcLo}, // [33] NKO LETTER A..NKO LETTER JONA RA + {0x07EB, 0x07F3, prCM, gcMn}, // [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE + {0x07F4, 0x07F5, prAL, gcLm}, // [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE + {0x07F6, 0x07F6, prAL, gcSo}, // NKO SYMBOL OO DENNEN + {0x07F7, 0x07F7, prAL, gcPo}, // NKO SYMBOL GBAKURUNEN + {0x07F8, 0x07F8, prIS, gcPo}, // NKO COMMA + {0x07F9, 0x07F9, prEX, gcPo}, // NKO EXCLAMATION MARK + {0x07FA, 0x07FA, prAL, gcLm}, // NKO LAJANYALAN + {0x07FD, 0x07FD, prCM, gcMn}, // NKO DANTAYALAN + {0x07FE, 0x07FF, prPR, gcSc}, // [2] NKO DOROME SIGN..NKO TAMAN SIGN + {0x0800, 0x0815, prAL, gcLo}, // [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF + {0x0816, 0x0819, prCM, gcMn}, // [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH + {0x081A, 0x081A, prAL, gcLm}, // SAMARITAN MODIFIER LETTER EPENTHETIC YUT + {0x081B, 0x0823, prCM, gcMn}, // [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A + {0x0824, 0x0824, prAL, gcLm}, // SAMARITAN MODIFIER LETTER SHORT A + {0x0825, 0x0827, prCM, gcMn}, // [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U + {0x0828, 0x0828, prAL, gcLm}, // SAMARITAN MODIFIER LETTER I + {0x0829, 0x082D, prCM, gcMn}, // [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA + {0x0830, 0x083E, prAL, gcPo}, // [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU + {0x0840, 0x0858, prAL, gcLo}, // [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN + {0x0859, 0x085B, prCM, gcMn}, // [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK + {0x085E, 0x085E, prAL, gcPo}, // MANDAIC PUNCTUATION + {0x0860, 0x086A, prAL, gcLo}, // [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA + {0x0870, 0x0887, prAL, gcLo}, // [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT + {0x0888, 0x0888, prAL, gcSk}, // ARABIC RAISED ROUND DOT + {0x0889, 0x088E, prAL, gcLo}, // [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL + {0x0890, 0x0891, prAL, gcCf}, // [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE + {0x0898, 0x089F, prCM, gcMn}, // [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA + {0x08A0, 0x08C8, prAL, gcLo}, // [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF + {0x08C9, 0x08C9, prAL, gcLm}, // ARABIC SMALL FARSI YEH + {0x08CA, 0x08E1, prCM, gcMn}, // [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA + {0x08E2, 0x08E2, prAL, gcCf}, // ARABIC DISPUTED END OF AYAH + {0x08E3, 0x08FF, prCM, gcMn}, // [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA + {0x0900, 0x0902, prCM, gcMn}, // [3] DEVANAGARI SIGN INVERTED CANDRABINDU..DEVANAGARI SIGN ANUSVARA + {0x0903, 0x0903, prCM, gcMc}, // DEVANAGARI SIGN VISARGA + {0x0904, 0x0939, prAL, gcLo}, // [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA + {0x093A, 0x093A, prCM, gcMn}, // DEVANAGARI VOWEL SIGN OE + {0x093B, 0x093B, prCM, gcMc}, // DEVANAGARI VOWEL SIGN OOE + {0x093C, 0x093C, prCM, gcMn}, // DEVANAGARI SIGN NUKTA + {0x093D, 0x093D, prAL, gcLo}, // DEVANAGARI SIGN AVAGRAHA + {0x093E, 0x0940, prCM, gcMc}, // [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II + {0x0941, 0x0948, prCM, gcMn}, // [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI + {0x0949, 0x094C, prCM, gcMc}, // [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU + {0x094D, 0x094D, prCM, gcMn}, // DEVANAGARI SIGN VIRAMA + {0x094E, 0x094F, prCM, gcMc}, // [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW + {0x0950, 0x0950, prAL, gcLo}, // DEVANAGARI OM + {0x0951, 0x0957, prCM, gcMn}, // [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE + {0x0958, 0x0961, prAL, gcLo}, // [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL + {0x0962, 0x0963, prCM, gcMn}, // [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL + {0x0964, 0x0965, prBA, gcPo}, // [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA + {0x0966, 0x096F, prNU, gcNd}, // [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE + {0x0970, 0x0970, prAL, gcPo}, // DEVANAGARI ABBREVIATION SIGN + {0x0971, 0x0971, prAL, gcLm}, // DEVANAGARI SIGN HIGH SPACING DOT + {0x0972, 0x097F, prAL, gcLo}, // [14] DEVANAGARI LETTER CANDRA A..DEVANAGARI LETTER BBA + {0x0980, 0x0980, prAL, gcLo}, // BENGALI ANJI + {0x0981, 0x0981, prCM, gcMn}, // BENGALI SIGN CANDRABINDU + {0x0982, 0x0983, prCM, gcMc}, // [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA + {0x0985, 0x098C, prAL, gcLo}, // [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L + {0x098F, 0x0990, prAL, gcLo}, // [2] BENGALI LETTER E..BENGALI LETTER AI + {0x0993, 0x09A8, prAL, gcLo}, // [22] BENGALI LETTER O..BENGALI LETTER NA + {0x09AA, 0x09B0, prAL, gcLo}, // [7] BENGALI LETTER PA..BENGALI LETTER RA + {0x09B2, 0x09B2, prAL, gcLo}, // BENGALI LETTER LA + {0x09B6, 0x09B9, prAL, gcLo}, // [4] BENGALI LETTER SHA..BENGALI LETTER HA + {0x09BC, 0x09BC, prCM, gcMn}, // BENGALI SIGN NUKTA + {0x09BD, 0x09BD, prAL, gcLo}, // BENGALI SIGN AVAGRAHA + {0x09BE, 0x09C0, prCM, gcMc}, // [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II + {0x09C1, 0x09C4, prCM, gcMn}, // [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR + {0x09C7, 0x09C8, prCM, gcMc}, // [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI + {0x09CB, 0x09CC, prCM, gcMc}, // [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU + {0x09CD, 0x09CD, prCM, gcMn}, // BENGALI SIGN VIRAMA + {0x09CE, 0x09CE, prAL, gcLo}, // BENGALI LETTER KHANDA TA + {0x09D7, 0x09D7, prCM, gcMc}, // BENGALI AU LENGTH MARK + {0x09DC, 0x09DD, prAL, gcLo}, // [2] BENGALI LETTER RRA..BENGALI LETTER RHA + {0x09DF, 0x09E1, prAL, gcLo}, // [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL + {0x09E2, 0x09E3, prCM, gcMn}, // [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL + {0x09E6, 0x09EF, prNU, gcNd}, // [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE + {0x09F0, 0x09F1, prAL, gcLo}, // [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL + {0x09F2, 0x09F3, prPO, gcSc}, // [2] BENGALI RUPEE MARK..BENGALI RUPEE SIGN + {0x09F4, 0x09F8, prAL, gcNo}, // [5] BENGALI CURRENCY NUMERATOR ONE..BENGALI CURRENCY NUMERATOR ONE LESS THAN THE DENOMINATOR + {0x09F9, 0x09F9, prPO, gcNo}, // BENGALI CURRENCY DENOMINATOR SIXTEEN + {0x09FA, 0x09FA, prAL, gcSo}, // BENGALI ISSHAR + {0x09FB, 0x09FB, prPR, gcSc}, // BENGALI GANDA MARK + {0x09FC, 0x09FC, prAL, gcLo}, // BENGALI LETTER VEDIC ANUSVARA + {0x09FD, 0x09FD, prAL, gcPo}, // BENGALI ABBREVIATION SIGN + {0x09FE, 0x09FE, prCM, gcMn}, // BENGALI SANDHI MARK + {0x0A01, 0x0A02, prCM, gcMn}, // [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI + {0x0A03, 0x0A03, prCM, gcMc}, // GURMUKHI SIGN VISARGA + {0x0A05, 0x0A0A, prAL, gcLo}, // [6] GURMUKHI LETTER A..GURMUKHI LETTER UU + {0x0A0F, 0x0A10, prAL, gcLo}, // [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI + {0x0A13, 0x0A28, prAL, gcLo}, // [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA + {0x0A2A, 0x0A30, prAL, gcLo}, // [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA + {0x0A32, 0x0A33, prAL, gcLo}, // [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA + {0x0A35, 0x0A36, prAL, gcLo}, // [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA + {0x0A38, 0x0A39, prAL, gcLo}, // [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA + {0x0A3C, 0x0A3C, prCM, gcMn}, // GURMUKHI SIGN NUKTA + {0x0A3E, 0x0A40, prCM, gcMc}, // [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II + {0x0A41, 0x0A42, prCM, gcMn}, // [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU + {0x0A47, 0x0A48, prCM, gcMn}, // [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI + {0x0A4B, 0x0A4D, prCM, gcMn}, // [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA + {0x0A51, 0x0A51, prCM, gcMn}, // GURMUKHI SIGN UDAAT + {0x0A59, 0x0A5C, prAL, gcLo}, // [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA + {0x0A5E, 0x0A5E, prAL, gcLo}, // GURMUKHI LETTER FA + {0x0A66, 0x0A6F, prNU, gcNd}, // [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE + {0x0A70, 0x0A71, prCM, gcMn}, // [2] GURMUKHI TIPPI..GURMUKHI ADDAK + {0x0A72, 0x0A74, prAL, gcLo}, // [3] GURMUKHI IRI..GURMUKHI EK ONKAR + {0x0A75, 0x0A75, prCM, gcMn}, // GURMUKHI SIGN YAKASH + {0x0A76, 0x0A76, prAL, gcPo}, // GURMUKHI ABBREVIATION SIGN + {0x0A81, 0x0A82, prCM, gcMn}, // [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA + {0x0A83, 0x0A83, prCM, gcMc}, // GUJARATI SIGN VISARGA + {0x0A85, 0x0A8D, prAL, gcLo}, // [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E + {0x0A8F, 0x0A91, prAL, gcLo}, // [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O + {0x0A93, 0x0AA8, prAL, gcLo}, // [22] GUJARATI LETTER O..GUJARATI LETTER NA + {0x0AAA, 0x0AB0, prAL, gcLo}, // [7] GUJARATI LETTER PA..GUJARATI LETTER RA + {0x0AB2, 0x0AB3, prAL, gcLo}, // [2] GUJARATI LETTER LA..GUJARATI LETTER LLA + {0x0AB5, 0x0AB9, prAL, gcLo}, // [5] GUJARATI LETTER VA..GUJARATI LETTER HA + {0x0ABC, 0x0ABC, prCM, gcMn}, // GUJARATI SIGN NUKTA + {0x0ABD, 0x0ABD, prAL, gcLo}, // GUJARATI SIGN AVAGRAHA + {0x0ABE, 0x0AC0, prCM, gcMc}, // [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II + {0x0AC1, 0x0AC5, prCM, gcMn}, // [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E + {0x0AC7, 0x0AC8, prCM, gcMn}, // [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI + {0x0AC9, 0x0AC9, prCM, gcMc}, // GUJARATI VOWEL SIGN CANDRA O + {0x0ACB, 0x0ACC, prCM, gcMc}, // [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU + {0x0ACD, 0x0ACD, prCM, gcMn}, // GUJARATI SIGN VIRAMA + {0x0AD0, 0x0AD0, prAL, gcLo}, // GUJARATI OM + {0x0AE0, 0x0AE1, prAL, gcLo}, // [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL + {0x0AE2, 0x0AE3, prCM, gcMn}, // [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL + {0x0AE6, 0x0AEF, prNU, gcNd}, // [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE + {0x0AF0, 0x0AF0, prAL, gcPo}, // GUJARATI ABBREVIATION SIGN + {0x0AF1, 0x0AF1, prPR, gcSc}, // GUJARATI RUPEE SIGN + {0x0AF9, 0x0AF9, prAL, gcLo}, // GUJARATI LETTER ZHA + {0x0AFA, 0x0AFF, prCM, gcMn}, // [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE + {0x0B01, 0x0B01, prCM, gcMn}, // ORIYA SIGN CANDRABINDU + {0x0B02, 0x0B03, prCM, gcMc}, // [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA + {0x0B05, 0x0B0C, prAL, gcLo}, // [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L + {0x0B0F, 0x0B10, prAL, gcLo}, // [2] ORIYA LETTER E..ORIYA LETTER AI + {0x0B13, 0x0B28, prAL, gcLo}, // [22] ORIYA LETTER O..ORIYA LETTER NA + {0x0B2A, 0x0B30, prAL, gcLo}, // [7] ORIYA LETTER PA..ORIYA LETTER RA + {0x0B32, 0x0B33, prAL, gcLo}, // [2] ORIYA LETTER LA..ORIYA LETTER LLA + {0x0B35, 0x0B39, prAL, gcLo}, // [5] ORIYA LETTER VA..ORIYA LETTER HA + {0x0B3C, 0x0B3C, prCM, gcMn}, // ORIYA SIGN NUKTA + {0x0B3D, 0x0B3D, prAL, gcLo}, // ORIYA SIGN AVAGRAHA + {0x0B3E, 0x0B3E, prCM, gcMc}, // ORIYA VOWEL SIGN AA + {0x0B3F, 0x0B3F, prCM, gcMn}, // ORIYA VOWEL SIGN I + {0x0B40, 0x0B40, prCM, gcMc}, // ORIYA VOWEL SIGN II + {0x0B41, 0x0B44, prCM, gcMn}, // [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR + {0x0B47, 0x0B48, prCM, gcMc}, // [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI + {0x0B4B, 0x0B4C, prCM, gcMc}, // [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU + {0x0B4D, 0x0B4D, prCM, gcMn}, // ORIYA SIGN VIRAMA + {0x0B55, 0x0B56, prCM, gcMn}, // [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK + {0x0B57, 0x0B57, prCM, gcMc}, // ORIYA AU LENGTH MARK + {0x0B5C, 0x0B5D, prAL, gcLo}, // [2] ORIYA LETTER RRA..ORIYA LETTER RHA + {0x0B5F, 0x0B61, prAL, gcLo}, // [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL + {0x0B62, 0x0B63, prCM, gcMn}, // [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL + {0x0B66, 0x0B6F, prNU, gcNd}, // [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE + {0x0B70, 0x0B70, prAL, gcSo}, // ORIYA ISSHAR + {0x0B71, 0x0B71, prAL, gcLo}, // ORIYA LETTER WA + {0x0B72, 0x0B77, prAL, gcNo}, // [6] ORIYA FRACTION ONE QUARTER..ORIYA FRACTION THREE SIXTEENTHS + {0x0B82, 0x0B82, prCM, gcMn}, // TAMIL SIGN ANUSVARA + {0x0B83, 0x0B83, prAL, gcLo}, // TAMIL SIGN VISARGA + {0x0B85, 0x0B8A, prAL, gcLo}, // [6] TAMIL LETTER A..TAMIL LETTER UU + {0x0B8E, 0x0B90, prAL, gcLo}, // [3] TAMIL LETTER E..TAMIL LETTER AI + {0x0B92, 0x0B95, prAL, gcLo}, // [4] TAMIL LETTER O..TAMIL LETTER KA + {0x0B99, 0x0B9A, prAL, gcLo}, // [2] TAMIL LETTER NGA..TAMIL LETTER CA + {0x0B9C, 0x0B9C, prAL, gcLo}, // TAMIL LETTER JA + {0x0B9E, 0x0B9F, prAL, gcLo}, // [2] TAMIL LETTER NYA..TAMIL LETTER TTA + {0x0BA3, 0x0BA4, prAL, gcLo}, // [2] TAMIL LETTER NNA..TAMIL LETTER TA + {0x0BA8, 0x0BAA, prAL, gcLo}, // [3] TAMIL LETTER NA..TAMIL LETTER PA + {0x0BAE, 0x0BB9, prAL, gcLo}, // [12] TAMIL LETTER MA..TAMIL LETTER HA + {0x0BBE, 0x0BBF, prCM, gcMc}, // [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I + {0x0BC0, 0x0BC0, prCM, gcMn}, // TAMIL VOWEL SIGN II + {0x0BC1, 0x0BC2, prCM, gcMc}, // [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU + {0x0BC6, 0x0BC8, prCM, gcMc}, // [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI + {0x0BCA, 0x0BCC, prCM, gcMc}, // [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU + {0x0BCD, 0x0BCD, prCM, gcMn}, // TAMIL SIGN VIRAMA + {0x0BD0, 0x0BD0, prAL, gcLo}, // TAMIL OM + {0x0BD7, 0x0BD7, prCM, gcMc}, // TAMIL AU LENGTH MARK + {0x0BE6, 0x0BEF, prNU, gcNd}, // [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE + {0x0BF0, 0x0BF2, prAL, gcNo}, // [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND + {0x0BF3, 0x0BF8, prAL, gcSo}, // [6] TAMIL DAY SIGN..TAMIL AS ABOVE SIGN + {0x0BF9, 0x0BF9, prPR, gcSc}, // TAMIL RUPEE SIGN + {0x0BFA, 0x0BFA, prAL, gcSo}, // TAMIL NUMBER SIGN + {0x0C00, 0x0C00, prCM, gcMn}, // TELUGU SIGN COMBINING CANDRABINDU ABOVE + {0x0C01, 0x0C03, prCM, gcMc}, // [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA + {0x0C04, 0x0C04, prCM, gcMn}, // TELUGU SIGN COMBINING ANUSVARA ABOVE + {0x0C05, 0x0C0C, prAL, gcLo}, // [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L + {0x0C0E, 0x0C10, prAL, gcLo}, // [3] TELUGU LETTER E..TELUGU LETTER AI + {0x0C12, 0x0C28, prAL, gcLo}, // [23] TELUGU LETTER O..TELUGU LETTER NA + {0x0C2A, 0x0C39, prAL, gcLo}, // [16] TELUGU LETTER PA..TELUGU LETTER HA + {0x0C3C, 0x0C3C, prCM, gcMn}, // TELUGU SIGN NUKTA + {0x0C3D, 0x0C3D, prAL, gcLo}, // TELUGU SIGN AVAGRAHA + {0x0C3E, 0x0C40, prCM, gcMn}, // [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II + {0x0C41, 0x0C44, prCM, gcMc}, // [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR + {0x0C46, 0x0C48, prCM, gcMn}, // [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI + {0x0C4A, 0x0C4D, prCM, gcMn}, // [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA + {0x0C55, 0x0C56, prCM, gcMn}, // [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK + {0x0C58, 0x0C5A, prAL, gcLo}, // [3] TELUGU LETTER TSA..TELUGU LETTER RRRA + {0x0C5D, 0x0C5D, prAL, gcLo}, // TELUGU LETTER NAKAARA POLLU + {0x0C60, 0x0C61, prAL, gcLo}, // [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL + {0x0C62, 0x0C63, prCM, gcMn}, // [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL + {0x0C66, 0x0C6F, prNU, gcNd}, // [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE + {0x0C77, 0x0C77, prBB, gcPo}, // TELUGU SIGN SIDDHAM + {0x0C78, 0x0C7E, prAL, gcNo}, // [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR + {0x0C7F, 0x0C7F, prAL, gcSo}, // TELUGU SIGN TUUMU + {0x0C80, 0x0C80, prAL, gcLo}, // KANNADA SIGN SPACING CANDRABINDU + {0x0C81, 0x0C81, prCM, gcMn}, // KANNADA SIGN CANDRABINDU + {0x0C82, 0x0C83, prCM, gcMc}, // [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA + {0x0C84, 0x0C84, prBB, gcPo}, // KANNADA SIGN SIDDHAM + {0x0C85, 0x0C8C, prAL, gcLo}, // [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L + {0x0C8E, 0x0C90, prAL, gcLo}, // [3] KANNADA LETTER E..KANNADA LETTER AI + {0x0C92, 0x0CA8, prAL, gcLo}, // [23] KANNADA LETTER O..KANNADA LETTER NA + {0x0CAA, 0x0CB3, prAL, gcLo}, // [10] KANNADA LETTER PA..KANNADA LETTER LLA + {0x0CB5, 0x0CB9, prAL, gcLo}, // [5] KANNADA LETTER VA..KANNADA LETTER HA + {0x0CBC, 0x0CBC, prCM, gcMn}, // KANNADA SIGN NUKTA + {0x0CBD, 0x0CBD, prAL, gcLo}, // KANNADA SIGN AVAGRAHA + {0x0CBE, 0x0CBE, prCM, gcMc}, // KANNADA VOWEL SIGN AA + {0x0CBF, 0x0CBF, prCM, gcMn}, // KANNADA VOWEL SIGN I + {0x0CC0, 0x0CC4, prCM, gcMc}, // [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR + {0x0CC6, 0x0CC6, prCM, gcMn}, // KANNADA VOWEL SIGN E + {0x0CC7, 0x0CC8, prCM, gcMc}, // [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI + {0x0CCA, 0x0CCB, prCM, gcMc}, // [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO + {0x0CCC, 0x0CCD, prCM, gcMn}, // [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA + {0x0CD5, 0x0CD6, prCM, gcMc}, // [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK + {0x0CDD, 0x0CDE, prAL, gcLo}, // [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA + {0x0CE0, 0x0CE1, prAL, gcLo}, // [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL + {0x0CE2, 0x0CE3, prCM, gcMn}, // [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL + {0x0CE6, 0x0CEF, prNU, gcNd}, // [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE + {0x0CF1, 0x0CF2, prAL, gcLo}, // [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA + {0x0CF3, 0x0CF3, prCM, gcMc}, // KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT + {0x0D00, 0x0D01, prCM, gcMn}, // [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU + {0x0D02, 0x0D03, prCM, gcMc}, // [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA + {0x0D04, 0x0D0C, prAL, gcLo}, // [9] MALAYALAM LETTER VEDIC ANUSVARA..MALAYALAM LETTER VOCALIC L + {0x0D0E, 0x0D10, prAL, gcLo}, // [3] MALAYALAM LETTER E..MALAYALAM LETTER AI + {0x0D12, 0x0D3A, prAL, gcLo}, // [41] MALAYALAM LETTER O..MALAYALAM LETTER TTTA + {0x0D3B, 0x0D3C, prCM, gcMn}, // [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA + {0x0D3D, 0x0D3D, prAL, gcLo}, // MALAYALAM SIGN AVAGRAHA + {0x0D3E, 0x0D40, prCM, gcMc}, // [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II + {0x0D41, 0x0D44, prCM, gcMn}, // [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR + {0x0D46, 0x0D48, prCM, gcMc}, // [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI + {0x0D4A, 0x0D4C, prCM, gcMc}, // [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU + {0x0D4D, 0x0D4D, prCM, gcMn}, // MALAYALAM SIGN VIRAMA + {0x0D4E, 0x0D4E, prAL, gcLo}, // MALAYALAM LETTER DOT REPH + {0x0D4F, 0x0D4F, prAL, gcSo}, // MALAYALAM SIGN PARA + {0x0D54, 0x0D56, prAL, gcLo}, // [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL + {0x0D57, 0x0D57, prCM, gcMc}, // MALAYALAM AU LENGTH MARK + {0x0D58, 0x0D5E, prAL, gcNo}, // [7] MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH..MALAYALAM FRACTION ONE FIFTH + {0x0D5F, 0x0D61, prAL, gcLo}, // [3] MALAYALAM LETTER ARCHAIC II..MALAYALAM LETTER VOCALIC LL + {0x0D62, 0x0D63, prCM, gcMn}, // [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL + {0x0D66, 0x0D6F, prNU, gcNd}, // [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE + {0x0D70, 0x0D78, prAL, gcNo}, // [9] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE SIXTEENTHS + {0x0D79, 0x0D79, prPO, gcSo}, // MALAYALAM DATE MARK + {0x0D7A, 0x0D7F, prAL, gcLo}, // [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K + {0x0D81, 0x0D81, prCM, gcMn}, // SINHALA SIGN CANDRABINDU + {0x0D82, 0x0D83, prCM, gcMc}, // [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA + {0x0D85, 0x0D96, prAL, gcLo}, // [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA + {0x0D9A, 0x0DB1, prAL, gcLo}, // [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA + {0x0DB3, 0x0DBB, prAL, gcLo}, // [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA + {0x0DBD, 0x0DBD, prAL, gcLo}, // SINHALA LETTER DANTAJA LAYANNA + {0x0DC0, 0x0DC6, prAL, gcLo}, // [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA + {0x0DCA, 0x0DCA, prCM, gcMn}, // SINHALA SIGN AL-LAKUNA + {0x0DCF, 0x0DD1, prCM, gcMc}, // [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA + {0x0DD2, 0x0DD4, prCM, gcMn}, // [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA + {0x0DD6, 0x0DD6, prCM, gcMn}, // SINHALA VOWEL SIGN DIGA PAA-PILLA + {0x0DD8, 0x0DDF, prCM, gcMc}, // [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA + {0x0DE6, 0x0DEF, prNU, gcNd}, // [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE + {0x0DF2, 0x0DF3, prCM, gcMc}, // [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA + {0x0DF4, 0x0DF4, prAL, gcPo}, // SINHALA PUNCTUATION KUNDDALIYA + {0x0E01, 0x0E30, prSA, gcLo}, // [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A + {0x0E31, 0x0E31, prSA, gcMn}, // THAI CHARACTER MAI HAN-AKAT + {0x0E32, 0x0E33, prSA, gcLo}, // [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM + {0x0E34, 0x0E3A, prSA, gcMn}, // [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU + {0x0E3F, 0x0E3F, prPR, gcSc}, // THAI CURRENCY SYMBOL BAHT + {0x0E40, 0x0E45, prSA, gcLo}, // [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO + {0x0E46, 0x0E46, prSA, gcLm}, // THAI CHARACTER MAIYAMOK + {0x0E47, 0x0E4E, prSA, gcMn}, // [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN + {0x0E4F, 0x0E4F, prAL, gcPo}, // THAI CHARACTER FONGMAN + {0x0E50, 0x0E59, prNU, gcNd}, // [10] THAI DIGIT ZERO..THAI DIGIT NINE + {0x0E5A, 0x0E5B, prBA, gcPo}, // [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT + {0x0E81, 0x0E82, prSA, gcLo}, // [2] LAO LETTER KO..LAO LETTER KHO SUNG + {0x0E84, 0x0E84, prSA, gcLo}, // LAO LETTER KHO TAM + {0x0E86, 0x0E8A, prSA, gcLo}, // [5] LAO LETTER PALI GHA..LAO LETTER SO TAM + {0x0E8C, 0x0EA3, prSA, gcLo}, // [24] LAO LETTER PALI JHA..LAO LETTER LO LING + {0x0EA5, 0x0EA5, prSA, gcLo}, // LAO LETTER LO LOOT + {0x0EA7, 0x0EB0, prSA, gcLo}, // [10] LAO LETTER WO..LAO VOWEL SIGN A + {0x0EB1, 0x0EB1, prSA, gcMn}, // LAO VOWEL SIGN MAI KAN + {0x0EB2, 0x0EB3, prSA, gcLo}, // [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM + {0x0EB4, 0x0EBC, prSA, gcMn}, // [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO + {0x0EBD, 0x0EBD, prSA, gcLo}, // LAO SEMIVOWEL SIGN NYO + {0x0EC0, 0x0EC4, prSA, gcLo}, // [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI + {0x0EC6, 0x0EC6, prSA, gcLm}, // LAO KO LA + {0x0EC8, 0x0ECE, prSA, gcMn}, // [7] LAO TONE MAI EK..LAO YAMAKKAN + {0x0ED0, 0x0ED9, prNU, gcNd}, // [10] LAO DIGIT ZERO..LAO DIGIT NINE + {0x0EDC, 0x0EDF, prSA, gcLo}, // [4] LAO HO NO..LAO LETTER KHMU NYO + {0x0F00, 0x0F00, prAL, gcLo}, // TIBETAN SYLLABLE OM + {0x0F01, 0x0F03, prBB, gcSo}, // [3] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK GTER YIG MGO -UM GTER TSHEG MA + {0x0F04, 0x0F04, prBB, gcPo}, // TIBETAN MARK INITIAL YIG MGO MDUN MA + {0x0F05, 0x0F05, prAL, gcPo}, // TIBETAN MARK CLOSING YIG MGO SGAB MA + {0x0F06, 0x0F07, prBB, gcPo}, // [2] TIBETAN MARK CARET YIG MGO PHUR SHAD MA..TIBETAN MARK YIG MGO TSHEG SHAD MA + {0x0F08, 0x0F08, prGL, gcPo}, // TIBETAN MARK SBRUL SHAD + {0x0F09, 0x0F0A, prBB, gcPo}, // [2] TIBETAN MARK BSKUR YIG MGO..TIBETAN MARK BKA- SHOG YIG MGO + {0x0F0B, 0x0F0B, prBA, gcPo}, // TIBETAN MARK INTERSYLLABIC TSHEG + {0x0F0C, 0x0F0C, prGL, gcPo}, // TIBETAN MARK DELIMITER TSHEG BSTAR + {0x0F0D, 0x0F11, prEX, gcPo}, // [5] TIBETAN MARK SHAD..TIBETAN MARK RIN CHEN SPUNGS SHAD + {0x0F12, 0x0F12, prGL, gcPo}, // TIBETAN MARK RGYA GRAM SHAD + {0x0F13, 0x0F13, prAL, gcSo}, // TIBETAN MARK CARET -DZUD RTAGS ME LONG CAN + {0x0F14, 0x0F14, prEX, gcPo}, // TIBETAN MARK GTER TSHEG + {0x0F15, 0x0F17, prAL, gcSo}, // [3] TIBETAN LOGOTYPE SIGN CHAD RTAGS..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS + {0x0F18, 0x0F19, prCM, gcMn}, // [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS + {0x0F1A, 0x0F1F, prAL, gcSo}, // [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG + {0x0F20, 0x0F29, prNU, gcNd}, // [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE + {0x0F2A, 0x0F33, prAL, gcNo}, // [10] TIBETAN DIGIT HALF ONE..TIBETAN DIGIT HALF ZERO + {0x0F34, 0x0F34, prBA, gcSo}, // TIBETAN MARK BSDUS RTAGS + {0x0F35, 0x0F35, prCM, gcMn}, // TIBETAN MARK NGAS BZUNG NYI ZLA + {0x0F36, 0x0F36, prAL, gcSo}, // TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN + {0x0F37, 0x0F37, prCM, gcMn}, // TIBETAN MARK NGAS BZUNG SGOR RTAGS + {0x0F38, 0x0F38, prAL, gcSo}, // TIBETAN MARK CHE MGO + {0x0F39, 0x0F39, prCM, gcMn}, // TIBETAN MARK TSA -PHRU + {0x0F3A, 0x0F3A, prOP, gcPs}, // TIBETAN MARK GUG RTAGS GYON + {0x0F3B, 0x0F3B, prCL, gcPe}, // TIBETAN MARK GUG RTAGS GYAS + {0x0F3C, 0x0F3C, prOP, gcPs}, // TIBETAN MARK ANG KHANG GYON + {0x0F3D, 0x0F3D, prCL, gcPe}, // TIBETAN MARK ANG KHANG GYAS + {0x0F3E, 0x0F3F, prCM, gcMc}, // [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES + {0x0F40, 0x0F47, prAL, gcLo}, // [8] TIBETAN LETTER KA..TIBETAN LETTER JA + {0x0F49, 0x0F6C, prAL, gcLo}, // [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA + {0x0F71, 0x0F7E, prCM, gcMn}, // [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO + {0x0F7F, 0x0F7F, prBA, gcMc}, // TIBETAN SIGN RNAM BCAD + {0x0F80, 0x0F84, prCM, gcMn}, // [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA + {0x0F85, 0x0F85, prBA, gcPo}, // TIBETAN MARK PALUTA + {0x0F86, 0x0F87, prCM, gcMn}, // [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS + {0x0F88, 0x0F8C, prAL, gcLo}, // [5] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN INVERTED MCHU CAN + {0x0F8D, 0x0F97, prCM, gcMn}, // [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA + {0x0F99, 0x0FBC, prCM, gcMn}, // [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA + {0x0FBE, 0x0FBF, prBA, gcSo}, // [2] TIBETAN KU RU KHA..TIBETAN KU RU KHA BZHI MIG CAN + {0x0FC0, 0x0FC5, prAL, gcSo}, // [6] TIBETAN CANTILLATION SIGN HEAVY BEAT..TIBETAN SYMBOL RDO RJE + {0x0FC6, 0x0FC6, prCM, gcMn}, // TIBETAN SYMBOL PADMA GDAN + {0x0FC7, 0x0FCC, prAL, gcSo}, // [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL + {0x0FCE, 0x0FCF, prAL, gcSo}, // [2] TIBETAN SIGN RDEL NAG RDEL DKAR..TIBETAN SIGN RDEL NAG GSUM + {0x0FD0, 0x0FD1, prBB, gcPo}, // [2] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK MNYAM YIG GI MGO RGYAN + {0x0FD2, 0x0FD2, prBA, gcPo}, // TIBETAN MARK NYIS TSHEG + {0x0FD3, 0x0FD3, prBB, gcPo}, // TIBETAN MARK INITIAL BRDA RNYING YIG MGO MDUN MA + {0x0FD4, 0x0FD4, prAL, gcPo}, // TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA + {0x0FD5, 0x0FD8, prAL, gcSo}, // [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS + {0x0FD9, 0x0FDA, prGL, gcPo}, // [2] TIBETAN MARK LEADING MCHAN RTAGS..TIBETAN MARK TRAILING MCHAN RTAGS + {0x1000, 0x102A, prSA, gcLo}, // [43] MYANMAR LETTER KA..MYANMAR LETTER AU + {0x102B, 0x102C, prSA, gcMc}, // [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA + {0x102D, 0x1030, prSA, gcMn}, // [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU + {0x1031, 0x1031, prSA, gcMc}, // MYANMAR VOWEL SIGN E + {0x1032, 0x1037, prSA, gcMn}, // [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW + {0x1038, 0x1038, prSA, gcMc}, // MYANMAR SIGN VISARGA + {0x1039, 0x103A, prSA, gcMn}, // [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT + {0x103B, 0x103C, prSA, gcMc}, // [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA + {0x103D, 0x103E, prSA, gcMn}, // [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA + {0x103F, 0x103F, prSA, gcLo}, // MYANMAR LETTER GREAT SA + {0x1040, 0x1049, prNU, gcNd}, // [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE + {0x104A, 0x104B, prBA, gcPo}, // [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION + {0x104C, 0x104F, prAL, gcPo}, // [4] MYANMAR SYMBOL LOCATIVE..MYANMAR SYMBOL GENITIVE + {0x1050, 0x1055, prSA, gcLo}, // [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL + {0x1056, 0x1057, prSA, gcMc}, // [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR + {0x1058, 0x1059, prSA, gcMn}, // [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL + {0x105A, 0x105D, prSA, gcLo}, // [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE + {0x105E, 0x1060, prSA, gcMn}, // [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA + {0x1061, 0x1061, prSA, gcLo}, // MYANMAR LETTER SGAW KAREN SHA + {0x1062, 0x1064, prSA, gcMc}, // [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO + {0x1065, 0x1066, prSA, gcLo}, // [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA + {0x1067, 0x106D, prSA, gcMc}, // [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 + {0x106E, 0x1070, prSA, gcLo}, // [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA + {0x1071, 0x1074, prSA, gcMn}, // [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE + {0x1075, 0x1081, prSA, gcLo}, // [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA + {0x1082, 0x1082, prSA, gcMn}, // MYANMAR CONSONANT SIGN SHAN MEDIAL WA + {0x1083, 0x1084, prSA, gcMc}, // [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E + {0x1085, 0x1086, prSA, gcMn}, // [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y + {0x1087, 0x108C, prSA, gcMc}, // [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 + {0x108D, 0x108D, prSA, gcMn}, // MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE + {0x108E, 0x108E, prSA, gcLo}, // MYANMAR LETTER RUMAI PALAUNG FA + {0x108F, 0x108F, prSA, gcMc}, // MYANMAR SIGN RUMAI PALAUNG TONE-5 + {0x1090, 0x1099, prNU, gcNd}, // [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE + {0x109A, 0x109C, prSA, gcMc}, // [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A + {0x109D, 0x109D, prSA, gcMn}, // MYANMAR VOWEL SIGN AITON AI + {0x109E, 0x109F, prSA, gcSo}, // [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION + {0x10A0, 0x10C5, prAL, gcLu}, // [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE + {0x10C7, 0x10C7, prAL, gcLu}, // GEORGIAN CAPITAL LETTER YN + {0x10CD, 0x10CD, prAL, gcLu}, // GEORGIAN CAPITAL LETTER AEN + {0x10D0, 0x10FA, prAL, gcLl}, // [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN + {0x10FB, 0x10FB, prAL, gcPo}, // GEORGIAN PARAGRAPH SEPARATOR + {0x10FC, 0x10FC, prAL, gcLm}, // MODIFIER LETTER GEORGIAN NAR + {0x10FD, 0x10FF, prAL, gcLl}, // [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN + {0x1100, 0x115F, prJL, gcLo}, // [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER + {0x1160, 0x11A7, prJV, gcLo}, // [72] HANGUL JUNGSEONG FILLER..HANGUL JUNGSEONG O-YAE + {0x11A8, 0x11FF, prJT, gcLo}, // [88] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG SSANGNIEUN + {0x1200, 0x1248, prAL, gcLo}, // [73] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE QWA + {0x124A, 0x124D, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE + {0x1250, 0x1256, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO + {0x1258, 0x1258, prAL, gcLo}, // ETHIOPIC SYLLABLE QHWA + {0x125A, 0x125D, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE + {0x1260, 0x1288, prAL, gcLo}, // [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA + {0x128A, 0x128D, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE + {0x1290, 0x12B0, prAL, gcLo}, // [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA + {0x12B2, 0x12B5, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE + {0x12B8, 0x12BE, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO + {0x12C0, 0x12C0, prAL, gcLo}, // ETHIOPIC SYLLABLE KXWA + {0x12C2, 0x12C5, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE + {0x12C8, 0x12D6, prAL, gcLo}, // [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O + {0x12D8, 0x1310, prAL, gcLo}, // [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA + {0x1312, 0x1315, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE + {0x1318, 0x135A, prAL, gcLo}, // [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA + {0x135D, 0x135F, prCM, gcMn}, // [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK + {0x1360, 0x1360, prAL, gcPo}, // ETHIOPIC SECTION MARK + {0x1361, 0x1361, prBA, gcPo}, // ETHIOPIC WORDSPACE + {0x1362, 0x1368, prAL, gcPo}, // [7] ETHIOPIC FULL STOP..ETHIOPIC PARAGRAPH SEPARATOR + {0x1369, 0x137C, prAL, gcNo}, // [20] ETHIOPIC DIGIT ONE..ETHIOPIC NUMBER TEN THOUSAND + {0x1380, 0x138F, prAL, gcLo}, // [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE + {0x1390, 0x1399, prAL, gcSo}, // [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT + {0x13A0, 0x13F5, prAL, gcLu}, // [86] CHEROKEE LETTER A..CHEROKEE LETTER MV + {0x13F8, 0x13FD, prAL, gcLl}, // [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV + {0x1400, 0x1400, prBA, gcPd}, // CANADIAN SYLLABICS HYPHEN + {0x1401, 0x166C, prAL, gcLo}, // [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA + {0x166D, 0x166D, prAL, gcSo}, // CANADIAN SYLLABICS CHI SIGN + {0x166E, 0x166E, prAL, gcPo}, // CANADIAN SYLLABICS FULL STOP + {0x166F, 0x167F, prAL, gcLo}, // [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W + {0x1680, 0x1680, prBA, gcZs}, // OGHAM SPACE MARK + {0x1681, 0x169A, prAL, gcLo}, // [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH + {0x169B, 0x169B, prOP, gcPs}, // OGHAM FEATHER MARK + {0x169C, 0x169C, prCL, gcPe}, // OGHAM REVERSED FEATHER MARK + {0x16A0, 0x16EA, prAL, gcLo}, // [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X + {0x16EB, 0x16ED, prBA, gcPo}, // [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION + {0x16EE, 0x16F0, prAL, gcNl}, // [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL + {0x16F1, 0x16F8, prAL, gcLo}, // [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC + {0x1700, 0x1711, prAL, gcLo}, // [18] TAGALOG LETTER A..TAGALOG LETTER HA + {0x1712, 0x1714, prCM, gcMn}, // [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA + {0x1715, 0x1715, prCM, gcMc}, // TAGALOG SIGN PAMUDPOD + {0x171F, 0x171F, prAL, gcLo}, // TAGALOG LETTER ARCHAIC RA + {0x1720, 0x1731, prAL, gcLo}, // [18] HANUNOO LETTER A..HANUNOO LETTER HA + {0x1732, 0x1733, prCM, gcMn}, // [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U + {0x1734, 0x1734, prCM, gcMc}, // HANUNOO SIGN PAMUDPOD + {0x1735, 0x1736, prBA, gcPo}, // [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION + {0x1740, 0x1751, prAL, gcLo}, // [18] BUHID LETTER A..BUHID LETTER HA + {0x1752, 0x1753, prCM, gcMn}, // [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U + {0x1760, 0x176C, prAL, gcLo}, // [13] TAGBANWA LETTER A..TAGBANWA LETTER YA + {0x176E, 0x1770, prAL, gcLo}, // [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA + {0x1772, 0x1773, prCM, gcMn}, // [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U + {0x1780, 0x17B3, prSA, gcLo}, // [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU + {0x17B4, 0x17B5, prSA, gcMn}, // [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA + {0x17B6, 0x17B6, prSA, gcMc}, // KHMER VOWEL SIGN AA + {0x17B7, 0x17BD, prSA, gcMn}, // [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA + {0x17BE, 0x17C5, prSA, gcMc}, // [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU + {0x17C6, 0x17C6, prSA, gcMn}, // KHMER SIGN NIKAHIT + {0x17C7, 0x17C8, prSA, gcMc}, // [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU + {0x17C9, 0x17D3, prSA, gcMn}, // [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT + {0x17D4, 0x17D5, prBA, gcPo}, // [2] KHMER SIGN KHAN..KHMER SIGN BARIYOOSAN + {0x17D6, 0x17D6, prNS, gcPo}, // KHMER SIGN CAMNUC PII KUUH + {0x17D7, 0x17D7, prSA, gcLm}, // KHMER SIGN LEK TOO + {0x17D8, 0x17D8, prBA, gcPo}, // KHMER SIGN BEYYAL + {0x17D9, 0x17D9, prAL, gcPo}, // KHMER SIGN PHNAEK MUAN + {0x17DA, 0x17DA, prBA, gcPo}, // KHMER SIGN KOOMUUT + {0x17DB, 0x17DB, prPR, gcSc}, // KHMER CURRENCY SYMBOL RIEL + {0x17DC, 0x17DC, prSA, gcLo}, // KHMER SIGN AVAKRAHASANYA + {0x17DD, 0x17DD, prSA, gcMn}, // KHMER SIGN ATTHACAN + {0x17E0, 0x17E9, prNU, gcNd}, // [10] KHMER DIGIT ZERO..KHMER DIGIT NINE + {0x17F0, 0x17F9, prAL, gcNo}, // [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON + {0x1800, 0x1801, prAL, gcPo}, // [2] MONGOLIAN BIRGA..MONGOLIAN ELLIPSIS + {0x1802, 0x1803, prEX, gcPo}, // [2] MONGOLIAN COMMA..MONGOLIAN FULL STOP + {0x1804, 0x1805, prBA, gcPo}, // [2] MONGOLIAN COLON..MONGOLIAN FOUR DOTS + {0x1806, 0x1806, prBB, gcPd}, // MONGOLIAN TODO SOFT HYPHEN + {0x1807, 0x1807, prAL, gcPo}, // MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER + {0x1808, 0x1809, prEX, gcPo}, // [2] MONGOLIAN MANCHU COMMA..MONGOLIAN MANCHU FULL STOP + {0x180A, 0x180A, prAL, gcPo}, // MONGOLIAN NIRUGU + {0x180B, 0x180D, prCM, gcMn}, // [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE + {0x180E, 0x180E, prGL, gcCf}, // MONGOLIAN VOWEL SEPARATOR + {0x180F, 0x180F, prCM, gcMn}, // MONGOLIAN FREE VARIATION SELECTOR FOUR + {0x1810, 0x1819, prNU, gcNd}, // [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE + {0x1820, 0x1842, prAL, gcLo}, // [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI + {0x1843, 0x1843, prAL, gcLm}, // MONGOLIAN LETTER TODO LONG VOWEL SIGN + {0x1844, 0x1878, prAL, gcLo}, // [53] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER CHA WITH TWO DOTS + {0x1880, 0x1884, prAL, gcLo}, // [5] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER ALI GALI INVERTED UBADAMA + {0x1885, 0x1886, prCM, gcMn}, // [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA + {0x1887, 0x18A8, prAL, gcLo}, // [34] MONGOLIAN LETTER ALI GALI A..MONGOLIAN LETTER MANCHU ALI GALI BHA + {0x18A9, 0x18A9, prCM, gcMn}, // MONGOLIAN LETTER ALI GALI DAGALGA + {0x18AA, 0x18AA, prAL, gcLo}, // MONGOLIAN LETTER MANCHU ALI GALI LHA + {0x18B0, 0x18F5, prAL, gcLo}, // [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S + {0x1900, 0x191E, prAL, gcLo}, // [31] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER TRA + {0x1920, 0x1922, prCM, gcMn}, // [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U + {0x1923, 0x1926, prCM, gcMc}, // [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU + {0x1927, 0x1928, prCM, gcMn}, // [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O + {0x1929, 0x192B, prCM, gcMc}, // [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA + {0x1930, 0x1931, prCM, gcMc}, // [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA + {0x1932, 0x1932, prCM, gcMn}, // LIMBU SMALL LETTER ANUSVARA + {0x1933, 0x1938, prCM, gcMc}, // [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA + {0x1939, 0x193B, prCM, gcMn}, // [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I + {0x1940, 0x1940, prAL, gcSo}, // LIMBU SIGN LOO + {0x1944, 0x1945, prEX, gcPo}, // [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK + {0x1946, 0x194F, prNU, gcNd}, // [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE + {0x1950, 0x196D, prSA, gcLo}, // [30] TAI LE LETTER KA..TAI LE LETTER AI + {0x1970, 0x1974, prSA, gcLo}, // [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 + {0x1980, 0x19AB, prSA, gcLo}, // [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA + {0x19B0, 0x19C9, prSA, gcLo}, // [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 + {0x19D0, 0x19D9, prNU, gcNd}, // [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE + {0x19DA, 0x19DA, prSA, gcNo}, // NEW TAI LUE THAM DIGIT ONE + {0x19DE, 0x19DF, prSA, gcSo}, // [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV + {0x19E0, 0x19FF, prAL, gcSo}, // [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC + {0x1A00, 0x1A16, prAL, gcLo}, // [23] BUGINESE LETTER KA..BUGINESE LETTER HA + {0x1A17, 0x1A18, prCM, gcMn}, // [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U + {0x1A19, 0x1A1A, prCM, gcMc}, // [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O + {0x1A1B, 0x1A1B, prCM, gcMn}, // BUGINESE VOWEL SIGN AE + {0x1A1E, 0x1A1F, prAL, gcPo}, // [2] BUGINESE PALLAWA..BUGINESE END OF SECTION + {0x1A20, 0x1A54, prSA, gcLo}, // [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA + {0x1A55, 0x1A55, prSA, gcMc}, // TAI THAM CONSONANT SIGN MEDIAL RA + {0x1A56, 0x1A56, prSA, gcMn}, // TAI THAM CONSONANT SIGN MEDIAL LA + {0x1A57, 0x1A57, prSA, gcMc}, // TAI THAM CONSONANT SIGN LA TANG LAI + {0x1A58, 0x1A5E, prSA, gcMn}, // [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA + {0x1A60, 0x1A60, prSA, gcMn}, // TAI THAM SIGN SAKOT + {0x1A61, 0x1A61, prSA, gcMc}, // TAI THAM VOWEL SIGN A + {0x1A62, 0x1A62, prSA, gcMn}, // TAI THAM VOWEL SIGN MAI SAT + {0x1A63, 0x1A64, prSA, gcMc}, // [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA + {0x1A65, 0x1A6C, prSA, gcMn}, // [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW + {0x1A6D, 0x1A72, prSA, gcMc}, // [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI + {0x1A73, 0x1A7C, prSA, gcMn}, // [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN + {0x1A7F, 0x1A7F, prCM, gcMn}, // TAI THAM COMBINING CRYPTOGRAMMIC DOT + {0x1A80, 0x1A89, prNU, gcNd}, // [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE + {0x1A90, 0x1A99, prNU, gcNd}, // [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE + {0x1AA0, 0x1AA6, prSA, gcPo}, // [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA + {0x1AA7, 0x1AA7, prSA, gcLm}, // TAI THAM SIGN MAI YAMOK + {0x1AA8, 0x1AAD, prSA, gcPo}, // [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG + {0x1AB0, 0x1ABD, prCM, gcMn}, // [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW + {0x1ABE, 0x1ABE, prCM, gcMe}, // COMBINING PARENTHESES OVERLAY + {0x1ABF, 0x1ACE, prCM, gcMn}, // [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T + {0x1B00, 0x1B03, prCM, gcMn}, // [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG + {0x1B04, 0x1B04, prCM, gcMc}, // BALINESE SIGN BISAH + {0x1B05, 0x1B33, prAL, gcLo}, // [47] BALINESE LETTER AKARA..BALINESE LETTER HA + {0x1B34, 0x1B34, prCM, gcMn}, // BALINESE SIGN REREKAN + {0x1B35, 0x1B35, prCM, gcMc}, // BALINESE VOWEL SIGN TEDUNG + {0x1B36, 0x1B3A, prCM, gcMn}, // [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA + {0x1B3B, 0x1B3B, prCM, gcMc}, // BALINESE VOWEL SIGN RA REPA TEDUNG + {0x1B3C, 0x1B3C, prCM, gcMn}, // BALINESE VOWEL SIGN LA LENGA + {0x1B3D, 0x1B41, prCM, gcMc}, // [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG + {0x1B42, 0x1B42, prCM, gcMn}, // BALINESE VOWEL SIGN PEPET + {0x1B43, 0x1B44, prCM, gcMc}, // [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG + {0x1B45, 0x1B4C, prAL, gcLo}, // [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA + {0x1B50, 0x1B59, prNU, gcNd}, // [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE + {0x1B5A, 0x1B5B, prBA, gcPo}, // [2] BALINESE PANTI..BALINESE PAMADA + {0x1B5C, 0x1B5C, prAL, gcPo}, // BALINESE WINDU + {0x1B5D, 0x1B60, prBA, gcPo}, // [4] BALINESE CARIK PAMUNGKAH..BALINESE PAMENENG + {0x1B61, 0x1B6A, prAL, gcSo}, // [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE + {0x1B6B, 0x1B73, prCM, gcMn}, // [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG + {0x1B74, 0x1B7C, prAL, gcSo}, // [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING + {0x1B7D, 0x1B7E, prBA, gcPo}, // [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG + {0x1B80, 0x1B81, prCM, gcMn}, // [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR + {0x1B82, 0x1B82, prCM, gcMc}, // SUNDANESE SIGN PANGWISAD + {0x1B83, 0x1BA0, prAL, gcLo}, // [30] SUNDANESE LETTER A..SUNDANESE LETTER HA + {0x1BA1, 0x1BA1, prCM, gcMc}, // SUNDANESE CONSONANT SIGN PAMINGKAL + {0x1BA2, 0x1BA5, prCM, gcMn}, // [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU + {0x1BA6, 0x1BA7, prCM, gcMc}, // [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG + {0x1BA8, 0x1BA9, prCM, gcMn}, // [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG + {0x1BAA, 0x1BAA, prCM, gcMc}, // SUNDANESE SIGN PAMAAEH + {0x1BAB, 0x1BAD, prCM, gcMn}, // [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA + {0x1BAE, 0x1BAF, prAL, gcLo}, // [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA + {0x1BB0, 0x1BB9, prNU, gcNd}, // [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE + {0x1BBA, 0x1BBF, prAL, gcLo}, // [6] SUNDANESE AVAGRAHA..SUNDANESE LETTER FINAL M + {0x1BC0, 0x1BE5, prAL, gcLo}, // [38] BATAK LETTER A..BATAK LETTER U + {0x1BE6, 0x1BE6, prCM, gcMn}, // BATAK SIGN TOMPI + {0x1BE7, 0x1BE7, prCM, gcMc}, // BATAK VOWEL SIGN E + {0x1BE8, 0x1BE9, prCM, gcMn}, // [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE + {0x1BEA, 0x1BEC, prCM, gcMc}, // [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O + {0x1BED, 0x1BED, prCM, gcMn}, // BATAK VOWEL SIGN KARO O + {0x1BEE, 0x1BEE, prCM, gcMc}, // BATAK VOWEL SIGN U + {0x1BEF, 0x1BF1, prCM, gcMn}, // [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H + {0x1BF2, 0x1BF3, prCM, gcMc}, // [2] BATAK PANGOLAT..BATAK PANONGONAN + {0x1BFC, 0x1BFF, prAL, gcPo}, // [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT + {0x1C00, 0x1C23, prAL, gcLo}, // [36] LEPCHA LETTER KA..LEPCHA LETTER A + {0x1C24, 0x1C2B, prCM, gcMc}, // [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU + {0x1C2C, 0x1C33, prCM, gcMn}, // [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T + {0x1C34, 0x1C35, prCM, gcMc}, // [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG + {0x1C36, 0x1C37, prCM, gcMn}, // [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA + {0x1C3B, 0x1C3F, prBA, gcPo}, // [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK + {0x1C40, 0x1C49, prNU, gcNd}, // [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE + {0x1C4D, 0x1C4F, prAL, gcLo}, // [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA + {0x1C50, 0x1C59, prNU, gcNd}, // [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE + {0x1C5A, 0x1C77, prAL, gcLo}, // [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH + {0x1C78, 0x1C7D, prAL, gcLm}, // [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD + {0x1C7E, 0x1C7F, prBA, gcPo}, // [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD + {0x1C80, 0x1C88, prAL, gcLl}, // [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK + {0x1C90, 0x1CBA, prAL, gcLu}, // [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN + {0x1CBD, 0x1CBF, prAL, gcLu}, // [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + {0x1CC0, 0x1CC7, prAL, gcPo}, // [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA + {0x1CD0, 0x1CD2, prCM, gcMn}, // [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA + {0x1CD3, 0x1CD3, prAL, gcPo}, // VEDIC SIGN NIHSHVASA + {0x1CD4, 0x1CE0, prCM, gcMn}, // [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA + {0x1CE1, 0x1CE1, prCM, gcMc}, // VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA + {0x1CE2, 0x1CE8, prCM, gcMn}, // [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL + {0x1CE9, 0x1CEC, prAL, gcLo}, // [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL + {0x1CED, 0x1CED, prCM, gcMn}, // VEDIC SIGN TIRYAK + {0x1CEE, 0x1CF3, prAL, gcLo}, // [6] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ROTATED ARDHAVISARGA + {0x1CF4, 0x1CF4, prCM, gcMn}, // VEDIC TONE CANDRA ABOVE + {0x1CF5, 0x1CF6, prAL, gcLo}, // [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN UPADHMANIYA + {0x1CF7, 0x1CF7, prCM, gcMc}, // VEDIC SIGN ATIKRAMA + {0x1CF8, 0x1CF9, prCM, gcMn}, // [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE + {0x1CFA, 0x1CFA, prAL, gcLo}, // VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA + {0x1D00, 0x1D2B, prAL, gcLl}, // [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL + {0x1D2C, 0x1D6A, prAL, gcLm}, // [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI + {0x1D6B, 0x1D77, prAL, gcLl}, // [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G + {0x1D78, 0x1D78, prAL, gcLm}, // MODIFIER LETTER CYRILLIC EN + {0x1D79, 0x1D7F, prAL, gcLl}, // [7] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER UPSILON WITH STROKE + {0x1D80, 0x1D9A, prAL, gcLl}, // [27] LATIN SMALL LETTER B WITH PALATAL HOOK..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK + {0x1D9B, 0x1DBF, prAL, gcLm}, // [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA + {0x1DC0, 0x1DCC, prCM, gcMn}, // [13] COMBINING DOTTED GRAVE ACCENT..COMBINING MACRON-BREVE + {0x1DCD, 0x1DCD, prGL, gcMn}, // COMBINING DOUBLE CIRCUMFLEX ABOVE + {0x1DCE, 0x1DFB, prCM, gcMn}, // [46] COMBINING OGONEK ABOVE..COMBINING DELETION MARK + {0x1DFC, 0x1DFC, prGL, gcMn}, // COMBINING DOUBLE INVERTED BREVE BELOW + {0x1DFD, 0x1DFF, prCM, gcMn}, // [3] COMBINING ALMOST EQUAL TO BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW + {0x1E00, 0x1EFF, prAL, gcLC}, // [256] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER Y WITH LOOP + {0x1F00, 0x1F15, prAL, gcLC}, // [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA + {0x1F18, 0x1F1D, prAL, gcLu}, // [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + {0x1F20, 0x1F45, prAL, gcLC}, // [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA + {0x1F48, 0x1F4D, prAL, gcLu}, // [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + {0x1F50, 0x1F57, prAL, gcLl}, // [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI + {0x1F59, 0x1F59, prAL, gcLu}, // GREEK CAPITAL LETTER UPSILON WITH DASIA + {0x1F5B, 0x1F5B, prAL, gcLu}, // GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + {0x1F5D, 0x1F5D, prAL, gcLu}, // GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + {0x1F5F, 0x1F7D, prAL, gcLC}, // [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA + {0x1F80, 0x1FB4, prAL, gcLC}, // [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + {0x1FB6, 0x1FBC, prAL, gcLC}, // [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + {0x1FBD, 0x1FBD, prAL, gcSk}, // GREEK KORONIS + {0x1FBE, 0x1FBE, prAL, gcLl}, // GREEK PROSGEGRAMMENI + {0x1FBF, 0x1FC1, prAL, gcSk}, // [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI + {0x1FC2, 0x1FC4, prAL, gcLl}, // [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + {0x1FC6, 0x1FCC, prAL, gcLC}, // [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + {0x1FCD, 0x1FCF, prAL, gcSk}, // [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI + {0x1FD0, 0x1FD3, prAL, gcLl}, // [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + {0x1FD6, 0x1FDB, prAL, gcLC}, // [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA + {0x1FDD, 0x1FDF, prAL, gcSk}, // [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI + {0x1FE0, 0x1FEC, prAL, gcLC}, // [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA + {0x1FED, 0x1FEF, prAL, gcSk}, // [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA + {0x1FF2, 0x1FF4, prAL, gcLl}, // [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + {0x1FF6, 0x1FFC, prAL, gcLC}, // [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + {0x1FFD, 0x1FFD, prBB, gcSk}, // GREEK OXIA + {0x1FFE, 0x1FFE, prAL, gcSk}, // GREEK DASIA + {0x2000, 0x2006, prBA, gcZs}, // [7] EN QUAD..SIX-PER-EM SPACE + {0x2007, 0x2007, prGL, gcZs}, // FIGURE SPACE + {0x2008, 0x200A, prBA, gcZs}, // [3] PUNCTUATION SPACE..HAIR SPACE + {0x200B, 0x200B, prZW, gcCf}, // ZERO WIDTH SPACE + {0x200C, 0x200C, prCM, gcCf}, // ZERO WIDTH NON-JOINER + {0x200D, 0x200D, prZWJ, gcCf}, // ZERO WIDTH JOINER + {0x200E, 0x200F, prCM, gcCf}, // [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK + {0x2010, 0x2010, prBA, gcPd}, // HYPHEN + {0x2011, 0x2011, prGL, gcPd}, // NON-BREAKING HYPHEN + {0x2012, 0x2013, prBA, gcPd}, // [2] FIGURE DASH..EN DASH + {0x2014, 0x2014, prB2, gcPd}, // EM DASH + {0x2015, 0x2015, prAI, gcPd}, // HORIZONTAL BAR + {0x2016, 0x2016, prAI, gcPo}, // DOUBLE VERTICAL LINE + {0x2017, 0x2017, prAL, gcPo}, // DOUBLE LOW LINE + {0x2018, 0x2018, prQU, gcPi}, // LEFT SINGLE QUOTATION MARK + {0x2019, 0x2019, prQU, gcPf}, // RIGHT SINGLE QUOTATION MARK + {0x201A, 0x201A, prOP, gcPs}, // SINGLE LOW-9 QUOTATION MARK + {0x201B, 0x201C, prQU, gcPi}, // [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK + {0x201D, 0x201D, prQU, gcPf}, // RIGHT DOUBLE QUOTATION MARK + {0x201E, 0x201E, prOP, gcPs}, // DOUBLE LOW-9 QUOTATION MARK + {0x201F, 0x201F, prQU, gcPi}, // DOUBLE HIGH-REVERSED-9 QUOTATION MARK + {0x2020, 0x2021, prAI, gcPo}, // [2] DAGGER..DOUBLE DAGGER + {0x2022, 0x2023, prAL, gcPo}, // [2] BULLET..TRIANGULAR BULLET + {0x2024, 0x2026, prIN, gcPo}, // [3] ONE DOT LEADER..HORIZONTAL ELLIPSIS + {0x2027, 0x2027, prBA, gcPo}, // HYPHENATION POINT + {0x2028, 0x2028, prBK, gcZl}, // LINE SEPARATOR + {0x2029, 0x2029, prBK, gcZp}, // PARAGRAPH SEPARATOR + {0x202A, 0x202E, prCM, gcCf}, // [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE + {0x202F, 0x202F, prGL, gcZs}, // NARROW NO-BREAK SPACE + {0x2030, 0x2037, prPO, gcPo}, // [8] PER MILLE SIGN..REVERSED TRIPLE PRIME + {0x2038, 0x2038, prAL, gcPo}, // CARET + {0x2039, 0x2039, prQU, gcPi}, // SINGLE LEFT-POINTING ANGLE QUOTATION MARK + {0x203A, 0x203A, prQU, gcPf}, // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + {0x203B, 0x203B, prAI, gcPo}, // REFERENCE MARK + {0x203C, 0x203D, prNS, gcPo}, // [2] DOUBLE EXCLAMATION MARK..INTERROBANG + {0x203E, 0x203E, prAL, gcPo}, // OVERLINE + {0x203F, 0x2040, prAL, gcPc}, // [2] UNDERTIE..CHARACTER TIE + {0x2041, 0x2043, prAL, gcPo}, // [3] CARET INSERTION POINT..HYPHEN BULLET + {0x2044, 0x2044, prIS, gcSm}, // FRACTION SLASH + {0x2045, 0x2045, prOP, gcPs}, // LEFT SQUARE BRACKET WITH QUILL + {0x2046, 0x2046, prCL, gcPe}, // RIGHT SQUARE BRACKET WITH QUILL + {0x2047, 0x2049, prNS, gcPo}, // [3] DOUBLE QUESTION MARK..EXCLAMATION QUESTION MARK + {0x204A, 0x2051, prAL, gcPo}, // [8] TIRONIAN SIGN ET..TWO ASTERISKS ALIGNED VERTICALLY + {0x2052, 0x2052, prAL, gcSm}, // COMMERCIAL MINUS SIGN + {0x2053, 0x2053, prAL, gcPo}, // SWUNG DASH + {0x2054, 0x2054, prAL, gcPc}, // INVERTED UNDERTIE + {0x2055, 0x2055, prAL, gcPo}, // FLOWER PUNCTUATION MARK + {0x2056, 0x2056, prBA, gcPo}, // THREE DOT PUNCTUATION + {0x2057, 0x2057, prPO, gcPo}, // QUADRUPLE PRIME + {0x2058, 0x205B, prBA, gcPo}, // [4] FOUR DOT PUNCTUATION..FOUR DOT MARK + {0x205C, 0x205C, prAL, gcPo}, // DOTTED CROSS + {0x205D, 0x205E, prBA, gcPo}, // [2] TRICOLON..VERTICAL FOUR DOTS + {0x205F, 0x205F, prBA, gcZs}, // MEDIUM MATHEMATICAL SPACE + {0x2060, 0x2060, prWJ, gcCf}, // WORD JOINER + {0x2061, 0x2064, prAL, gcCf}, // [4] FUNCTION APPLICATION..INVISIBLE PLUS + {0x2066, 0x206F, prCM, gcCf}, // [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES + {0x2070, 0x2070, prAL, gcNo}, // SUPERSCRIPT ZERO + {0x2071, 0x2071, prAL, gcLm}, // SUPERSCRIPT LATIN SMALL LETTER I + {0x2074, 0x2074, prAI, gcNo}, // SUPERSCRIPT FOUR + {0x2075, 0x2079, prAL, gcNo}, // [5] SUPERSCRIPT FIVE..SUPERSCRIPT NINE + {0x207A, 0x207C, prAL, gcSm}, // [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN + {0x207D, 0x207D, prOP, gcPs}, // SUPERSCRIPT LEFT PARENTHESIS + {0x207E, 0x207E, prCL, gcPe}, // SUPERSCRIPT RIGHT PARENTHESIS + {0x207F, 0x207F, prAI, gcLm}, // SUPERSCRIPT LATIN SMALL LETTER N + {0x2080, 0x2080, prAL, gcNo}, // SUBSCRIPT ZERO + {0x2081, 0x2084, prAI, gcNo}, // [4] SUBSCRIPT ONE..SUBSCRIPT FOUR + {0x2085, 0x2089, prAL, gcNo}, // [5] SUBSCRIPT FIVE..SUBSCRIPT NINE + {0x208A, 0x208C, prAL, gcSm}, // [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN + {0x208D, 0x208D, prOP, gcPs}, // SUBSCRIPT LEFT PARENTHESIS + {0x208E, 0x208E, prCL, gcPe}, // SUBSCRIPT RIGHT PARENTHESIS + {0x2090, 0x209C, prAL, gcLm}, // [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T + {0x20A0, 0x20A6, prPR, gcSc}, // [7] EURO-CURRENCY SIGN..NAIRA SIGN + {0x20A7, 0x20A7, prPO, gcSc}, // PESETA SIGN + {0x20A8, 0x20B5, prPR, gcSc}, // [14] RUPEE SIGN..CEDI SIGN + {0x20B6, 0x20B6, prPO, gcSc}, // LIVRE TOURNOIS SIGN + {0x20B7, 0x20BA, prPR, gcSc}, // [4] SPESMILO SIGN..TURKISH LIRA SIGN + {0x20BB, 0x20BB, prPO, gcSc}, // NORDIC MARK SIGN + {0x20BC, 0x20BD, prPR, gcSc}, // [2] MANAT SIGN..RUBLE SIGN + {0x20BE, 0x20BE, prPO, gcSc}, // LARI SIGN + {0x20BF, 0x20BF, prPR, gcSc}, // BITCOIN SIGN + {0x20C0, 0x20C0, prPO, gcSc}, // SOM SIGN + {0x20C1, 0x20CF, prPR, gcCn}, // [15] .. + {0x20D0, 0x20DC, prCM, gcMn}, // [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE + {0x20DD, 0x20E0, prCM, gcMe}, // [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH + {0x20E1, 0x20E1, prCM, gcMn}, // COMBINING LEFT RIGHT ARROW ABOVE + {0x20E2, 0x20E4, prCM, gcMe}, // [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE + {0x20E5, 0x20F0, prCM, gcMn}, // [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE + {0x2100, 0x2101, prAL, gcSo}, // [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT + {0x2102, 0x2102, prAL, gcLu}, // DOUBLE-STRUCK CAPITAL C + {0x2103, 0x2103, prPO, gcSo}, // DEGREE CELSIUS + {0x2104, 0x2104, prAL, gcSo}, // CENTRE LINE SYMBOL + {0x2105, 0x2105, prAI, gcSo}, // CARE OF + {0x2106, 0x2106, prAL, gcSo}, // CADA UNA + {0x2107, 0x2107, prAL, gcLu}, // EULER CONSTANT + {0x2108, 0x2108, prAL, gcSo}, // SCRUPLE + {0x2109, 0x2109, prPO, gcSo}, // DEGREE FAHRENHEIT + {0x210A, 0x2112, prAL, gcLC}, // [9] SCRIPT SMALL G..SCRIPT CAPITAL L + {0x2113, 0x2113, prAI, gcLl}, // SCRIPT SMALL L + {0x2114, 0x2114, prAL, gcSo}, // L B BAR SYMBOL + {0x2115, 0x2115, prAL, gcLu}, // DOUBLE-STRUCK CAPITAL N + {0x2116, 0x2116, prPR, gcSo}, // NUMERO SIGN + {0x2117, 0x2117, prAL, gcSo}, // SOUND RECORDING COPYRIGHT + {0x2118, 0x2118, prAL, gcSm}, // SCRIPT CAPITAL P + {0x2119, 0x211D, prAL, gcLu}, // [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R + {0x211E, 0x2120, prAL, gcSo}, // [3] PRESCRIPTION TAKE..SERVICE MARK + {0x2121, 0x2122, prAI, gcSo}, // [2] TELEPHONE SIGN..TRADE MARK SIGN + {0x2123, 0x2123, prAL, gcSo}, // VERSICLE + {0x2124, 0x2124, prAL, gcLu}, // DOUBLE-STRUCK CAPITAL Z + {0x2125, 0x2125, prAL, gcSo}, // OUNCE SIGN + {0x2126, 0x2126, prAL, gcLu}, // OHM SIGN + {0x2127, 0x2127, prAL, gcSo}, // INVERTED OHM SIGN + {0x2128, 0x2128, prAL, gcLu}, // BLACK-LETTER CAPITAL Z + {0x2129, 0x2129, prAL, gcSo}, // TURNED GREEK SMALL LETTER IOTA + {0x212A, 0x212A, prAL, gcLu}, // KELVIN SIGN + {0x212B, 0x212B, prAI, gcLu}, // ANGSTROM SIGN + {0x212C, 0x212D, prAL, gcLu}, // [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C + {0x212E, 0x212E, prAL, gcSo}, // ESTIMATED SYMBOL + {0x212F, 0x2134, prAL, gcLC}, // [6] SCRIPT SMALL E..SCRIPT SMALL O + {0x2135, 0x2138, prAL, gcLo}, // [4] ALEF SYMBOL..DALET SYMBOL + {0x2139, 0x2139, prAL, gcLl}, // INFORMATION SOURCE + {0x213A, 0x213B, prAL, gcSo}, // [2] ROTATED CAPITAL Q..FACSIMILE SIGN + {0x213C, 0x213F, prAL, gcLC}, // [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI + {0x2140, 0x2144, prAL, gcSm}, // [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y + {0x2145, 0x2149, prAL, gcLC}, // [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J + {0x214A, 0x214A, prAL, gcSo}, // PROPERTY LINE + {0x214B, 0x214B, prAL, gcSm}, // TURNED AMPERSAND + {0x214C, 0x214D, prAL, gcSo}, // [2] PER SIGN..AKTIESELSKAB + {0x214E, 0x214E, prAL, gcLl}, // TURNED SMALL F + {0x214F, 0x214F, prAL, gcSo}, // SYMBOL FOR SAMARITAN SOURCE + {0x2150, 0x2153, prAL, gcNo}, // [4] VULGAR FRACTION ONE SEVENTH..VULGAR FRACTION ONE THIRD + {0x2154, 0x2155, prAI, gcNo}, // [2] VULGAR FRACTION TWO THIRDS..VULGAR FRACTION ONE FIFTH + {0x2156, 0x215A, prAL, gcNo}, // [5] VULGAR FRACTION TWO FIFTHS..VULGAR FRACTION FIVE SIXTHS + {0x215B, 0x215B, prAI, gcNo}, // VULGAR FRACTION ONE EIGHTH + {0x215C, 0x215D, prAL, gcNo}, // [2] VULGAR FRACTION THREE EIGHTHS..VULGAR FRACTION FIVE EIGHTHS + {0x215E, 0x215E, prAI, gcNo}, // VULGAR FRACTION SEVEN EIGHTHS + {0x215F, 0x215F, prAL, gcNo}, // FRACTION NUMERATOR ONE + {0x2160, 0x216B, prAI, gcNl}, // [12] ROMAN NUMERAL ONE..ROMAN NUMERAL TWELVE + {0x216C, 0x216F, prAL, gcNl}, // [4] ROMAN NUMERAL FIFTY..ROMAN NUMERAL ONE THOUSAND + {0x2170, 0x2179, prAI, gcNl}, // [10] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL TEN + {0x217A, 0x2182, prAL, gcNl}, // [9] SMALL ROMAN NUMERAL ELEVEN..ROMAN NUMERAL TEN THOUSAND + {0x2183, 0x2184, prAL, gcLC}, // [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C + {0x2185, 0x2188, prAL, gcNl}, // [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND + {0x2189, 0x2189, prAI, gcNo}, // VULGAR FRACTION ZERO THIRDS + {0x218A, 0x218B, prAL, gcSo}, // [2] TURNED DIGIT TWO..TURNED DIGIT THREE + {0x2190, 0x2194, prAI, gcSm}, // [5] LEFTWARDS ARROW..LEFT RIGHT ARROW + {0x2195, 0x2199, prAI, gcSo}, // [5] UP DOWN ARROW..SOUTH WEST ARROW + {0x219A, 0x219B, prAL, gcSm}, // [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE + {0x219C, 0x219F, prAL, gcSo}, // [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW + {0x21A0, 0x21A0, prAL, gcSm}, // RIGHTWARDS TWO HEADED ARROW + {0x21A1, 0x21A2, prAL, gcSo}, // [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL + {0x21A3, 0x21A3, prAL, gcSm}, // RIGHTWARDS ARROW WITH TAIL + {0x21A4, 0x21A5, prAL, gcSo}, // [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR + {0x21A6, 0x21A6, prAL, gcSm}, // RIGHTWARDS ARROW FROM BAR + {0x21A7, 0x21AD, prAL, gcSo}, // [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW + {0x21AE, 0x21AE, prAL, gcSm}, // LEFT RIGHT ARROW WITH STROKE + {0x21AF, 0x21CD, prAL, gcSo}, // [31] DOWNWARDS ZIGZAG ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE + {0x21CE, 0x21CF, prAL, gcSm}, // [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE + {0x21D0, 0x21D1, prAL, gcSo}, // [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW + {0x21D2, 0x21D2, prAI, gcSm}, // RIGHTWARDS DOUBLE ARROW + {0x21D3, 0x21D3, prAL, gcSo}, // DOWNWARDS DOUBLE ARROW + {0x21D4, 0x21D4, prAI, gcSm}, // LEFT RIGHT DOUBLE ARROW + {0x21D5, 0x21F3, prAL, gcSo}, // [31] UP DOWN DOUBLE ARROW..UP DOWN WHITE ARROW + {0x21F4, 0x21FF, prAL, gcSm}, // [12] RIGHT ARROW WITH SMALL CIRCLE..LEFT RIGHT OPEN-HEADED ARROW + {0x2200, 0x2200, prAI, gcSm}, // FOR ALL + {0x2201, 0x2201, prAL, gcSm}, // COMPLEMENT + {0x2202, 0x2203, prAI, gcSm}, // [2] PARTIAL DIFFERENTIAL..THERE EXISTS + {0x2204, 0x2206, prAL, gcSm}, // [3] THERE DOES NOT EXIST..INCREMENT + {0x2207, 0x2208, prAI, gcSm}, // [2] NABLA..ELEMENT OF + {0x2209, 0x220A, prAL, gcSm}, // [2] NOT AN ELEMENT OF..SMALL ELEMENT OF + {0x220B, 0x220B, prAI, gcSm}, // CONTAINS AS MEMBER + {0x220C, 0x220E, prAL, gcSm}, // [3] DOES NOT CONTAIN AS MEMBER..END OF PROOF + {0x220F, 0x220F, prAI, gcSm}, // N-ARY PRODUCT + {0x2210, 0x2210, prAL, gcSm}, // N-ARY COPRODUCT + {0x2211, 0x2211, prAI, gcSm}, // N-ARY SUMMATION + {0x2212, 0x2213, prPR, gcSm}, // [2] MINUS SIGN..MINUS-OR-PLUS SIGN + {0x2214, 0x2214, prAL, gcSm}, // DOT PLUS + {0x2215, 0x2215, prAI, gcSm}, // DIVISION SLASH + {0x2216, 0x2219, prAL, gcSm}, // [4] SET MINUS..BULLET OPERATOR + {0x221A, 0x221A, prAI, gcSm}, // SQUARE ROOT + {0x221B, 0x221C, prAL, gcSm}, // [2] CUBE ROOT..FOURTH ROOT + {0x221D, 0x2220, prAI, gcSm}, // [4] PROPORTIONAL TO..ANGLE + {0x2221, 0x2222, prAL, gcSm}, // [2] MEASURED ANGLE..SPHERICAL ANGLE + {0x2223, 0x2223, prAI, gcSm}, // DIVIDES + {0x2224, 0x2224, prAL, gcSm}, // DOES NOT DIVIDE + {0x2225, 0x2225, prAI, gcSm}, // PARALLEL TO + {0x2226, 0x2226, prAL, gcSm}, // NOT PARALLEL TO + {0x2227, 0x222C, prAI, gcSm}, // [6] LOGICAL AND..DOUBLE INTEGRAL + {0x222D, 0x222D, prAL, gcSm}, // TRIPLE INTEGRAL + {0x222E, 0x222E, prAI, gcSm}, // CONTOUR INTEGRAL + {0x222F, 0x2233, prAL, gcSm}, // [5] SURFACE INTEGRAL..ANTICLOCKWISE CONTOUR INTEGRAL + {0x2234, 0x2237, prAI, gcSm}, // [4] THEREFORE..PROPORTION + {0x2238, 0x223B, prAL, gcSm}, // [4] DOT MINUS..HOMOTHETIC + {0x223C, 0x223D, prAI, gcSm}, // [2] TILDE OPERATOR..REVERSED TILDE + {0x223E, 0x2247, prAL, gcSm}, // [10] INVERTED LAZY S..NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO + {0x2248, 0x2248, prAI, gcSm}, // ALMOST EQUAL TO + {0x2249, 0x224B, prAL, gcSm}, // [3] NOT ALMOST EQUAL TO..TRIPLE TILDE + {0x224C, 0x224C, prAI, gcSm}, // ALL EQUAL TO + {0x224D, 0x2251, prAL, gcSm}, // [5] EQUIVALENT TO..GEOMETRICALLY EQUAL TO + {0x2252, 0x2252, prAI, gcSm}, // APPROXIMATELY EQUAL TO OR THE IMAGE OF + {0x2253, 0x225F, prAL, gcSm}, // [13] IMAGE OF OR APPROXIMATELY EQUAL TO..QUESTIONED EQUAL TO + {0x2260, 0x2261, prAI, gcSm}, // [2] NOT EQUAL TO..IDENTICAL TO + {0x2262, 0x2263, prAL, gcSm}, // [2] NOT IDENTICAL TO..STRICTLY EQUIVALENT TO + {0x2264, 0x2267, prAI, gcSm}, // [4] LESS-THAN OR EQUAL TO..GREATER-THAN OVER EQUAL TO + {0x2268, 0x2269, prAL, gcSm}, // [2] LESS-THAN BUT NOT EQUAL TO..GREATER-THAN BUT NOT EQUAL TO + {0x226A, 0x226B, prAI, gcSm}, // [2] MUCH LESS-THAN..MUCH GREATER-THAN + {0x226C, 0x226D, prAL, gcSm}, // [2] BETWEEN..NOT EQUIVALENT TO + {0x226E, 0x226F, prAI, gcSm}, // [2] NOT LESS-THAN..NOT GREATER-THAN + {0x2270, 0x2281, prAL, gcSm}, // [18] NEITHER LESS-THAN NOR EQUAL TO..DOES NOT SUCCEED + {0x2282, 0x2283, prAI, gcSm}, // [2] SUBSET OF..SUPERSET OF + {0x2284, 0x2285, prAL, gcSm}, // [2] NOT A SUBSET OF..NOT A SUPERSET OF + {0x2286, 0x2287, prAI, gcSm}, // [2] SUBSET OF OR EQUAL TO..SUPERSET OF OR EQUAL TO + {0x2288, 0x2294, prAL, gcSm}, // [13] NEITHER A SUBSET OF NOR EQUAL TO..SQUARE CUP + {0x2295, 0x2295, prAI, gcSm}, // CIRCLED PLUS + {0x2296, 0x2298, prAL, gcSm}, // [3] CIRCLED MINUS..CIRCLED DIVISION SLASH + {0x2299, 0x2299, prAI, gcSm}, // CIRCLED DOT OPERATOR + {0x229A, 0x22A4, prAL, gcSm}, // [11] CIRCLED RING OPERATOR..DOWN TACK + {0x22A5, 0x22A5, prAI, gcSm}, // UP TACK + {0x22A6, 0x22BE, prAL, gcSm}, // [25] ASSERTION..RIGHT ANGLE WITH ARC + {0x22BF, 0x22BF, prAI, gcSm}, // RIGHT TRIANGLE + {0x22C0, 0x22EE, prAL, gcSm}, // [47] N-ARY LOGICAL AND..VERTICAL ELLIPSIS + {0x22EF, 0x22EF, prIN, gcSm}, // MIDLINE HORIZONTAL ELLIPSIS + {0x22F0, 0x22FF, prAL, gcSm}, // [16] UP RIGHT DIAGONAL ELLIPSIS..Z NOTATION BAG MEMBERSHIP + {0x2300, 0x2307, prAL, gcSo}, // [8] DIAMETER SIGN..WAVY LINE + {0x2308, 0x2308, prOP, gcPs}, // LEFT CEILING + {0x2309, 0x2309, prCL, gcPe}, // RIGHT CEILING + {0x230A, 0x230A, prOP, gcPs}, // LEFT FLOOR + {0x230B, 0x230B, prCL, gcPe}, // RIGHT FLOOR + {0x230C, 0x2311, prAL, gcSo}, // [6] BOTTOM RIGHT CROP..SQUARE LOZENGE + {0x2312, 0x2312, prAI, gcSo}, // ARC + {0x2313, 0x2319, prAL, gcSo}, // [7] SEGMENT..TURNED NOT SIGN + {0x231A, 0x231B, prID, gcSo}, // [2] WATCH..HOURGLASS + {0x231C, 0x231F, prAL, gcSo}, // [4] TOP LEFT CORNER..BOTTOM RIGHT CORNER + {0x2320, 0x2321, prAL, gcSm}, // [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL + {0x2322, 0x2328, prAL, gcSo}, // [7] FROWN..KEYBOARD + {0x2329, 0x2329, prOP, gcPs}, // LEFT-POINTING ANGLE BRACKET + {0x232A, 0x232A, prCL, gcPe}, // RIGHT-POINTING ANGLE BRACKET + {0x232B, 0x237B, prAL, gcSo}, // [81] ERASE TO THE LEFT..NOT CHECK MARK + {0x237C, 0x237C, prAL, gcSm}, // RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW + {0x237D, 0x239A, prAL, gcSo}, // [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL + {0x239B, 0x23B3, prAL, gcSm}, // [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM + {0x23B4, 0x23DB, prAL, gcSo}, // [40] TOP SQUARE BRACKET..FUSE + {0x23DC, 0x23E1, prAL, gcSm}, // [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET + {0x23E2, 0x23EF, prAL, gcSo}, // [14] WHITE TRAPEZIUM..BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR + {0x23F0, 0x23F3, prID, gcSo}, // [4] ALARM CLOCK..HOURGLASS WITH FLOWING SAND + {0x23F4, 0x23FF, prAL, gcSo}, // [12] BLACK MEDIUM LEFT-POINTING TRIANGLE..OBSERVER EYE SYMBOL + {0x2400, 0x2426, prAL, gcSo}, // [39] SYMBOL FOR NULL..SYMBOL FOR SUBSTITUTE FORM TWO + {0x2440, 0x244A, prAL, gcSo}, // [11] OCR HOOK..OCR DOUBLE BACKSLASH + {0x2460, 0x249B, prAI, gcNo}, // [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP + {0x249C, 0x24E9, prAI, gcSo}, // [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z + {0x24EA, 0x24FE, prAI, gcNo}, // [21] CIRCLED DIGIT ZERO..DOUBLE CIRCLED NUMBER TEN + {0x24FF, 0x24FF, prAL, gcNo}, // NEGATIVE CIRCLED DIGIT ZERO + {0x2500, 0x254B, prAI, gcSo}, // [76] BOX DRAWINGS LIGHT HORIZONTAL..BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL + {0x254C, 0x254F, prAL, gcSo}, // [4] BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL..BOX DRAWINGS HEAVY DOUBLE DASH VERTICAL + {0x2550, 0x2574, prAI, gcSo}, // [37] BOX DRAWINGS DOUBLE HORIZONTAL..BOX DRAWINGS LIGHT LEFT + {0x2575, 0x257F, prAL, gcSo}, // [11] BOX DRAWINGS LIGHT UP..BOX DRAWINGS HEAVY UP AND LIGHT DOWN + {0x2580, 0x258F, prAI, gcSo}, // [16] UPPER HALF BLOCK..LEFT ONE EIGHTH BLOCK + {0x2590, 0x2591, prAL, gcSo}, // [2] RIGHT HALF BLOCK..LIGHT SHADE + {0x2592, 0x2595, prAI, gcSo}, // [4] MEDIUM SHADE..RIGHT ONE EIGHTH BLOCK + {0x2596, 0x259F, prAL, gcSo}, // [10] QUADRANT LOWER LEFT..QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT + {0x25A0, 0x25A1, prAI, gcSo}, // [2] BLACK SQUARE..WHITE SQUARE + {0x25A2, 0x25A2, prAL, gcSo}, // WHITE SQUARE WITH ROUNDED CORNERS + {0x25A3, 0x25A9, prAI, gcSo}, // [7] WHITE SQUARE CONTAINING BLACK SMALL SQUARE..SQUARE WITH DIAGONAL CROSSHATCH FILL + {0x25AA, 0x25B1, prAL, gcSo}, // [8] BLACK SMALL SQUARE..WHITE PARALLELOGRAM + {0x25B2, 0x25B3, prAI, gcSo}, // [2] BLACK UP-POINTING TRIANGLE..WHITE UP-POINTING TRIANGLE + {0x25B4, 0x25B5, prAL, gcSo}, // [2] BLACK UP-POINTING SMALL TRIANGLE..WHITE UP-POINTING SMALL TRIANGLE + {0x25B6, 0x25B6, prAI, gcSo}, // BLACK RIGHT-POINTING TRIANGLE + {0x25B7, 0x25B7, prAI, gcSm}, // WHITE RIGHT-POINTING TRIANGLE + {0x25B8, 0x25BB, prAL, gcSo}, // [4] BLACK RIGHT-POINTING SMALL TRIANGLE..WHITE RIGHT-POINTING POINTER + {0x25BC, 0x25BD, prAI, gcSo}, // [2] BLACK DOWN-POINTING TRIANGLE..WHITE DOWN-POINTING TRIANGLE + {0x25BE, 0x25BF, prAL, gcSo}, // [2] BLACK DOWN-POINTING SMALL TRIANGLE..WHITE DOWN-POINTING SMALL TRIANGLE + {0x25C0, 0x25C0, prAI, gcSo}, // BLACK LEFT-POINTING TRIANGLE + {0x25C1, 0x25C1, prAI, gcSm}, // WHITE LEFT-POINTING TRIANGLE + {0x25C2, 0x25C5, prAL, gcSo}, // [4] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE LEFT-POINTING POINTER + {0x25C6, 0x25C8, prAI, gcSo}, // [3] BLACK DIAMOND..WHITE DIAMOND CONTAINING BLACK SMALL DIAMOND + {0x25C9, 0x25CA, prAL, gcSo}, // [2] FISHEYE..LOZENGE + {0x25CB, 0x25CB, prAI, gcSo}, // WHITE CIRCLE + {0x25CC, 0x25CD, prAL, gcSo}, // [2] DOTTED CIRCLE..CIRCLE WITH VERTICAL FILL + {0x25CE, 0x25D1, prAI, gcSo}, // [4] BULLSEYE..CIRCLE WITH RIGHT HALF BLACK + {0x25D2, 0x25E1, prAL, gcSo}, // [16] CIRCLE WITH LOWER HALF BLACK..LOWER HALF CIRCLE + {0x25E2, 0x25E5, prAI, gcSo}, // [4] BLACK LOWER RIGHT TRIANGLE..BLACK UPPER RIGHT TRIANGLE + {0x25E6, 0x25EE, prAL, gcSo}, // [9] WHITE BULLET..UP-POINTING TRIANGLE WITH RIGHT HALF BLACK + {0x25EF, 0x25EF, prAI, gcSo}, // LARGE CIRCLE + {0x25F0, 0x25F7, prAL, gcSo}, // [8] WHITE SQUARE WITH UPPER LEFT QUADRANT..WHITE CIRCLE WITH UPPER RIGHT QUADRANT + {0x25F8, 0x25FF, prAL, gcSm}, // [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE + {0x2600, 0x2603, prID, gcSo}, // [4] BLACK SUN WITH RAYS..SNOWMAN + {0x2604, 0x2604, prAL, gcSo}, // COMET + {0x2605, 0x2606, prAI, gcSo}, // [2] BLACK STAR..WHITE STAR + {0x2607, 0x2608, prAL, gcSo}, // [2] LIGHTNING..THUNDERSTORM + {0x2609, 0x2609, prAI, gcSo}, // SUN + {0x260A, 0x260D, prAL, gcSo}, // [4] ASCENDING NODE..OPPOSITION + {0x260E, 0x260F, prAI, gcSo}, // [2] BLACK TELEPHONE..WHITE TELEPHONE + {0x2610, 0x2613, prAL, gcSo}, // [4] BALLOT BOX..SALTIRE + {0x2614, 0x2615, prID, gcSo}, // [2] UMBRELLA WITH RAIN DROPS..HOT BEVERAGE + {0x2616, 0x2617, prAI, gcSo}, // [2] WHITE SHOGI PIECE..BLACK SHOGI PIECE + {0x2618, 0x2618, prID, gcSo}, // SHAMROCK + {0x2619, 0x2619, prAL, gcSo}, // REVERSED ROTATED FLORAL HEART BULLET + {0x261A, 0x261C, prID, gcSo}, // [3] BLACK LEFT POINTING INDEX..WHITE LEFT POINTING INDEX + {0x261D, 0x261D, prEB, gcSo}, // WHITE UP POINTING INDEX + {0x261E, 0x261F, prID, gcSo}, // [2] WHITE RIGHT POINTING INDEX..WHITE DOWN POINTING INDEX + {0x2620, 0x2638, prAL, gcSo}, // [25] SKULL AND CROSSBONES..WHEEL OF DHARMA + {0x2639, 0x263B, prID, gcSo}, // [3] WHITE FROWNING FACE..BLACK SMILING FACE + {0x263C, 0x263F, prAL, gcSo}, // [4] WHITE SUN WITH RAYS..MERCURY + {0x2640, 0x2640, prAI, gcSo}, // FEMALE SIGN + {0x2641, 0x2641, prAL, gcSo}, // EARTH + {0x2642, 0x2642, prAI, gcSo}, // MALE SIGN + {0x2643, 0x265F, prAL, gcSo}, // [29] JUPITER..BLACK CHESS PAWN + {0x2660, 0x2661, prAI, gcSo}, // [2] BLACK SPADE SUIT..WHITE HEART SUIT + {0x2662, 0x2662, prAL, gcSo}, // WHITE DIAMOND SUIT + {0x2663, 0x2665, prAI, gcSo}, // [3] BLACK CLUB SUIT..BLACK HEART SUIT + {0x2666, 0x2666, prAL, gcSo}, // BLACK DIAMOND SUIT + {0x2667, 0x2667, prAI, gcSo}, // WHITE CLUB SUIT + {0x2668, 0x2668, prID, gcSo}, // HOT SPRINGS + {0x2669, 0x266A, prAI, gcSo}, // [2] QUARTER NOTE..EIGHTH NOTE + {0x266B, 0x266B, prAL, gcSo}, // BEAMED EIGHTH NOTES + {0x266C, 0x266D, prAI, gcSo}, // [2] BEAMED SIXTEENTH NOTES..MUSIC FLAT SIGN + {0x266E, 0x266E, prAL, gcSo}, // MUSIC NATURAL SIGN + {0x266F, 0x266F, prAI, gcSm}, // MUSIC SHARP SIGN + {0x2670, 0x267E, prAL, gcSo}, // [15] WEST SYRIAC CROSS..PERMANENT PAPER SIGN + {0x267F, 0x267F, prID, gcSo}, // WHEELCHAIR SYMBOL + {0x2680, 0x269D, prAL, gcSo}, // [30] DIE FACE-1..OUTLINED WHITE STAR + {0x269E, 0x269F, prAI, gcSo}, // [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT + {0x26A0, 0x26BC, prAL, gcSo}, // [29] WARNING SIGN..SESQUIQUADRATE + {0x26BD, 0x26C8, prID, gcSo}, // [12] SOCCER BALL..THUNDER CLOUD AND RAIN + {0x26C9, 0x26CC, prAI, gcSo}, // [4] TURNED WHITE SHOGI PIECE..CROSSING LANES + {0x26CD, 0x26CD, prID, gcSo}, // DISABLED CAR + {0x26CE, 0x26CE, prAL, gcSo}, // OPHIUCHUS + {0x26CF, 0x26D1, prID, gcSo}, // [3] PICK..HELMET WITH WHITE CROSS + {0x26D2, 0x26D2, prAI, gcSo}, // CIRCLED CROSSING LANES + {0x26D3, 0x26D4, prID, gcSo}, // [2] CHAINS..NO ENTRY + {0x26D5, 0x26D7, prAI, gcSo}, // [3] ALTERNATE ONE-WAY LEFT WAY TRAFFIC..WHITE TWO-WAY LEFT WAY TRAFFIC + {0x26D8, 0x26D9, prID, gcSo}, // [2] BLACK LEFT LANE MERGE..WHITE LEFT LANE MERGE + {0x26DA, 0x26DB, prAI, gcSo}, // [2] DRIVE SLOW SIGN..HEAVY WHITE DOWN-POINTING TRIANGLE + {0x26DC, 0x26DC, prID, gcSo}, // LEFT CLOSED ENTRY + {0x26DD, 0x26DE, prAI, gcSo}, // [2] SQUARED SALTIRE..FALLING DIAGONAL IN WHITE CIRCLE IN BLACK SQUARE + {0x26DF, 0x26E1, prID, gcSo}, // [3] BLACK TRUCK..RESTRICTED LEFT ENTRY-2 + {0x26E2, 0x26E2, prAL, gcSo}, // ASTRONOMICAL SYMBOL FOR URANUS + {0x26E3, 0x26E3, prAI, gcSo}, // HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE + {0x26E4, 0x26E7, prAL, gcSo}, // [4] PENTAGRAM..INVERTED PENTAGRAM + {0x26E8, 0x26E9, prAI, gcSo}, // [2] BLACK CROSS ON SHIELD..SHINTO SHRINE + {0x26EA, 0x26EA, prID, gcSo}, // CHURCH + {0x26EB, 0x26F0, prAI, gcSo}, // [6] CASTLE..MOUNTAIN + {0x26F1, 0x26F5, prID, gcSo}, // [5] UMBRELLA ON GROUND..SAILBOAT + {0x26F6, 0x26F6, prAI, gcSo}, // SQUARE FOUR CORNERS + {0x26F7, 0x26F8, prID, gcSo}, // [2] SKIER..ICE SKATE + {0x26F9, 0x26F9, prEB, gcSo}, // PERSON WITH BALL + {0x26FA, 0x26FA, prID, gcSo}, // TENT + {0x26FB, 0x26FC, prAI, gcSo}, // [2] JAPANESE BANK SYMBOL..HEADSTONE GRAVEYARD SYMBOL + {0x26FD, 0x26FF, prID, gcSo}, // [3] FUEL PUMP..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE + {0x2700, 0x2704, prID, gcSo}, // [5] BLACK SAFETY SCISSORS..WHITE SCISSORS + {0x2705, 0x2707, prAL, gcSo}, // [3] WHITE HEAVY CHECK MARK..TAPE DRIVE + {0x2708, 0x2709, prID, gcSo}, // [2] AIRPLANE..ENVELOPE + {0x270A, 0x270D, prEB, gcSo}, // [4] RAISED FIST..WRITING HAND + {0x270E, 0x2756, prAL, gcSo}, // [73] LOWER RIGHT PENCIL..BLACK DIAMOND MINUS WHITE X + {0x2757, 0x2757, prAI, gcSo}, // HEAVY EXCLAMATION MARK SYMBOL + {0x2758, 0x275A, prAL, gcSo}, // [3] LIGHT VERTICAL BAR..HEAVY VERTICAL BAR + {0x275B, 0x2760, prQU, gcSo}, // [6] HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT..HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT + {0x2761, 0x2761, prAL, gcSo}, // CURVED STEM PARAGRAPH SIGN ORNAMENT + {0x2762, 0x2763, prEX, gcSo}, // [2] HEAVY EXCLAMATION MARK ORNAMENT..HEAVY HEART EXCLAMATION MARK ORNAMENT + {0x2764, 0x2764, prID, gcSo}, // HEAVY BLACK HEART + {0x2765, 0x2767, prAL, gcSo}, // [3] ROTATED HEAVY BLACK HEART BULLET..ROTATED FLORAL HEART BULLET + {0x2768, 0x2768, prOP, gcPs}, // MEDIUM LEFT PARENTHESIS ORNAMENT + {0x2769, 0x2769, prCL, gcPe}, // MEDIUM RIGHT PARENTHESIS ORNAMENT + {0x276A, 0x276A, prOP, gcPs}, // MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT + {0x276B, 0x276B, prCL, gcPe}, // MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT + {0x276C, 0x276C, prOP, gcPs}, // MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT + {0x276D, 0x276D, prCL, gcPe}, // MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT + {0x276E, 0x276E, prOP, gcPs}, // HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT + {0x276F, 0x276F, prCL, gcPe}, // HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT + {0x2770, 0x2770, prOP, gcPs}, // HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT + {0x2771, 0x2771, prCL, gcPe}, // HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT + {0x2772, 0x2772, prOP, gcPs}, // LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT + {0x2773, 0x2773, prCL, gcPe}, // LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT + {0x2774, 0x2774, prOP, gcPs}, // MEDIUM LEFT CURLY BRACKET ORNAMENT + {0x2775, 0x2775, prCL, gcPe}, // MEDIUM RIGHT CURLY BRACKET ORNAMENT + {0x2776, 0x2793, prAI, gcNo}, // [30] DINGBAT NEGATIVE CIRCLED DIGIT ONE..DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN + {0x2794, 0x27BF, prAL, gcSo}, // [44] HEAVY WIDE-HEADED RIGHTWARDS ARROW..DOUBLE CURLY LOOP + {0x27C0, 0x27C4, prAL, gcSm}, // [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET + {0x27C5, 0x27C5, prOP, gcPs}, // LEFT S-SHAPED BAG DELIMITER + {0x27C6, 0x27C6, prCL, gcPe}, // RIGHT S-SHAPED BAG DELIMITER + {0x27C7, 0x27E5, prAL, gcSm}, // [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK + {0x27E6, 0x27E6, prOP, gcPs}, // MATHEMATICAL LEFT WHITE SQUARE BRACKET + {0x27E7, 0x27E7, prCL, gcPe}, // MATHEMATICAL RIGHT WHITE SQUARE BRACKET + {0x27E8, 0x27E8, prOP, gcPs}, // MATHEMATICAL LEFT ANGLE BRACKET + {0x27E9, 0x27E9, prCL, gcPe}, // MATHEMATICAL RIGHT ANGLE BRACKET + {0x27EA, 0x27EA, prOP, gcPs}, // MATHEMATICAL LEFT DOUBLE ANGLE BRACKET + {0x27EB, 0x27EB, prCL, gcPe}, // MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET + {0x27EC, 0x27EC, prOP, gcPs}, // MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET + {0x27ED, 0x27ED, prCL, gcPe}, // MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET + {0x27EE, 0x27EE, prOP, gcPs}, // MATHEMATICAL LEFT FLATTENED PARENTHESIS + {0x27EF, 0x27EF, prCL, gcPe}, // MATHEMATICAL RIGHT FLATTENED PARENTHESIS + {0x27F0, 0x27FF, prAL, gcSm}, // [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW + {0x2800, 0x28FF, prAL, gcSo}, // [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 + {0x2900, 0x297F, prAL, gcSm}, // [128] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..DOWN FISH TAIL + {0x2980, 0x2982, prAL, gcSm}, // [3] TRIPLE VERTICAL BAR DELIMITER..Z NOTATION TYPE COLON + {0x2983, 0x2983, prOP, gcPs}, // LEFT WHITE CURLY BRACKET + {0x2984, 0x2984, prCL, gcPe}, // RIGHT WHITE CURLY BRACKET + {0x2985, 0x2985, prOP, gcPs}, // LEFT WHITE PARENTHESIS + {0x2986, 0x2986, prCL, gcPe}, // RIGHT WHITE PARENTHESIS + {0x2987, 0x2987, prOP, gcPs}, // Z NOTATION LEFT IMAGE BRACKET + {0x2988, 0x2988, prCL, gcPe}, // Z NOTATION RIGHT IMAGE BRACKET + {0x2989, 0x2989, prOP, gcPs}, // Z NOTATION LEFT BINDING BRACKET + {0x298A, 0x298A, prCL, gcPe}, // Z NOTATION RIGHT BINDING BRACKET + {0x298B, 0x298B, prOP, gcPs}, // LEFT SQUARE BRACKET WITH UNDERBAR + {0x298C, 0x298C, prCL, gcPe}, // RIGHT SQUARE BRACKET WITH UNDERBAR + {0x298D, 0x298D, prOP, gcPs}, // LEFT SQUARE BRACKET WITH TICK IN TOP CORNER + {0x298E, 0x298E, prCL, gcPe}, // RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + {0x298F, 0x298F, prOP, gcPs}, // LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + {0x2990, 0x2990, prCL, gcPe}, // RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER + {0x2991, 0x2991, prOP, gcPs}, // LEFT ANGLE BRACKET WITH DOT + {0x2992, 0x2992, prCL, gcPe}, // RIGHT ANGLE BRACKET WITH DOT + {0x2993, 0x2993, prOP, gcPs}, // LEFT ARC LESS-THAN BRACKET + {0x2994, 0x2994, prCL, gcPe}, // RIGHT ARC GREATER-THAN BRACKET + {0x2995, 0x2995, prOP, gcPs}, // DOUBLE LEFT ARC GREATER-THAN BRACKET + {0x2996, 0x2996, prCL, gcPe}, // DOUBLE RIGHT ARC LESS-THAN BRACKET + {0x2997, 0x2997, prOP, gcPs}, // LEFT BLACK TORTOISE SHELL BRACKET + {0x2998, 0x2998, prCL, gcPe}, // RIGHT BLACK TORTOISE SHELL BRACKET + {0x2999, 0x29D7, prAL, gcSm}, // [63] DOTTED FENCE..BLACK HOURGLASS + {0x29D8, 0x29D8, prOP, gcPs}, // LEFT WIGGLY FENCE + {0x29D9, 0x29D9, prCL, gcPe}, // RIGHT WIGGLY FENCE + {0x29DA, 0x29DA, prOP, gcPs}, // LEFT DOUBLE WIGGLY FENCE + {0x29DB, 0x29DB, prCL, gcPe}, // RIGHT DOUBLE WIGGLY FENCE + {0x29DC, 0x29FB, prAL, gcSm}, // [32] INCOMPLETE INFINITY..TRIPLE PLUS + {0x29FC, 0x29FC, prOP, gcPs}, // LEFT-POINTING CURVED ANGLE BRACKET + {0x29FD, 0x29FD, prCL, gcPe}, // RIGHT-POINTING CURVED ANGLE BRACKET + {0x29FE, 0x29FF, prAL, gcSm}, // [2] TINY..MINY + {0x2A00, 0x2AFF, prAL, gcSm}, // [256] N-ARY CIRCLED DOT OPERATOR..N-ARY WHITE VERTICAL BAR + {0x2B00, 0x2B2F, prAL, gcSo}, // [48] NORTH EAST WHITE ARROW..WHITE VERTICAL ELLIPSE + {0x2B30, 0x2B44, prAL, gcSm}, // [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET + {0x2B45, 0x2B46, prAL, gcSo}, // [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW + {0x2B47, 0x2B4C, prAL, gcSm}, // [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR + {0x2B4D, 0x2B54, prAL, gcSo}, // [8] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..WHITE RIGHT-POINTING PENTAGON + {0x2B55, 0x2B59, prAI, gcSo}, // [5] HEAVY LARGE CIRCLE..HEAVY CIRCLED SALTIRE + {0x2B5A, 0x2B73, prAL, gcSo}, // [26] SLANTED NORTH ARROW WITH HOOKED HEAD..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR + {0x2B76, 0x2B95, prAL, gcSo}, // [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW + {0x2B97, 0x2BFF, prAL, gcSo}, // [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL + {0x2C00, 0x2C5F, prAL, gcLC}, // [96] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC SMALL LETTER CAUDATE CHRIVI + {0x2C60, 0x2C7B, prAL, gcLC}, // [28] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN LETTER SMALL CAPITAL TURNED E + {0x2C7C, 0x2C7D, prAL, gcLm}, // [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V + {0x2C7E, 0x2C7F, prAL, gcLu}, // [2] LATIN CAPITAL LETTER S WITH SWASH TAIL..LATIN CAPITAL LETTER Z WITH SWASH TAIL + {0x2C80, 0x2CE4, prAL, gcLC}, // [101] COPTIC CAPITAL LETTER ALFA..COPTIC SYMBOL KAI + {0x2CE5, 0x2CEA, prAL, gcSo}, // [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA + {0x2CEB, 0x2CEE, prAL, gcLC}, // [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA + {0x2CEF, 0x2CF1, prCM, gcMn}, // [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS + {0x2CF2, 0x2CF3, prAL, gcLC}, // [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI + {0x2CF9, 0x2CF9, prEX, gcPo}, // COPTIC OLD NUBIAN FULL STOP + {0x2CFA, 0x2CFC, prBA, gcPo}, // [3] COPTIC OLD NUBIAN DIRECT QUESTION MARK..COPTIC OLD NUBIAN VERSE DIVIDER + {0x2CFD, 0x2CFD, prAL, gcNo}, // COPTIC FRACTION ONE HALF + {0x2CFE, 0x2CFE, prEX, gcPo}, // COPTIC FULL STOP + {0x2CFF, 0x2CFF, prBA, gcPo}, // COPTIC MORPHOLOGICAL DIVIDER + {0x2D00, 0x2D25, prAL, gcLl}, // [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE + {0x2D27, 0x2D27, prAL, gcLl}, // GEORGIAN SMALL LETTER YN + {0x2D2D, 0x2D2D, prAL, gcLl}, // GEORGIAN SMALL LETTER AEN + {0x2D30, 0x2D67, prAL, gcLo}, // [56] TIFINAGH LETTER YA..TIFINAGH LETTER YO + {0x2D6F, 0x2D6F, prAL, gcLm}, // TIFINAGH MODIFIER LETTER LABIALIZATION MARK + {0x2D70, 0x2D70, prBA, gcPo}, // TIFINAGH SEPARATOR MARK + {0x2D7F, 0x2D7F, prCM, gcMn}, // TIFINAGH CONSONANT JOINER + {0x2D80, 0x2D96, prAL, gcLo}, // [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE + {0x2DA0, 0x2DA6, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO + {0x2DA8, 0x2DAE, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO + {0x2DB0, 0x2DB6, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO + {0x2DB8, 0x2DBE, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO + {0x2DC0, 0x2DC6, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO + {0x2DC8, 0x2DCE, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO + {0x2DD0, 0x2DD6, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO + {0x2DD8, 0x2DDE, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO + {0x2DE0, 0x2DFF, prCM, gcMn}, // [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS + {0x2E00, 0x2E01, prQU, gcPo}, // [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER + {0x2E02, 0x2E02, prQU, gcPi}, // LEFT SUBSTITUTION BRACKET + {0x2E03, 0x2E03, prQU, gcPf}, // RIGHT SUBSTITUTION BRACKET + {0x2E04, 0x2E04, prQU, gcPi}, // LEFT DOTTED SUBSTITUTION BRACKET + {0x2E05, 0x2E05, prQU, gcPf}, // RIGHT DOTTED SUBSTITUTION BRACKET + {0x2E06, 0x2E08, prQU, gcPo}, // [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER + {0x2E09, 0x2E09, prQU, gcPi}, // LEFT TRANSPOSITION BRACKET + {0x2E0A, 0x2E0A, prQU, gcPf}, // RIGHT TRANSPOSITION BRACKET + {0x2E0B, 0x2E0B, prQU, gcPo}, // RAISED SQUARE + {0x2E0C, 0x2E0C, prQU, gcPi}, // LEFT RAISED OMISSION BRACKET + {0x2E0D, 0x2E0D, prQU, gcPf}, // RIGHT RAISED OMISSION BRACKET + {0x2E0E, 0x2E15, prBA, gcPo}, // [8] EDITORIAL CORONIS..UPWARDS ANCORA + {0x2E16, 0x2E16, prAL, gcPo}, // DOTTED RIGHT-POINTING ANGLE + {0x2E17, 0x2E17, prBA, gcPd}, // DOUBLE OBLIQUE HYPHEN + {0x2E18, 0x2E18, prOP, gcPo}, // INVERTED INTERROBANG + {0x2E19, 0x2E19, prBA, gcPo}, // PALM BRANCH + {0x2E1A, 0x2E1A, prAL, gcPd}, // HYPHEN WITH DIAERESIS + {0x2E1B, 0x2E1B, prAL, gcPo}, // TILDE WITH RING ABOVE + {0x2E1C, 0x2E1C, prQU, gcPi}, // LEFT LOW PARAPHRASE BRACKET + {0x2E1D, 0x2E1D, prQU, gcPf}, // RIGHT LOW PARAPHRASE BRACKET + {0x2E1E, 0x2E1F, prAL, gcPo}, // [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW + {0x2E20, 0x2E20, prQU, gcPi}, // LEFT VERTICAL BAR WITH QUILL + {0x2E21, 0x2E21, prQU, gcPf}, // RIGHT VERTICAL BAR WITH QUILL + {0x2E22, 0x2E22, prOP, gcPs}, // TOP LEFT HALF BRACKET + {0x2E23, 0x2E23, prCL, gcPe}, // TOP RIGHT HALF BRACKET + {0x2E24, 0x2E24, prOP, gcPs}, // BOTTOM LEFT HALF BRACKET + {0x2E25, 0x2E25, prCL, gcPe}, // BOTTOM RIGHT HALF BRACKET + {0x2E26, 0x2E26, prOP, gcPs}, // LEFT SIDEWAYS U BRACKET + {0x2E27, 0x2E27, prCL, gcPe}, // RIGHT SIDEWAYS U BRACKET + {0x2E28, 0x2E28, prOP, gcPs}, // LEFT DOUBLE PARENTHESIS + {0x2E29, 0x2E29, prCL, gcPe}, // RIGHT DOUBLE PARENTHESIS + {0x2E2A, 0x2E2D, prBA, gcPo}, // [4] TWO DOTS OVER ONE DOT PUNCTUATION..FIVE DOT MARK + {0x2E2E, 0x2E2E, prEX, gcPo}, // REVERSED QUESTION MARK + {0x2E2F, 0x2E2F, prAL, gcLm}, // VERTICAL TILDE + {0x2E30, 0x2E31, prBA, gcPo}, // [2] RING POINT..WORD SEPARATOR MIDDLE DOT + {0x2E32, 0x2E32, prAL, gcPo}, // TURNED COMMA + {0x2E33, 0x2E34, prBA, gcPo}, // [2] RAISED DOT..RAISED COMMA + {0x2E35, 0x2E39, prAL, gcPo}, // [5] TURNED SEMICOLON..TOP HALF SECTION SIGN + {0x2E3A, 0x2E3B, prB2, gcPd}, // [2] TWO-EM DASH..THREE-EM DASH + {0x2E3C, 0x2E3E, prBA, gcPo}, // [3] STENOGRAPHIC FULL STOP..WIGGLY VERTICAL LINE + {0x2E3F, 0x2E3F, prAL, gcPo}, // CAPITULUM + {0x2E40, 0x2E40, prBA, gcPd}, // DOUBLE HYPHEN + {0x2E41, 0x2E41, prBA, gcPo}, // REVERSED COMMA + {0x2E42, 0x2E42, prOP, gcPs}, // DOUBLE LOW-REVERSED-9 QUOTATION MARK + {0x2E43, 0x2E4A, prBA, gcPo}, // [8] DASH WITH LEFT UPTURN..DOTTED SOLIDUS + {0x2E4B, 0x2E4B, prAL, gcPo}, // TRIPLE DAGGER + {0x2E4C, 0x2E4C, prBA, gcPo}, // MEDIEVAL COMMA + {0x2E4D, 0x2E4D, prAL, gcPo}, // PARAGRAPHUS MARK + {0x2E4E, 0x2E4F, prBA, gcPo}, // [2] PUNCTUS ELEVATUS MARK..CORNISH VERSE DIVIDER + {0x2E50, 0x2E51, prAL, gcSo}, // [2] CROSS PATTY WITH RIGHT CROSSBAR..CROSS PATTY WITH LEFT CROSSBAR + {0x2E52, 0x2E52, prAL, gcPo}, // TIRONIAN SIGN CAPITAL ET + {0x2E53, 0x2E54, prEX, gcPo}, // [2] MEDIEVAL EXCLAMATION MARK..MEDIEVAL QUESTION MARK + {0x2E55, 0x2E55, prOP, gcPs}, // LEFT SQUARE BRACKET WITH STROKE + {0x2E56, 0x2E56, prCL, gcPe}, // RIGHT SQUARE BRACKET WITH STROKE + {0x2E57, 0x2E57, prOP, gcPs}, // LEFT SQUARE BRACKET WITH DOUBLE STROKE + {0x2E58, 0x2E58, prCL, gcPe}, // RIGHT SQUARE BRACKET WITH DOUBLE STROKE + {0x2E59, 0x2E59, prOP, gcPs}, // TOP HALF LEFT PARENTHESIS + {0x2E5A, 0x2E5A, prCL, gcPe}, // TOP HALF RIGHT PARENTHESIS + {0x2E5B, 0x2E5B, prOP, gcPs}, // BOTTOM HALF LEFT PARENTHESIS + {0x2E5C, 0x2E5C, prCL, gcPe}, // BOTTOM HALF RIGHT PARENTHESIS + {0x2E5D, 0x2E5D, prBA, gcPd}, // OBLIQUE HYPHEN + {0x2E80, 0x2E99, prID, gcSo}, // [26] CJK RADICAL REPEAT..CJK RADICAL RAP + {0x2E9B, 0x2EF3, prID, gcSo}, // [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE + {0x2F00, 0x2FD5, prID, gcSo}, // [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE + {0x2FF0, 0x2FFB, prID, gcSo}, // [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID + {0x3000, 0x3000, prBA, gcZs}, // IDEOGRAPHIC SPACE + {0x3001, 0x3002, prCL, gcPo}, // [2] IDEOGRAPHIC COMMA..IDEOGRAPHIC FULL STOP + {0x3003, 0x3003, prID, gcPo}, // DITTO MARK + {0x3004, 0x3004, prID, gcSo}, // JAPANESE INDUSTRIAL STANDARD SYMBOL + {0x3005, 0x3005, prNS, gcLm}, // IDEOGRAPHIC ITERATION MARK + {0x3006, 0x3006, prID, gcLo}, // IDEOGRAPHIC CLOSING MARK + {0x3007, 0x3007, prID, gcNl}, // IDEOGRAPHIC NUMBER ZERO + {0x3008, 0x3008, prOP, gcPs}, // LEFT ANGLE BRACKET + {0x3009, 0x3009, prCL, gcPe}, // RIGHT ANGLE BRACKET + {0x300A, 0x300A, prOP, gcPs}, // LEFT DOUBLE ANGLE BRACKET + {0x300B, 0x300B, prCL, gcPe}, // RIGHT DOUBLE ANGLE BRACKET + {0x300C, 0x300C, prOP, gcPs}, // LEFT CORNER BRACKET + {0x300D, 0x300D, prCL, gcPe}, // RIGHT CORNER BRACKET + {0x300E, 0x300E, prOP, gcPs}, // LEFT WHITE CORNER BRACKET + {0x300F, 0x300F, prCL, gcPe}, // RIGHT WHITE CORNER BRACKET + {0x3010, 0x3010, prOP, gcPs}, // LEFT BLACK LENTICULAR BRACKET + {0x3011, 0x3011, prCL, gcPe}, // RIGHT BLACK LENTICULAR BRACKET + {0x3012, 0x3013, prID, gcSo}, // [2] POSTAL MARK..GETA MARK + {0x3014, 0x3014, prOP, gcPs}, // LEFT TORTOISE SHELL BRACKET + {0x3015, 0x3015, prCL, gcPe}, // RIGHT TORTOISE SHELL BRACKET + {0x3016, 0x3016, prOP, gcPs}, // LEFT WHITE LENTICULAR BRACKET + {0x3017, 0x3017, prCL, gcPe}, // RIGHT WHITE LENTICULAR BRACKET + {0x3018, 0x3018, prOP, gcPs}, // LEFT WHITE TORTOISE SHELL BRACKET + {0x3019, 0x3019, prCL, gcPe}, // RIGHT WHITE TORTOISE SHELL BRACKET + {0x301A, 0x301A, prOP, gcPs}, // LEFT WHITE SQUARE BRACKET + {0x301B, 0x301B, prCL, gcPe}, // RIGHT WHITE SQUARE BRACKET + {0x301C, 0x301C, prNS, gcPd}, // WAVE DASH + {0x301D, 0x301D, prOP, gcPs}, // REVERSED DOUBLE PRIME QUOTATION MARK + {0x301E, 0x301F, prCL, gcPe}, // [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK + {0x3020, 0x3020, prID, gcSo}, // POSTAL MARK FACE + {0x3021, 0x3029, prID, gcNl}, // [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE + {0x302A, 0x302D, prCM, gcMn}, // [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK + {0x302E, 0x302F, prCM, gcMc}, // [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK + {0x3030, 0x3030, prID, gcPd}, // WAVY DASH + {0x3031, 0x3034, prID, gcLm}, // [4] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HALF + {0x3035, 0x3035, prCM, gcLm}, // VERTICAL KANA REPEAT MARK LOWER HALF + {0x3036, 0x3037, prID, gcSo}, // [2] CIRCLED POSTAL MARK..IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL + {0x3038, 0x303A, prID, gcNl}, // [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY + {0x303B, 0x303B, prNS, gcLm}, // VERTICAL IDEOGRAPHIC ITERATION MARK + {0x303C, 0x303C, prNS, gcLo}, // MASU MARK + {0x303D, 0x303D, prID, gcPo}, // PART ALTERNATION MARK + {0x303E, 0x303F, prID, gcSo}, // [2] IDEOGRAPHIC VARIATION INDICATOR..IDEOGRAPHIC HALF FILL SPACE + {0x3041, 0x3041, prCJ, gcLo}, // HIRAGANA LETTER SMALL A + {0x3042, 0x3042, prID, gcLo}, // HIRAGANA LETTER A + {0x3043, 0x3043, prCJ, gcLo}, // HIRAGANA LETTER SMALL I + {0x3044, 0x3044, prID, gcLo}, // HIRAGANA LETTER I + {0x3045, 0x3045, prCJ, gcLo}, // HIRAGANA LETTER SMALL U + {0x3046, 0x3046, prID, gcLo}, // HIRAGANA LETTER U + {0x3047, 0x3047, prCJ, gcLo}, // HIRAGANA LETTER SMALL E + {0x3048, 0x3048, prID, gcLo}, // HIRAGANA LETTER E + {0x3049, 0x3049, prCJ, gcLo}, // HIRAGANA LETTER SMALL O + {0x304A, 0x3062, prID, gcLo}, // [25] HIRAGANA LETTER O..HIRAGANA LETTER DI + {0x3063, 0x3063, prCJ, gcLo}, // HIRAGANA LETTER SMALL TU + {0x3064, 0x3082, prID, gcLo}, // [31] HIRAGANA LETTER TU..HIRAGANA LETTER MO + {0x3083, 0x3083, prCJ, gcLo}, // HIRAGANA LETTER SMALL YA + {0x3084, 0x3084, prID, gcLo}, // HIRAGANA LETTER YA + {0x3085, 0x3085, prCJ, gcLo}, // HIRAGANA LETTER SMALL YU + {0x3086, 0x3086, prID, gcLo}, // HIRAGANA LETTER YU + {0x3087, 0x3087, prCJ, gcLo}, // HIRAGANA LETTER SMALL YO + {0x3088, 0x308D, prID, gcLo}, // [6] HIRAGANA LETTER YO..HIRAGANA LETTER RO + {0x308E, 0x308E, prCJ, gcLo}, // HIRAGANA LETTER SMALL WA + {0x308F, 0x3094, prID, gcLo}, // [6] HIRAGANA LETTER WA..HIRAGANA LETTER VU + {0x3095, 0x3096, prCJ, gcLo}, // [2] HIRAGANA LETTER SMALL KA..HIRAGANA LETTER SMALL KE + {0x3099, 0x309A, prCM, gcMn}, // [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x309B, 0x309C, prNS, gcSk}, // [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x309D, 0x309E, prNS, gcLm}, // [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK + {0x309F, 0x309F, prID, gcLo}, // HIRAGANA DIGRAPH YORI + {0x30A0, 0x30A0, prNS, gcPd}, // KATAKANA-HIRAGANA DOUBLE HYPHEN + {0x30A1, 0x30A1, prCJ, gcLo}, // KATAKANA LETTER SMALL A + {0x30A2, 0x30A2, prID, gcLo}, // KATAKANA LETTER A + {0x30A3, 0x30A3, prCJ, gcLo}, // KATAKANA LETTER SMALL I + {0x30A4, 0x30A4, prID, gcLo}, // KATAKANA LETTER I + {0x30A5, 0x30A5, prCJ, gcLo}, // KATAKANA LETTER SMALL U + {0x30A6, 0x30A6, prID, gcLo}, // KATAKANA LETTER U + {0x30A7, 0x30A7, prCJ, gcLo}, // KATAKANA LETTER SMALL E + {0x30A8, 0x30A8, prID, gcLo}, // KATAKANA LETTER E + {0x30A9, 0x30A9, prCJ, gcLo}, // KATAKANA LETTER SMALL O + {0x30AA, 0x30C2, prID, gcLo}, // [25] KATAKANA LETTER O..KATAKANA LETTER DI + {0x30C3, 0x30C3, prCJ, gcLo}, // KATAKANA LETTER SMALL TU + {0x30C4, 0x30E2, prID, gcLo}, // [31] KATAKANA LETTER TU..KATAKANA LETTER MO + {0x30E3, 0x30E3, prCJ, gcLo}, // KATAKANA LETTER SMALL YA + {0x30E4, 0x30E4, prID, gcLo}, // KATAKANA LETTER YA + {0x30E5, 0x30E5, prCJ, gcLo}, // KATAKANA LETTER SMALL YU + {0x30E6, 0x30E6, prID, gcLo}, // KATAKANA LETTER YU + {0x30E7, 0x30E7, prCJ, gcLo}, // KATAKANA LETTER SMALL YO + {0x30E8, 0x30ED, prID, gcLo}, // [6] KATAKANA LETTER YO..KATAKANA LETTER RO + {0x30EE, 0x30EE, prCJ, gcLo}, // KATAKANA LETTER SMALL WA + {0x30EF, 0x30F4, prID, gcLo}, // [6] KATAKANA LETTER WA..KATAKANA LETTER VU + {0x30F5, 0x30F6, prCJ, gcLo}, // [2] KATAKANA LETTER SMALL KA..KATAKANA LETTER SMALL KE + {0x30F7, 0x30FA, prID, gcLo}, // [4] KATAKANA LETTER VA..KATAKANA LETTER VO + {0x30FB, 0x30FB, prNS, gcPo}, // KATAKANA MIDDLE DOT + {0x30FC, 0x30FC, prCJ, gcLm}, // KATAKANA-HIRAGANA PROLONGED SOUND MARK + {0x30FD, 0x30FE, prNS, gcLm}, // [2] KATAKANA ITERATION MARK..KATAKANA VOICED ITERATION MARK + {0x30FF, 0x30FF, prID, gcLo}, // KATAKANA DIGRAPH KOTO + {0x3105, 0x312F, prID, gcLo}, // [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN + {0x3131, 0x318E, prID, gcLo}, // [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE + {0x3190, 0x3191, prID, gcSo}, // [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK + {0x3192, 0x3195, prID, gcNo}, // [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK + {0x3196, 0x319F, prID, gcSo}, // [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK + {0x31A0, 0x31BF, prID, gcLo}, // [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH + {0x31C0, 0x31E3, prID, gcSo}, // [36] CJK STROKE T..CJK STROKE Q + {0x31F0, 0x31FF, prCJ, gcLo}, // [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO + {0x3200, 0x321E, prID, gcSo}, // [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU + {0x3220, 0x3229, prID, gcNo}, // [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN + {0x322A, 0x3247, prID, gcSo}, // [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO + {0x3248, 0x324F, prAI, gcNo}, // [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE + {0x3250, 0x3250, prID, gcSo}, // PARTNERSHIP SIGN + {0x3251, 0x325F, prID, gcNo}, // [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE + {0x3260, 0x327F, prID, gcSo}, // [32] CIRCLED HANGUL KIYEOK..KOREAN STANDARD SYMBOL + {0x3280, 0x3289, prID, gcNo}, // [10] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN + {0x328A, 0x32B0, prID, gcSo}, // [39] CIRCLED IDEOGRAPH MOON..CIRCLED IDEOGRAPH NIGHT + {0x32B1, 0x32BF, prID, gcNo}, // [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY + {0x32C0, 0x32FF, prID, gcSo}, // [64] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..SQUARE ERA NAME REIWA + {0x3300, 0x33FF, prID, gcSo}, // [256] SQUARE APAATO..SQUARE GAL + {0x3400, 0x4DBF, prID, gcLo}, // [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF + {0x4DC0, 0x4DFF, prAL, gcSo}, // [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION + {0x4E00, 0x9FFF, prID, gcLo}, // [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF + {0xA000, 0xA014, prID, gcLo}, // [21] YI SYLLABLE IT..YI SYLLABLE E + {0xA015, 0xA015, prNS, gcLm}, // YI SYLLABLE WU + {0xA016, 0xA48C, prID, gcLo}, // [1143] YI SYLLABLE BIT..YI SYLLABLE YYR + {0xA490, 0xA4C6, prID, gcSo}, // [55] YI RADICAL QOT..YI RADICAL KE + {0xA4D0, 0xA4F7, prAL, gcLo}, // [40] LISU LETTER BA..LISU LETTER OE + {0xA4F8, 0xA4FD, prAL, gcLm}, // [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU + {0xA4FE, 0xA4FF, prBA, gcPo}, // [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP + {0xA500, 0xA60B, prAL, gcLo}, // [268] VAI SYLLABLE EE..VAI SYLLABLE NG + {0xA60C, 0xA60C, prAL, gcLm}, // VAI SYLLABLE LENGTHENER + {0xA60D, 0xA60D, prBA, gcPo}, // VAI COMMA + {0xA60E, 0xA60E, prEX, gcPo}, // VAI FULL STOP + {0xA60F, 0xA60F, prBA, gcPo}, // VAI QUESTION MARK + {0xA610, 0xA61F, prAL, gcLo}, // [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG + {0xA620, 0xA629, prNU, gcNd}, // [10] VAI DIGIT ZERO..VAI DIGIT NINE + {0xA62A, 0xA62B, prAL, gcLo}, // [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO + {0xA640, 0xA66D, prAL, gcLC}, // [46] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O + {0xA66E, 0xA66E, prAL, gcLo}, // CYRILLIC LETTER MULTIOCULAR O + {0xA66F, 0xA66F, prCM, gcMn}, // COMBINING CYRILLIC VZMET + {0xA670, 0xA672, prCM, gcMe}, // [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN + {0xA673, 0xA673, prAL, gcPo}, // SLAVONIC ASTERISK + {0xA674, 0xA67D, prCM, gcMn}, // [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK + {0xA67E, 0xA67E, prAL, gcPo}, // CYRILLIC KAVYKA + {0xA67F, 0xA67F, prAL, gcLm}, // CYRILLIC PAYEROK + {0xA680, 0xA69B, prAL, gcLC}, // [28] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER CROSSED O + {0xA69C, 0xA69D, prAL, gcLm}, // [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN + {0xA69E, 0xA69F, prCM, gcMn}, // [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E + {0xA6A0, 0xA6E5, prAL, gcLo}, // [70] BAMUM LETTER A..BAMUM LETTER KI + {0xA6E6, 0xA6EF, prAL, gcNl}, // [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM + {0xA6F0, 0xA6F1, prCM, gcMn}, // [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS + {0xA6F2, 0xA6F2, prAL, gcPo}, // BAMUM NJAEMLI + {0xA6F3, 0xA6F7, prBA, gcPo}, // [5] BAMUM FULL STOP..BAMUM QUESTION MARK + {0xA700, 0xA716, prAL, gcSk}, // [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR + {0xA717, 0xA71F, prAL, gcLm}, // [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK + {0xA720, 0xA721, prAL, gcSk}, // [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE + {0xA722, 0xA76F, prAL, gcLC}, // [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON + {0xA770, 0xA770, prAL, gcLm}, // MODIFIER LETTER US + {0xA771, 0xA787, prAL, gcLC}, // [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T + {0xA788, 0xA788, prAL, gcLm}, // MODIFIER LETTER LOW CIRCUMFLEX ACCENT + {0xA789, 0xA78A, prAL, gcSk}, // [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN + {0xA78B, 0xA78E, prAL, gcLC}, // [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT + {0xA78F, 0xA78F, prAL, gcLo}, // LATIN LETTER SINOLOGICAL DOT + {0xA790, 0xA7CA, prAL, gcLC}, // [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY + {0xA7D0, 0xA7D1, prAL, gcLC}, // [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G + {0xA7D3, 0xA7D3, prAL, gcLl}, // LATIN SMALL LETTER DOUBLE THORN + {0xA7D5, 0xA7D9, prAL, gcLC}, // [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S + {0xA7F2, 0xA7F4, prAL, gcLm}, // [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q + {0xA7F5, 0xA7F6, prAL, gcLC}, // [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H + {0xA7F7, 0xA7F7, prAL, gcLo}, // LATIN EPIGRAPHIC LETTER SIDEWAYS I + {0xA7F8, 0xA7F9, prAL, gcLm}, // [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE + {0xA7FA, 0xA7FA, prAL, gcLl}, // LATIN LETTER SMALL CAPITAL TURNED M + {0xA7FB, 0xA7FF, prAL, gcLo}, // [5] LATIN EPIGRAPHIC LETTER REVERSED F..LATIN EPIGRAPHIC LETTER ARCHAIC M + {0xA800, 0xA801, prAL, gcLo}, // [2] SYLOTI NAGRI LETTER A..SYLOTI NAGRI LETTER I + {0xA802, 0xA802, prCM, gcMn}, // SYLOTI NAGRI SIGN DVISVARA + {0xA803, 0xA805, prAL, gcLo}, // [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O + {0xA806, 0xA806, prCM, gcMn}, // SYLOTI NAGRI SIGN HASANTA + {0xA807, 0xA80A, prAL, gcLo}, // [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO + {0xA80B, 0xA80B, prCM, gcMn}, // SYLOTI NAGRI SIGN ANUSVARA + {0xA80C, 0xA822, prAL, gcLo}, // [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO + {0xA823, 0xA824, prCM, gcMc}, // [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I + {0xA825, 0xA826, prCM, gcMn}, // [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E + {0xA827, 0xA827, prCM, gcMc}, // SYLOTI NAGRI VOWEL SIGN OO + {0xA828, 0xA82B, prAL, gcSo}, // [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 + {0xA82C, 0xA82C, prCM, gcMn}, // SYLOTI NAGRI SIGN ALTERNATE HASANTA + {0xA830, 0xA835, prAL, gcNo}, // [6] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS + {0xA836, 0xA837, prAL, gcSo}, // [2] NORTH INDIC QUARTER MARK..NORTH INDIC PLACEHOLDER MARK + {0xA838, 0xA838, prPO, gcSc}, // NORTH INDIC RUPEE MARK + {0xA839, 0xA839, prAL, gcSo}, // NORTH INDIC QUANTITY MARK + {0xA840, 0xA873, prAL, gcLo}, // [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU + {0xA874, 0xA875, prBB, gcPo}, // [2] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA DOUBLE HEAD MARK + {0xA876, 0xA877, prEX, gcPo}, // [2] PHAGS-PA MARK SHAD..PHAGS-PA MARK DOUBLE SHAD + {0xA880, 0xA881, prCM, gcMc}, // [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA + {0xA882, 0xA8B3, prAL, gcLo}, // [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA + {0xA8B4, 0xA8C3, prCM, gcMc}, // [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU + {0xA8C4, 0xA8C5, prCM, gcMn}, // [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU + {0xA8CE, 0xA8CF, prBA, gcPo}, // [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA + {0xA8D0, 0xA8D9, prNU, gcNd}, // [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE + {0xA8E0, 0xA8F1, prCM, gcMn}, // [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA + {0xA8F2, 0xA8F7, prAL, gcLo}, // [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA + {0xA8F8, 0xA8FA, prAL, gcPo}, // [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET + {0xA8FB, 0xA8FB, prAL, gcLo}, // DEVANAGARI HEADSTROKE + {0xA8FC, 0xA8FC, prBB, gcPo}, // DEVANAGARI SIGN SIDDHAM + {0xA8FD, 0xA8FE, prAL, gcLo}, // [2] DEVANAGARI JAIN OM..DEVANAGARI LETTER AY + {0xA8FF, 0xA8FF, prCM, gcMn}, // DEVANAGARI VOWEL SIGN AY + {0xA900, 0xA909, prNU, gcNd}, // [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE + {0xA90A, 0xA925, prAL, gcLo}, // [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO + {0xA926, 0xA92D, prCM, gcMn}, // [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU + {0xA92E, 0xA92F, prBA, gcPo}, // [2] KAYAH LI SIGN CWI..KAYAH LI SIGN SHYA + {0xA930, 0xA946, prAL, gcLo}, // [23] REJANG LETTER KA..REJANG LETTER A + {0xA947, 0xA951, prCM, gcMn}, // [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R + {0xA952, 0xA953, prCM, gcMc}, // [2] REJANG CONSONANT SIGN H..REJANG VIRAMA + {0xA95F, 0xA95F, prAL, gcPo}, // REJANG SECTION MARK + {0xA960, 0xA97C, prJL, gcLo}, // [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH + {0xA980, 0xA982, prCM, gcMn}, // [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR + {0xA983, 0xA983, prCM, gcMc}, // JAVANESE SIGN WIGNYAN + {0xA984, 0xA9B2, prAL, gcLo}, // [47] JAVANESE LETTER A..JAVANESE LETTER HA + {0xA9B3, 0xA9B3, prCM, gcMn}, // JAVANESE SIGN CECAK TELU + {0xA9B4, 0xA9B5, prCM, gcMc}, // [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG + {0xA9B6, 0xA9B9, prCM, gcMn}, // [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT + {0xA9BA, 0xA9BB, prCM, gcMc}, // [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE + {0xA9BC, 0xA9BD, prCM, gcMn}, // [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET + {0xA9BE, 0xA9C0, prCM, gcMc}, // [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON + {0xA9C1, 0xA9C6, prAL, gcPo}, // [6] JAVANESE LEFT RERENGGAN..JAVANESE PADA WINDU + {0xA9C7, 0xA9C9, prBA, gcPo}, // [3] JAVANESE PADA PANGKAT..JAVANESE PADA LUNGSI + {0xA9CA, 0xA9CD, prAL, gcPo}, // [4] JAVANESE PADA ADEG..JAVANESE TURNED PADA PISELEH + {0xA9CF, 0xA9CF, prAL, gcLm}, // JAVANESE PANGRANGKEP + {0xA9D0, 0xA9D9, prNU, gcNd}, // [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE + {0xA9DE, 0xA9DF, prAL, gcPo}, // [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN + {0xA9E0, 0xA9E4, prSA, gcLo}, // [5] MYANMAR LETTER SHAN GHA..MYANMAR LETTER SHAN BHA + {0xA9E5, 0xA9E5, prSA, gcMn}, // MYANMAR SIGN SHAN SAW + {0xA9E6, 0xA9E6, prSA, gcLm}, // MYANMAR MODIFIER LETTER SHAN REDUPLICATION + {0xA9E7, 0xA9EF, prSA, gcLo}, // [9] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING NNA + {0xA9F0, 0xA9F9, prNU, gcNd}, // [10] MYANMAR TAI LAING DIGIT ZERO..MYANMAR TAI LAING DIGIT NINE + {0xA9FA, 0xA9FE, prSA, gcLo}, // [5] MYANMAR LETTER TAI LAING LLA..MYANMAR LETTER TAI LAING BHA + {0xAA00, 0xAA28, prAL, gcLo}, // [41] CHAM LETTER A..CHAM LETTER HA + {0xAA29, 0xAA2E, prCM, gcMn}, // [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE + {0xAA2F, 0xAA30, prCM, gcMc}, // [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI + {0xAA31, 0xAA32, prCM, gcMn}, // [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE + {0xAA33, 0xAA34, prCM, gcMc}, // [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA + {0xAA35, 0xAA36, prCM, gcMn}, // [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA + {0xAA40, 0xAA42, prAL, gcLo}, // [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG + {0xAA43, 0xAA43, prCM, gcMn}, // CHAM CONSONANT SIGN FINAL NG + {0xAA44, 0xAA4B, prAL, gcLo}, // [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS + {0xAA4C, 0xAA4C, prCM, gcMn}, // CHAM CONSONANT SIGN FINAL M + {0xAA4D, 0xAA4D, prCM, gcMc}, // CHAM CONSONANT SIGN FINAL H + {0xAA50, 0xAA59, prNU, gcNd}, // [10] CHAM DIGIT ZERO..CHAM DIGIT NINE + {0xAA5C, 0xAA5C, prAL, gcPo}, // CHAM PUNCTUATION SPIRAL + {0xAA5D, 0xAA5F, prBA, gcPo}, // [3] CHAM PUNCTUATION DANDA..CHAM PUNCTUATION TRIPLE DANDA + {0xAA60, 0xAA6F, prSA, gcLo}, // [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA + {0xAA70, 0xAA70, prSA, gcLm}, // MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION + {0xAA71, 0xAA76, prSA, gcLo}, // [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM + {0xAA77, 0xAA79, prSA, gcSo}, // [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO + {0xAA7A, 0xAA7A, prSA, gcLo}, // MYANMAR LETTER AITON RA + {0xAA7B, 0xAA7B, prSA, gcMc}, // MYANMAR SIGN PAO KAREN TONE + {0xAA7C, 0xAA7C, prSA, gcMn}, // MYANMAR SIGN TAI LAING TONE-2 + {0xAA7D, 0xAA7D, prSA, gcMc}, // MYANMAR SIGN TAI LAING TONE-5 + {0xAA7E, 0xAA7F, prSA, gcLo}, // [2] MYANMAR LETTER SHWE PALAUNG CHA..MYANMAR LETTER SHWE PALAUNG SHA + {0xAA80, 0xAAAF, prSA, gcLo}, // [48] TAI VIET LETTER LOW KO..TAI VIET LETTER HIGH O + {0xAAB0, 0xAAB0, prSA, gcMn}, // TAI VIET MAI KANG + {0xAAB1, 0xAAB1, prSA, gcLo}, // TAI VIET VOWEL AA + {0xAAB2, 0xAAB4, prSA, gcMn}, // [3] TAI VIET VOWEL I..TAI VIET VOWEL U + {0xAAB5, 0xAAB6, prSA, gcLo}, // [2] TAI VIET VOWEL E..TAI VIET VOWEL O + {0xAAB7, 0xAAB8, prSA, gcMn}, // [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA + {0xAAB9, 0xAABD, prSA, gcLo}, // [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN + {0xAABE, 0xAABF, prSA, gcMn}, // [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK + {0xAAC0, 0xAAC0, prSA, gcLo}, // TAI VIET TONE MAI NUENG + {0xAAC1, 0xAAC1, prSA, gcMn}, // TAI VIET TONE MAI THO + {0xAAC2, 0xAAC2, prSA, gcLo}, // TAI VIET TONE MAI SONG + {0xAADB, 0xAADC, prSA, gcLo}, // [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG + {0xAADD, 0xAADD, prSA, gcLm}, // TAI VIET SYMBOL SAM + {0xAADE, 0xAADF, prSA, gcPo}, // [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI + {0xAAE0, 0xAAEA, prAL, gcLo}, // [11] MEETEI MAYEK LETTER E..MEETEI MAYEK LETTER SSA + {0xAAEB, 0xAAEB, prCM, gcMc}, // MEETEI MAYEK VOWEL SIGN II + {0xAAEC, 0xAAED, prCM, gcMn}, // [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI + {0xAAEE, 0xAAEF, prCM, gcMc}, // [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU + {0xAAF0, 0xAAF1, prBA, gcPo}, // [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM + {0xAAF2, 0xAAF2, prAL, gcLo}, // MEETEI MAYEK ANJI + {0xAAF3, 0xAAF4, prAL, gcLm}, // [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK + {0xAAF5, 0xAAF5, prCM, gcMc}, // MEETEI MAYEK VOWEL SIGN VISARGA + {0xAAF6, 0xAAF6, prCM, gcMn}, // MEETEI MAYEK VIRAMA + {0xAB01, 0xAB06, prAL, gcLo}, // [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO + {0xAB09, 0xAB0E, prAL, gcLo}, // [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO + {0xAB11, 0xAB16, prAL, gcLo}, // [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO + {0xAB20, 0xAB26, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO + {0xAB28, 0xAB2E, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO + {0xAB30, 0xAB5A, prAL, gcLl}, // [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG + {0xAB5B, 0xAB5B, prAL, gcSk}, // MODIFIER BREVE WITH INVERTED BREVE + {0xAB5C, 0xAB5F, prAL, gcLm}, // [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK + {0xAB60, 0xAB68, prAL, gcLl}, // [9] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE + {0xAB69, 0xAB69, prAL, gcLm}, // MODIFIER LETTER SMALL TURNED W + {0xAB6A, 0xAB6B, prAL, gcSk}, // [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK + {0xAB70, 0xABBF, prAL, gcLl}, // [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA + {0xABC0, 0xABE2, prAL, gcLo}, // [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM + {0xABE3, 0xABE4, prCM, gcMc}, // [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP + {0xABE5, 0xABE5, prCM, gcMn}, // MEETEI MAYEK VOWEL SIGN ANAP + {0xABE6, 0xABE7, prCM, gcMc}, // [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP + {0xABE8, 0xABE8, prCM, gcMn}, // MEETEI MAYEK VOWEL SIGN UNAP + {0xABE9, 0xABEA, prCM, gcMc}, // [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG + {0xABEB, 0xABEB, prBA, gcPo}, // MEETEI MAYEK CHEIKHEI + {0xABEC, 0xABEC, prCM, gcMc}, // MEETEI MAYEK LUM IYEK + {0xABED, 0xABED, prCM, gcMn}, // MEETEI MAYEK APUN IYEK + {0xABF0, 0xABF9, prNU, gcNd}, // [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE + {0xAC00, 0xAC00, prH2, gcLo}, // HANGUL SYLLABLE GA + {0xAC01, 0xAC1B, prH3, gcLo}, // [27] HANGUL SYLLABLE GAG..HANGUL SYLLABLE GAH + {0xAC1C, 0xAC1C, prH2, gcLo}, // HANGUL SYLLABLE GAE + {0xAC1D, 0xAC37, prH3, gcLo}, // [27] HANGUL SYLLABLE GAEG..HANGUL SYLLABLE GAEH + {0xAC38, 0xAC38, prH2, gcLo}, // HANGUL SYLLABLE GYA + {0xAC39, 0xAC53, prH3, gcLo}, // [27] HANGUL SYLLABLE GYAG..HANGUL SYLLABLE GYAH + {0xAC54, 0xAC54, prH2, gcLo}, // HANGUL SYLLABLE GYAE + {0xAC55, 0xAC6F, prH3, gcLo}, // [27] HANGUL SYLLABLE GYAEG..HANGUL SYLLABLE GYAEH + {0xAC70, 0xAC70, prH2, gcLo}, // HANGUL SYLLABLE GEO + {0xAC71, 0xAC8B, prH3, gcLo}, // [27] HANGUL SYLLABLE GEOG..HANGUL SYLLABLE GEOH + {0xAC8C, 0xAC8C, prH2, gcLo}, // HANGUL SYLLABLE GE + {0xAC8D, 0xACA7, prH3, gcLo}, // [27] HANGUL SYLLABLE GEG..HANGUL SYLLABLE GEH + {0xACA8, 0xACA8, prH2, gcLo}, // HANGUL SYLLABLE GYEO + {0xACA9, 0xACC3, prH3, gcLo}, // [27] HANGUL SYLLABLE GYEOG..HANGUL SYLLABLE GYEOH + {0xACC4, 0xACC4, prH2, gcLo}, // HANGUL SYLLABLE GYE + {0xACC5, 0xACDF, prH3, gcLo}, // [27] HANGUL SYLLABLE GYEG..HANGUL SYLLABLE GYEH + {0xACE0, 0xACE0, prH2, gcLo}, // HANGUL SYLLABLE GO + {0xACE1, 0xACFB, prH3, gcLo}, // [27] HANGUL SYLLABLE GOG..HANGUL SYLLABLE GOH + {0xACFC, 0xACFC, prH2, gcLo}, // HANGUL SYLLABLE GWA + {0xACFD, 0xAD17, prH3, gcLo}, // [27] HANGUL SYLLABLE GWAG..HANGUL SYLLABLE GWAH + {0xAD18, 0xAD18, prH2, gcLo}, // HANGUL SYLLABLE GWAE + {0xAD19, 0xAD33, prH3, gcLo}, // [27] HANGUL SYLLABLE GWAEG..HANGUL SYLLABLE GWAEH + {0xAD34, 0xAD34, prH2, gcLo}, // HANGUL SYLLABLE GOE + {0xAD35, 0xAD4F, prH3, gcLo}, // [27] HANGUL SYLLABLE GOEG..HANGUL SYLLABLE GOEH + {0xAD50, 0xAD50, prH2, gcLo}, // HANGUL SYLLABLE GYO + {0xAD51, 0xAD6B, prH3, gcLo}, // [27] HANGUL SYLLABLE GYOG..HANGUL SYLLABLE GYOH + {0xAD6C, 0xAD6C, prH2, gcLo}, // HANGUL SYLLABLE GU + {0xAD6D, 0xAD87, prH3, gcLo}, // [27] HANGUL SYLLABLE GUG..HANGUL SYLLABLE GUH + {0xAD88, 0xAD88, prH2, gcLo}, // HANGUL SYLLABLE GWEO + {0xAD89, 0xADA3, prH3, gcLo}, // [27] HANGUL SYLLABLE GWEOG..HANGUL SYLLABLE GWEOH + {0xADA4, 0xADA4, prH2, gcLo}, // HANGUL SYLLABLE GWE + {0xADA5, 0xADBF, prH3, gcLo}, // [27] HANGUL SYLLABLE GWEG..HANGUL SYLLABLE GWEH + {0xADC0, 0xADC0, prH2, gcLo}, // HANGUL SYLLABLE GWI + {0xADC1, 0xADDB, prH3, gcLo}, // [27] HANGUL SYLLABLE GWIG..HANGUL SYLLABLE GWIH + {0xADDC, 0xADDC, prH2, gcLo}, // HANGUL SYLLABLE GYU + {0xADDD, 0xADF7, prH3, gcLo}, // [27] HANGUL SYLLABLE GYUG..HANGUL SYLLABLE GYUH + {0xADF8, 0xADF8, prH2, gcLo}, // HANGUL SYLLABLE GEU + {0xADF9, 0xAE13, prH3, gcLo}, // [27] HANGUL SYLLABLE GEUG..HANGUL SYLLABLE GEUH + {0xAE14, 0xAE14, prH2, gcLo}, // HANGUL SYLLABLE GYI + {0xAE15, 0xAE2F, prH3, gcLo}, // [27] HANGUL SYLLABLE GYIG..HANGUL SYLLABLE GYIH + {0xAE30, 0xAE30, prH2, gcLo}, // HANGUL SYLLABLE GI + {0xAE31, 0xAE4B, prH3, gcLo}, // [27] HANGUL SYLLABLE GIG..HANGUL SYLLABLE GIH + {0xAE4C, 0xAE4C, prH2, gcLo}, // HANGUL SYLLABLE GGA + {0xAE4D, 0xAE67, prH3, gcLo}, // [27] HANGUL SYLLABLE GGAG..HANGUL SYLLABLE GGAH + {0xAE68, 0xAE68, prH2, gcLo}, // HANGUL SYLLABLE GGAE + {0xAE69, 0xAE83, prH3, gcLo}, // [27] HANGUL SYLLABLE GGAEG..HANGUL SYLLABLE GGAEH + {0xAE84, 0xAE84, prH2, gcLo}, // HANGUL SYLLABLE GGYA + {0xAE85, 0xAE9F, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYAG..HANGUL SYLLABLE GGYAH + {0xAEA0, 0xAEA0, prH2, gcLo}, // HANGUL SYLLABLE GGYAE + {0xAEA1, 0xAEBB, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYAEG..HANGUL SYLLABLE GGYAEH + {0xAEBC, 0xAEBC, prH2, gcLo}, // HANGUL SYLLABLE GGEO + {0xAEBD, 0xAED7, prH3, gcLo}, // [27] HANGUL SYLLABLE GGEOG..HANGUL SYLLABLE GGEOH + {0xAED8, 0xAED8, prH2, gcLo}, // HANGUL SYLLABLE GGE + {0xAED9, 0xAEF3, prH3, gcLo}, // [27] HANGUL SYLLABLE GGEG..HANGUL SYLLABLE GGEH + {0xAEF4, 0xAEF4, prH2, gcLo}, // HANGUL SYLLABLE GGYEO + {0xAEF5, 0xAF0F, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYEOG..HANGUL SYLLABLE GGYEOH + {0xAF10, 0xAF10, prH2, gcLo}, // HANGUL SYLLABLE GGYE + {0xAF11, 0xAF2B, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYEG..HANGUL SYLLABLE GGYEH + {0xAF2C, 0xAF2C, prH2, gcLo}, // HANGUL SYLLABLE GGO + {0xAF2D, 0xAF47, prH3, gcLo}, // [27] HANGUL SYLLABLE GGOG..HANGUL SYLLABLE GGOH + {0xAF48, 0xAF48, prH2, gcLo}, // HANGUL SYLLABLE GGWA + {0xAF49, 0xAF63, prH3, gcLo}, // [27] HANGUL SYLLABLE GGWAG..HANGUL SYLLABLE GGWAH + {0xAF64, 0xAF64, prH2, gcLo}, // HANGUL SYLLABLE GGWAE + {0xAF65, 0xAF7F, prH3, gcLo}, // [27] HANGUL SYLLABLE GGWAEG..HANGUL SYLLABLE GGWAEH + {0xAF80, 0xAF80, prH2, gcLo}, // HANGUL SYLLABLE GGOE + {0xAF81, 0xAF9B, prH3, gcLo}, // [27] HANGUL SYLLABLE GGOEG..HANGUL SYLLABLE GGOEH + {0xAF9C, 0xAF9C, prH2, gcLo}, // HANGUL SYLLABLE GGYO + {0xAF9D, 0xAFB7, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYOG..HANGUL SYLLABLE GGYOH + {0xAFB8, 0xAFB8, prH2, gcLo}, // HANGUL SYLLABLE GGU + {0xAFB9, 0xAFD3, prH3, gcLo}, // [27] HANGUL SYLLABLE GGUG..HANGUL SYLLABLE GGUH + {0xAFD4, 0xAFD4, prH2, gcLo}, // HANGUL SYLLABLE GGWEO + {0xAFD5, 0xAFEF, prH3, gcLo}, // [27] HANGUL SYLLABLE GGWEOG..HANGUL SYLLABLE GGWEOH + {0xAFF0, 0xAFF0, prH2, gcLo}, // HANGUL SYLLABLE GGWE + {0xAFF1, 0xB00B, prH3, gcLo}, // [27] HANGUL SYLLABLE GGWEG..HANGUL SYLLABLE GGWEH + {0xB00C, 0xB00C, prH2, gcLo}, // HANGUL SYLLABLE GGWI + {0xB00D, 0xB027, prH3, gcLo}, // [27] HANGUL SYLLABLE GGWIG..HANGUL SYLLABLE GGWIH + {0xB028, 0xB028, prH2, gcLo}, // HANGUL SYLLABLE GGYU + {0xB029, 0xB043, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYUG..HANGUL SYLLABLE GGYUH + {0xB044, 0xB044, prH2, gcLo}, // HANGUL SYLLABLE GGEU + {0xB045, 0xB05F, prH3, gcLo}, // [27] HANGUL SYLLABLE GGEUG..HANGUL SYLLABLE GGEUH + {0xB060, 0xB060, prH2, gcLo}, // HANGUL SYLLABLE GGYI + {0xB061, 0xB07B, prH3, gcLo}, // [27] HANGUL SYLLABLE GGYIG..HANGUL SYLLABLE GGYIH + {0xB07C, 0xB07C, prH2, gcLo}, // HANGUL SYLLABLE GGI + {0xB07D, 0xB097, prH3, gcLo}, // [27] HANGUL SYLLABLE GGIG..HANGUL SYLLABLE GGIH + {0xB098, 0xB098, prH2, gcLo}, // HANGUL SYLLABLE NA + {0xB099, 0xB0B3, prH3, gcLo}, // [27] HANGUL SYLLABLE NAG..HANGUL SYLLABLE NAH + {0xB0B4, 0xB0B4, prH2, gcLo}, // HANGUL SYLLABLE NAE + {0xB0B5, 0xB0CF, prH3, gcLo}, // [27] HANGUL SYLLABLE NAEG..HANGUL SYLLABLE NAEH + {0xB0D0, 0xB0D0, prH2, gcLo}, // HANGUL SYLLABLE NYA + {0xB0D1, 0xB0EB, prH3, gcLo}, // [27] HANGUL SYLLABLE NYAG..HANGUL SYLLABLE NYAH + {0xB0EC, 0xB0EC, prH2, gcLo}, // HANGUL SYLLABLE NYAE + {0xB0ED, 0xB107, prH3, gcLo}, // [27] HANGUL SYLLABLE NYAEG..HANGUL SYLLABLE NYAEH + {0xB108, 0xB108, prH2, gcLo}, // HANGUL SYLLABLE NEO + {0xB109, 0xB123, prH3, gcLo}, // [27] HANGUL SYLLABLE NEOG..HANGUL SYLLABLE NEOH + {0xB124, 0xB124, prH2, gcLo}, // HANGUL SYLLABLE NE + {0xB125, 0xB13F, prH3, gcLo}, // [27] HANGUL SYLLABLE NEG..HANGUL SYLLABLE NEH + {0xB140, 0xB140, prH2, gcLo}, // HANGUL SYLLABLE NYEO + {0xB141, 0xB15B, prH3, gcLo}, // [27] HANGUL SYLLABLE NYEOG..HANGUL SYLLABLE NYEOH + {0xB15C, 0xB15C, prH2, gcLo}, // HANGUL SYLLABLE NYE + {0xB15D, 0xB177, prH3, gcLo}, // [27] HANGUL SYLLABLE NYEG..HANGUL SYLLABLE NYEH + {0xB178, 0xB178, prH2, gcLo}, // HANGUL SYLLABLE NO + {0xB179, 0xB193, prH3, gcLo}, // [27] HANGUL SYLLABLE NOG..HANGUL SYLLABLE NOH + {0xB194, 0xB194, prH2, gcLo}, // HANGUL SYLLABLE NWA + {0xB195, 0xB1AF, prH3, gcLo}, // [27] HANGUL SYLLABLE NWAG..HANGUL SYLLABLE NWAH + {0xB1B0, 0xB1B0, prH2, gcLo}, // HANGUL SYLLABLE NWAE + {0xB1B1, 0xB1CB, prH3, gcLo}, // [27] HANGUL SYLLABLE NWAEG..HANGUL SYLLABLE NWAEH + {0xB1CC, 0xB1CC, prH2, gcLo}, // HANGUL SYLLABLE NOE + {0xB1CD, 0xB1E7, prH3, gcLo}, // [27] HANGUL SYLLABLE NOEG..HANGUL SYLLABLE NOEH + {0xB1E8, 0xB1E8, prH2, gcLo}, // HANGUL SYLLABLE NYO + {0xB1E9, 0xB203, prH3, gcLo}, // [27] HANGUL SYLLABLE NYOG..HANGUL SYLLABLE NYOH + {0xB204, 0xB204, prH2, gcLo}, // HANGUL SYLLABLE NU + {0xB205, 0xB21F, prH3, gcLo}, // [27] HANGUL SYLLABLE NUG..HANGUL SYLLABLE NUH + {0xB220, 0xB220, prH2, gcLo}, // HANGUL SYLLABLE NWEO + {0xB221, 0xB23B, prH3, gcLo}, // [27] HANGUL SYLLABLE NWEOG..HANGUL SYLLABLE NWEOH + {0xB23C, 0xB23C, prH2, gcLo}, // HANGUL SYLLABLE NWE + {0xB23D, 0xB257, prH3, gcLo}, // [27] HANGUL SYLLABLE NWEG..HANGUL SYLLABLE NWEH + {0xB258, 0xB258, prH2, gcLo}, // HANGUL SYLLABLE NWI + {0xB259, 0xB273, prH3, gcLo}, // [27] HANGUL SYLLABLE NWIG..HANGUL SYLLABLE NWIH + {0xB274, 0xB274, prH2, gcLo}, // HANGUL SYLLABLE NYU + {0xB275, 0xB28F, prH3, gcLo}, // [27] HANGUL SYLLABLE NYUG..HANGUL SYLLABLE NYUH + {0xB290, 0xB290, prH2, gcLo}, // HANGUL SYLLABLE NEU + {0xB291, 0xB2AB, prH3, gcLo}, // [27] HANGUL SYLLABLE NEUG..HANGUL SYLLABLE NEUH + {0xB2AC, 0xB2AC, prH2, gcLo}, // HANGUL SYLLABLE NYI + {0xB2AD, 0xB2C7, prH3, gcLo}, // [27] HANGUL SYLLABLE NYIG..HANGUL SYLLABLE NYIH + {0xB2C8, 0xB2C8, prH2, gcLo}, // HANGUL SYLLABLE NI + {0xB2C9, 0xB2E3, prH3, gcLo}, // [27] HANGUL SYLLABLE NIG..HANGUL SYLLABLE NIH + {0xB2E4, 0xB2E4, prH2, gcLo}, // HANGUL SYLLABLE DA + {0xB2E5, 0xB2FF, prH3, gcLo}, // [27] HANGUL SYLLABLE DAG..HANGUL SYLLABLE DAH + {0xB300, 0xB300, prH2, gcLo}, // HANGUL SYLLABLE DAE + {0xB301, 0xB31B, prH3, gcLo}, // [27] HANGUL SYLLABLE DAEG..HANGUL SYLLABLE DAEH + {0xB31C, 0xB31C, prH2, gcLo}, // HANGUL SYLLABLE DYA + {0xB31D, 0xB337, prH3, gcLo}, // [27] HANGUL SYLLABLE DYAG..HANGUL SYLLABLE DYAH + {0xB338, 0xB338, prH2, gcLo}, // HANGUL SYLLABLE DYAE + {0xB339, 0xB353, prH3, gcLo}, // [27] HANGUL SYLLABLE DYAEG..HANGUL SYLLABLE DYAEH + {0xB354, 0xB354, prH2, gcLo}, // HANGUL SYLLABLE DEO + {0xB355, 0xB36F, prH3, gcLo}, // [27] HANGUL SYLLABLE DEOG..HANGUL SYLLABLE DEOH + {0xB370, 0xB370, prH2, gcLo}, // HANGUL SYLLABLE DE + {0xB371, 0xB38B, prH3, gcLo}, // [27] HANGUL SYLLABLE DEG..HANGUL SYLLABLE DEH + {0xB38C, 0xB38C, prH2, gcLo}, // HANGUL SYLLABLE DYEO + {0xB38D, 0xB3A7, prH3, gcLo}, // [27] HANGUL SYLLABLE DYEOG..HANGUL SYLLABLE DYEOH + {0xB3A8, 0xB3A8, prH2, gcLo}, // HANGUL SYLLABLE DYE + {0xB3A9, 0xB3C3, prH3, gcLo}, // [27] HANGUL SYLLABLE DYEG..HANGUL SYLLABLE DYEH + {0xB3C4, 0xB3C4, prH2, gcLo}, // HANGUL SYLLABLE DO + {0xB3C5, 0xB3DF, prH3, gcLo}, // [27] HANGUL SYLLABLE DOG..HANGUL SYLLABLE DOH + {0xB3E0, 0xB3E0, prH2, gcLo}, // HANGUL SYLLABLE DWA + {0xB3E1, 0xB3FB, prH3, gcLo}, // [27] HANGUL SYLLABLE DWAG..HANGUL SYLLABLE DWAH + {0xB3FC, 0xB3FC, prH2, gcLo}, // HANGUL SYLLABLE DWAE + {0xB3FD, 0xB417, prH3, gcLo}, // [27] HANGUL SYLLABLE DWAEG..HANGUL SYLLABLE DWAEH + {0xB418, 0xB418, prH2, gcLo}, // HANGUL SYLLABLE DOE + {0xB419, 0xB433, prH3, gcLo}, // [27] HANGUL SYLLABLE DOEG..HANGUL SYLLABLE DOEH + {0xB434, 0xB434, prH2, gcLo}, // HANGUL SYLLABLE DYO + {0xB435, 0xB44F, prH3, gcLo}, // [27] HANGUL SYLLABLE DYOG..HANGUL SYLLABLE DYOH + {0xB450, 0xB450, prH2, gcLo}, // HANGUL SYLLABLE DU + {0xB451, 0xB46B, prH3, gcLo}, // [27] HANGUL SYLLABLE DUG..HANGUL SYLLABLE DUH + {0xB46C, 0xB46C, prH2, gcLo}, // HANGUL SYLLABLE DWEO + {0xB46D, 0xB487, prH3, gcLo}, // [27] HANGUL SYLLABLE DWEOG..HANGUL SYLLABLE DWEOH + {0xB488, 0xB488, prH2, gcLo}, // HANGUL SYLLABLE DWE + {0xB489, 0xB4A3, prH3, gcLo}, // [27] HANGUL SYLLABLE DWEG..HANGUL SYLLABLE DWEH + {0xB4A4, 0xB4A4, prH2, gcLo}, // HANGUL SYLLABLE DWI + {0xB4A5, 0xB4BF, prH3, gcLo}, // [27] HANGUL SYLLABLE DWIG..HANGUL SYLLABLE DWIH + {0xB4C0, 0xB4C0, prH2, gcLo}, // HANGUL SYLLABLE DYU + {0xB4C1, 0xB4DB, prH3, gcLo}, // [27] HANGUL SYLLABLE DYUG..HANGUL SYLLABLE DYUH + {0xB4DC, 0xB4DC, prH2, gcLo}, // HANGUL SYLLABLE DEU + {0xB4DD, 0xB4F7, prH3, gcLo}, // [27] HANGUL SYLLABLE DEUG..HANGUL SYLLABLE DEUH + {0xB4F8, 0xB4F8, prH2, gcLo}, // HANGUL SYLLABLE DYI + {0xB4F9, 0xB513, prH3, gcLo}, // [27] HANGUL SYLLABLE DYIG..HANGUL SYLLABLE DYIH + {0xB514, 0xB514, prH2, gcLo}, // HANGUL SYLLABLE DI + {0xB515, 0xB52F, prH3, gcLo}, // [27] HANGUL SYLLABLE DIG..HANGUL SYLLABLE DIH + {0xB530, 0xB530, prH2, gcLo}, // HANGUL SYLLABLE DDA + {0xB531, 0xB54B, prH3, gcLo}, // [27] HANGUL SYLLABLE DDAG..HANGUL SYLLABLE DDAH + {0xB54C, 0xB54C, prH2, gcLo}, // HANGUL SYLLABLE DDAE + {0xB54D, 0xB567, prH3, gcLo}, // [27] HANGUL SYLLABLE DDAEG..HANGUL SYLLABLE DDAEH + {0xB568, 0xB568, prH2, gcLo}, // HANGUL SYLLABLE DDYA + {0xB569, 0xB583, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYAG..HANGUL SYLLABLE DDYAH + {0xB584, 0xB584, prH2, gcLo}, // HANGUL SYLLABLE DDYAE + {0xB585, 0xB59F, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYAEG..HANGUL SYLLABLE DDYAEH + {0xB5A0, 0xB5A0, prH2, gcLo}, // HANGUL SYLLABLE DDEO + {0xB5A1, 0xB5BB, prH3, gcLo}, // [27] HANGUL SYLLABLE DDEOG..HANGUL SYLLABLE DDEOH + {0xB5BC, 0xB5BC, prH2, gcLo}, // HANGUL SYLLABLE DDE + {0xB5BD, 0xB5D7, prH3, gcLo}, // [27] HANGUL SYLLABLE DDEG..HANGUL SYLLABLE DDEH + {0xB5D8, 0xB5D8, prH2, gcLo}, // HANGUL SYLLABLE DDYEO + {0xB5D9, 0xB5F3, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYEOG..HANGUL SYLLABLE DDYEOH + {0xB5F4, 0xB5F4, prH2, gcLo}, // HANGUL SYLLABLE DDYE + {0xB5F5, 0xB60F, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYEG..HANGUL SYLLABLE DDYEH + {0xB610, 0xB610, prH2, gcLo}, // HANGUL SYLLABLE DDO + {0xB611, 0xB62B, prH3, gcLo}, // [27] HANGUL SYLLABLE DDOG..HANGUL SYLLABLE DDOH + {0xB62C, 0xB62C, prH2, gcLo}, // HANGUL SYLLABLE DDWA + {0xB62D, 0xB647, prH3, gcLo}, // [27] HANGUL SYLLABLE DDWAG..HANGUL SYLLABLE DDWAH + {0xB648, 0xB648, prH2, gcLo}, // HANGUL SYLLABLE DDWAE + {0xB649, 0xB663, prH3, gcLo}, // [27] HANGUL SYLLABLE DDWAEG..HANGUL SYLLABLE DDWAEH + {0xB664, 0xB664, prH2, gcLo}, // HANGUL SYLLABLE DDOE + {0xB665, 0xB67F, prH3, gcLo}, // [27] HANGUL SYLLABLE DDOEG..HANGUL SYLLABLE DDOEH + {0xB680, 0xB680, prH2, gcLo}, // HANGUL SYLLABLE DDYO + {0xB681, 0xB69B, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYOG..HANGUL SYLLABLE DDYOH + {0xB69C, 0xB69C, prH2, gcLo}, // HANGUL SYLLABLE DDU + {0xB69D, 0xB6B7, prH3, gcLo}, // [27] HANGUL SYLLABLE DDUG..HANGUL SYLLABLE DDUH + {0xB6B8, 0xB6B8, prH2, gcLo}, // HANGUL SYLLABLE DDWEO + {0xB6B9, 0xB6D3, prH3, gcLo}, // [27] HANGUL SYLLABLE DDWEOG..HANGUL SYLLABLE DDWEOH + {0xB6D4, 0xB6D4, prH2, gcLo}, // HANGUL SYLLABLE DDWE + {0xB6D5, 0xB6EF, prH3, gcLo}, // [27] HANGUL SYLLABLE DDWEG..HANGUL SYLLABLE DDWEH + {0xB6F0, 0xB6F0, prH2, gcLo}, // HANGUL SYLLABLE DDWI + {0xB6F1, 0xB70B, prH3, gcLo}, // [27] HANGUL SYLLABLE DDWIG..HANGUL SYLLABLE DDWIH + {0xB70C, 0xB70C, prH2, gcLo}, // HANGUL SYLLABLE DDYU + {0xB70D, 0xB727, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYUG..HANGUL SYLLABLE DDYUH + {0xB728, 0xB728, prH2, gcLo}, // HANGUL SYLLABLE DDEU + {0xB729, 0xB743, prH3, gcLo}, // [27] HANGUL SYLLABLE DDEUG..HANGUL SYLLABLE DDEUH + {0xB744, 0xB744, prH2, gcLo}, // HANGUL SYLLABLE DDYI + {0xB745, 0xB75F, prH3, gcLo}, // [27] HANGUL SYLLABLE DDYIG..HANGUL SYLLABLE DDYIH + {0xB760, 0xB760, prH2, gcLo}, // HANGUL SYLLABLE DDI + {0xB761, 0xB77B, prH3, gcLo}, // [27] HANGUL SYLLABLE DDIG..HANGUL SYLLABLE DDIH + {0xB77C, 0xB77C, prH2, gcLo}, // HANGUL SYLLABLE RA + {0xB77D, 0xB797, prH3, gcLo}, // [27] HANGUL SYLLABLE RAG..HANGUL SYLLABLE RAH + {0xB798, 0xB798, prH2, gcLo}, // HANGUL SYLLABLE RAE + {0xB799, 0xB7B3, prH3, gcLo}, // [27] HANGUL SYLLABLE RAEG..HANGUL SYLLABLE RAEH + {0xB7B4, 0xB7B4, prH2, gcLo}, // HANGUL SYLLABLE RYA + {0xB7B5, 0xB7CF, prH3, gcLo}, // [27] HANGUL SYLLABLE RYAG..HANGUL SYLLABLE RYAH + {0xB7D0, 0xB7D0, prH2, gcLo}, // HANGUL SYLLABLE RYAE + {0xB7D1, 0xB7EB, prH3, gcLo}, // [27] HANGUL SYLLABLE RYAEG..HANGUL SYLLABLE RYAEH + {0xB7EC, 0xB7EC, prH2, gcLo}, // HANGUL SYLLABLE REO + {0xB7ED, 0xB807, prH3, gcLo}, // [27] HANGUL SYLLABLE REOG..HANGUL SYLLABLE REOH + {0xB808, 0xB808, prH2, gcLo}, // HANGUL SYLLABLE RE + {0xB809, 0xB823, prH3, gcLo}, // [27] HANGUL SYLLABLE REG..HANGUL SYLLABLE REH + {0xB824, 0xB824, prH2, gcLo}, // HANGUL SYLLABLE RYEO + {0xB825, 0xB83F, prH3, gcLo}, // [27] HANGUL SYLLABLE RYEOG..HANGUL SYLLABLE RYEOH + {0xB840, 0xB840, prH2, gcLo}, // HANGUL SYLLABLE RYE + {0xB841, 0xB85B, prH3, gcLo}, // [27] HANGUL SYLLABLE RYEG..HANGUL SYLLABLE RYEH + {0xB85C, 0xB85C, prH2, gcLo}, // HANGUL SYLLABLE RO + {0xB85D, 0xB877, prH3, gcLo}, // [27] HANGUL SYLLABLE ROG..HANGUL SYLLABLE ROH + {0xB878, 0xB878, prH2, gcLo}, // HANGUL SYLLABLE RWA + {0xB879, 0xB893, prH3, gcLo}, // [27] HANGUL SYLLABLE RWAG..HANGUL SYLLABLE RWAH + {0xB894, 0xB894, prH2, gcLo}, // HANGUL SYLLABLE RWAE + {0xB895, 0xB8AF, prH3, gcLo}, // [27] HANGUL SYLLABLE RWAEG..HANGUL SYLLABLE RWAEH + {0xB8B0, 0xB8B0, prH2, gcLo}, // HANGUL SYLLABLE ROE + {0xB8B1, 0xB8CB, prH3, gcLo}, // [27] HANGUL SYLLABLE ROEG..HANGUL SYLLABLE ROEH + {0xB8CC, 0xB8CC, prH2, gcLo}, // HANGUL SYLLABLE RYO + {0xB8CD, 0xB8E7, prH3, gcLo}, // [27] HANGUL SYLLABLE RYOG..HANGUL SYLLABLE RYOH + {0xB8E8, 0xB8E8, prH2, gcLo}, // HANGUL SYLLABLE RU + {0xB8E9, 0xB903, prH3, gcLo}, // [27] HANGUL SYLLABLE RUG..HANGUL SYLLABLE RUH + {0xB904, 0xB904, prH2, gcLo}, // HANGUL SYLLABLE RWEO + {0xB905, 0xB91F, prH3, gcLo}, // [27] HANGUL SYLLABLE RWEOG..HANGUL SYLLABLE RWEOH + {0xB920, 0xB920, prH2, gcLo}, // HANGUL SYLLABLE RWE + {0xB921, 0xB93B, prH3, gcLo}, // [27] HANGUL SYLLABLE RWEG..HANGUL SYLLABLE RWEH + {0xB93C, 0xB93C, prH2, gcLo}, // HANGUL SYLLABLE RWI + {0xB93D, 0xB957, prH3, gcLo}, // [27] HANGUL SYLLABLE RWIG..HANGUL SYLLABLE RWIH + {0xB958, 0xB958, prH2, gcLo}, // HANGUL SYLLABLE RYU + {0xB959, 0xB973, prH3, gcLo}, // [27] HANGUL SYLLABLE RYUG..HANGUL SYLLABLE RYUH + {0xB974, 0xB974, prH2, gcLo}, // HANGUL SYLLABLE REU + {0xB975, 0xB98F, prH3, gcLo}, // [27] HANGUL SYLLABLE REUG..HANGUL SYLLABLE REUH + {0xB990, 0xB990, prH2, gcLo}, // HANGUL SYLLABLE RYI + {0xB991, 0xB9AB, prH3, gcLo}, // [27] HANGUL SYLLABLE RYIG..HANGUL SYLLABLE RYIH + {0xB9AC, 0xB9AC, prH2, gcLo}, // HANGUL SYLLABLE RI + {0xB9AD, 0xB9C7, prH3, gcLo}, // [27] HANGUL SYLLABLE RIG..HANGUL SYLLABLE RIH + {0xB9C8, 0xB9C8, prH2, gcLo}, // HANGUL SYLLABLE MA + {0xB9C9, 0xB9E3, prH3, gcLo}, // [27] HANGUL SYLLABLE MAG..HANGUL SYLLABLE MAH + {0xB9E4, 0xB9E4, prH2, gcLo}, // HANGUL SYLLABLE MAE + {0xB9E5, 0xB9FF, prH3, gcLo}, // [27] HANGUL SYLLABLE MAEG..HANGUL SYLLABLE MAEH + {0xBA00, 0xBA00, prH2, gcLo}, // HANGUL SYLLABLE MYA + {0xBA01, 0xBA1B, prH3, gcLo}, // [27] HANGUL SYLLABLE MYAG..HANGUL SYLLABLE MYAH + {0xBA1C, 0xBA1C, prH2, gcLo}, // HANGUL SYLLABLE MYAE + {0xBA1D, 0xBA37, prH3, gcLo}, // [27] HANGUL SYLLABLE MYAEG..HANGUL SYLLABLE MYAEH + {0xBA38, 0xBA38, prH2, gcLo}, // HANGUL SYLLABLE MEO + {0xBA39, 0xBA53, prH3, gcLo}, // [27] HANGUL SYLLABLE MEOG..HANGUL SYLLABLE MEOH + {0xBA54, 0xBA54, prH2, gcLo}, // HANGUL SYLLABLE ME + {0xBA55, 0xBA6F, prH3, gcLo}, // [27] HANGUL SYLLABLE MEG..HANGUL SYLLABLE MEH + {0xBA70, 0xBA70, prH2, gcLo}, // HANGUL SYLLABLE MYEO + {0xBA71, 0xBA8B, prH3, gcLo}, // [27] HANGUL SYLLABLE MYEOG..HANGUL SYLLABLE MYEOH + {0xBA8C, 0xBA8C, prH2, gcLo}, // HANGUL SYLLABLE MYE + {0xBA8D, 0xBAA7, prH3, gcLo}, // [27] HANGUL SYLLABLE MYEG..HANGUL SYLLABLE MYEH + {0xBAA8, 0xBAA8, prH2, gcLo}, // HANGUL SYLLABLE MO + {0xBAA9, 0xBAC3, prH3, gcLo}, // [27] HANGUL SYLLABLE MOG..HANGUL SYLLABLE MOH + {0xBAC4, 0xBAC4, prH2, gcLo}, // HANGUL SYLLABLE MWA + {0xBAC5, 0xBADF, prH3, gcLo}, // [27] HANGUL SYLLABLE MWAG..HANGUL SYLLABLE MWAH + {0xBAE0, 0xBAE0, prH2, gcLo}, // HANGUL SYLLABLE MWAE + {0xBAE1, 0xBAFB, prH3, gcLo}, // [27] HANGUL SYLLABLE MWAEG..HANGUL SYLLABLE MWAEH + {0xBAFC, 0xBAFC, prH2, gcLo}, // HANGUL SYLLABLE MOE + {0xBAFD, 0xBB17, prH3, gcLo}, // [27] HANGUL SYLLABLE MOEG..HANGUL SYLLABLE MOEH + {0xBB18, 0xBB18, prH2, gcLo}, // HANGUL SYLLABLE MYO + {0xBB19, 0xBB33, prH3, gcLo}, // [27] HANGUL SYLLABLE MYOG..HANGUL SYLLABLE MYOH + {0xBB34, 0xBB34, prH2, gcLo}, // HANGUL SYLLABLE MU + {0xBB35, 0xBB4F, prH3, gcLo}, // [27] HANGUL SYLLABLE MUG..HANGUL SYLLABLE MUH + {0xBB50, 0xBB50, prH2, gcLo}, // HANGUL SYLLABLE MWEO + {0xBB51, 0xBB6B, prH3, gcLo}, // [27] HANGUL SYLLABLE MWEOG..HANGUL SYLLABLE MWEOH + {0xBB6C, 0xBB6C, prH2, gcLo}, // HANGUL SYLLABLE MWE + {0xBB6D, 0xBB87, prH3, gcLo}, // [27] HANGUL SYLLABLE MWEG..HANGUL SYLLABLE MWEH + {0xBB88, 0xBB88, prH2, gcLo}, // HANGUL SYLLABLE MWI + {0xBB89, 0xBBA3, prH3, gcLo}, // [27] HANGUL SYLLABLE MWIG..HANGUL SYLLABLE MWIH + {0xBBA4, 0xBBA4, prH2, gcLo}, // HANGUL SYLLABLE MYU + {0xBBA5, 0xBBBF, prH3, gcLo}, // [27] HANGUL SYLLABLE MYUG..HANGUL SYLLABLE MYUH + {0xBBC0, 0xBBC0, prH2, gcLo}, // HANGUL SYLLABLE MEU + {0xBBC1, 0xBBDB, prH3, gcLo}, // [27] HANGUL SYLLABLE MEUG..HANGUL SYLLABLE MEUH + {0xBBDC, 0xBBDC, prH2, gcLo}, // HANGUL SYLLABLE MYI + {0xBBDD, 0xBBF7, prH3, gcLo}, // [27] HANGUL SYLLABLE MYIG..HANGUL SYLLABLE MYIH + {0xBBF8, 0xBBF8, prH2, gcLo}, // HANGUL SYLLABLE MI + {0xBBF9, 0xBC13, prH3, gcLo}, // [27] HANGUL SYLLABLE MIG..HANGUL SYLLABLE MIH + {0xBC14, 0xBC14, prH2, gcLo}, // HANGUL SYLLABLE BA + {0xBC15, 0xBC2F, prH3, gcLo}, // [27] HANGUL SYLLABLE BAG..HANGUL SYLLABLE BAH + {0xBC30, 0xBC30, prH2, gcLo}, // HANGUL SYLLABLE BAE + {0xBC31, 0xBC4B, prH3, gcLo}, // [27] HANGUL SYLLABLE BAEG..HANGUL SYLLABLE BAEH + {0xBC4C, 0xBC4C, prH2, gcLo}, // HANGUL SYLLABLE BYA + {0xBC4D, 0xBC67, prH3, gcLo}, // [27] HANGUL SYLLABLE BYAG..HANGUL SYLLABLE BYAH + {0xBC68, 0xBC68, prH2, gcLo}, // HANGUL SYLLABLE BYAE + {0xBC69, 0xBC83, prH3, gcLo}, // [27] HANGUL SYLLABLE BYAEG..HANGUL SYLLABLE BYAEH + {0xBC84, 0xBC84, prH2, gcLo}, // HANGUL SYLLABLE BEO + {0xBC85, 0xBC9F, prH3, gcLo}, // [27] HANGUL SYLLABLE BEOG..HANGUL SYLLABLE BEOH + {0xBCA0, 0xBCA0, prH2, gcLo}, // HANGUL SYLLABLE BE + {0xBCA1, 0xBCBB, prH3, gcLo}, // [27] HANGUL SYLLABLE BEG..HANGUL SYLLABLE BEH + {0xBCBC, 0xBCBC, prH2, gcLo}, // HANGUL SYLLABLE BYEO + {0xBCBD, 0xBCD7, prH3, gcLo}, // [27] HANGUL SYLLABLE BYEOG..HANGUL SYLLABLE BYEOH + {0xBCD8, 0xBCD8, prH2, gcLo}, // HANGUL SYLLABLE BYE + {0xBCD9, 0xBCF3, prH3, gcLo}, // [27] HANGUL SYLLABLE BYEG..HANGUL SYLLABLE BYEH + {0xBCF4, 0xBCF4, prH2, gcLo}, // HANGUL SYLLABLE BO + {0xBCF5, 0xBD0F, prH3, gcLo}, // [27] HANGUL SYLLABLE BOG..HANGUL SYLLABLE BOH + {0xBD10, 0xBD10, prH2, gcLo}, // HANGUL SYLLABLE BWA + {0xBD11, 0xBD2B, prH3, gcLo}, // [27] HANGUL SYLLABLE BWAG..HANGUL SYLLABLE BWAH + {0xBD2C, 0xBD2C, prH2, gcLo}, // HANGUL SYLLABLE BWAE + {0xBD2D, 0xBD47, prH3, gcLo}, // [27] HANGUL SYLLABLE BWAEG..HANGUL SYLLABLE BWAEH + {0xBD48, 0xBD48, prH2, gcLo}, // HANGUL SYLLABLE BOE + {0xBD49, 0xBD63, prH3, gcLo}, // [27] HANGUL SYLLABLE BOEG..HANGUL SYLLABLE BOEH + {0xBD64, 0xBD64, prH2, gcLo}, // HANGUL SYLLABLE BYO + {0xBD65, 0xBD7F, prH3, gcLo}, // [27] HANGUL SYLLABLE BYOG..HANGUL SYLLABLE BYOH + {0xBD80, 0xBD80, prH2, gcLo}, // HANGUL SYLLABLE BU + {0xBD81, 0xBD9B, prH3, gcLo}, // [27] HANGUL SYLLABLE BUG..HANGUL SYLLABLE BUH + {0xBD9C, 0xBD9C, prH2, gcLo}, // HANGUL SYLLABLE BWEO + {0xBD9D, 0xBDB7, prH3, gcLo}, // [27] HANGUL SYLLABLE BWEOG..HANGUL SYLLABLE BWEOH + {0xBDB8, 0xBDB8, prH2, gcLo}, // HANGUL SYLLABLE BWE + {0xBDB9, 0xBDD3, prH3, gcLo}, // [27] HANGUL SYLLABLE BWEG..HANGUL SYLLABLE BWEH + {0xBDD4, 0xBDD4, prH2, gcLo}, // HANGUL SYLLABLE BWI + {0xBDD5, 0xBDEF, prH3, gcLo}, // [27] HANGUL SYLLABLE BWIG..HANGUL SYLLABLE BWIH + {0xBDF0, 0xBDF0, prH2, gcLo}, // HANGUL SYLLABLE BYU + {0xBDF1, 0xBE0B, prH3, gcLo}, // [27] HANGUL SYLLABLE BYUG..HANGUL SYLLABLE BYUH + {0xBE0C, 0xBE0C, prH2, gcLo}, // HANGUL SYLLABLE BEU + {0xBE0D, 0xBE27, prH3, gcLo}, // [27] HANGUL SYLLABLE BEUG..HANGUL SYLLABLE BEUH + {0xBE28, 0xBE28, prH2, gcLo}, // HANGUL SYLLABLE BYI + {0xBE29, 0xBE43, prH3, gcLo}, // [27] HANGUL SYLLABLE BYIG..HANGUL SYLLABLE BYIH + {0xBE44, 0xBE44, prH2, gcLo}, // HANGUL SYLLABLE BI + {0xBE45, 0xBE5F, prH3, gcLo}, // [27] HANGUL SYLLABLE BIG..HANGUL SYLLABLE BIH + {0xBE60, 0xBE60, prH2, gcLo}, // HANGUL SYLLABLE BBA + {0xBE61, 0xBE7B, prH3, gcLo}, // [27] HANGUL SYLLABLE BBAG..HANGUL SYLLABLE BBAH + {0xBE7C, 0xBE7C, prH2, gcLo}, // HANGUL SYLLABLE BBAE + {0xBE7D, 0xBE97, prH3, gcLo}, // [27] HANGUL SYLLABLE BBAEG..HANGUL SYLLABLE BBAEH + {0xBE98, 0xBE98, prH2, gcLo}, // HANGUL SYLLABLE BBYA + {0xBE99, 0xBEB3, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYAG..HANGUL SYLLABLE BBYAH + {0xBEB4, 0xBEB4, prH2, gcLo}, // HANGUL SYLLABLE BBYAE + {0xBEB5, 0xBECF, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYAEG..HANGUL SYLLABLE BBYAEH + {0xBED0, 0xBED0, prH2, gcLo}, // HANGUL SYLLABLE BBEO + {0xBED1, 0xBEEB, prH3, gcLo}, // [27] HANGUL SYLLABLE BBEOG..HANGUL SYLLABLE BBEOH + {0xBEEC, 0xBEEC, prH2, gcLo}, // HANGUL SYLLABLE BBE + {0xBEED, 0xBF07, prH3, gcLo}, // [27] HANGUL SYLLABLE BBEG..HANGUL SYLLABLE BBEH + {0xBF08, 0xBF08, prH2, gcLo}, // HANGUL SYLLABLE BBYEO + {0xBF09, 0xBF23, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYEOG..HANGUL SYLLABLE BBYEOH + {0xBF24, 0xBF24, prH2, gcLo}, // HANGUL SYLLABLE BBYE + {0xBF25, 0xBF3F, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYEG..HANGUL SYLLABLE BBYEH + {0xBF40, 0xBF40, prH2, gcLo}, // HANGUL SYLLABLE BBO + {0xBF41, 0xBF5B, prH3, gcLo}, // [27] HANGUL SYLLABLE BBOG..HANGUL SYLLABLE BBOH + {0xBF5C, 0xBF5C, prH2, gcLo}, // HANGUL SYLLABLE BBWA + {0xBF5D, 0xBF77, prH3, gcLo}, // [27] HANGUL SYLLABLE BBWAG..HANGUL SYLLABLE BBWAH + {0xBF78, 0xBF78, prH2, gcLo}, // HANGUL SYLLABLE BBWAE + {0xBF79, 0xBF93, prH3, gcLo}, // [27] HANGUL SYLLABLE BBWAEG..HANGUL SYLLABLE BBWAEH + {0xBF94, 0xBF94, prH2, gcLo}, // HANGUL SYLLABLE BBOE + {0xBF95, 0xBFAF, prH3, gcLo}, // [27] HANGUL SYLLABLE BBOEG..HANGUL SYLLABLE BBOEH + {0xBFB0, 0xBFB0, prH2, gcLo}, // HANGUL SYLLABLE BBYO + {0xBFB1, 0xBFCB, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYOG..HANGUL SYLLABLE BBYOH + {0xBFCC, 0xBFCC, prH2, gcLo}, // HANGUL SYLLABLE BBU + {0xBFCD, 0xBFE7, prH3, gcLo}, // [27] HANGUL SYLLABLE BBUG..HANGUL SYLLABLE BBUH + {0xBFE8, 0xBFE8, prH2, gcLo}, // HANGUL SYLLABLE BBWEO + {0xBFE9, 0xC003, prH3, gcLo}, // [27] HANGUL SYLLABLE BBWEOG..HANGUL SYLLABLE BBWEOH + {0xC004, 0xC004, prH2, gcLo}, // HANGUL SYLLABLE BBWE + {0xC005, 0xC01F, prH3, gcLo}, // [27] HANGUL SYLLABLE BBWEG..HANGUL SYLLABLE BBWEH + {0xC020, 0xC020, prH2, gcLo}, // HANGUL SYLLABLE BBWI + {0xC021, 0xC03B, prH3, gcLo}, // [27] HANGUL SYLLABLE BBWIG..HANGUL SYLLABLE BBWIH + {0xC03C, 0xC03C, prH2, gcLo}, // HANGUL SYLLABLE BBYU + {0xC03D, 0xC057, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYUG..HANGUL SYLLABLE BBYUH + {0xC058, 0xC058, prH2, gcLo}, // HANGUL SYLLABLE BBEU + {0xC059, 0xC073, prH3, gcLo}, // [27] HANGUL SYLLABLE BBEUG..HANGUL SYLLABLE BBEUH + {0xC074, 0xC074, prH2, gcLo}, // HANGUL SYLLABLE BBYI + {0xC075, 0xC08F, prH3, gcLo}, // [27] HANGUL SYLLABLE BBYIG..HANGUL SYLLABLE BBYIH + {0xC090, 0xC090, prH2, gcLo}, // HANGUL SYLLABLE BBI + {0xC091, 0xC0AB, prH3, gcLo}, // [27] HANGUL SYLLABLE BBIG..HANGUL SYLLABLE BBIH + {0xC0AC, 0xC0AC, prH2, gcLo}, // HANGUL SYLLABLE SA + {0xC0AD, 0xC0C7, prH3, gcLo}, // [27] HANGUL SYLLABLE SAG..HANGUL SYLLABLE SAH + {0xC0C8, 0xC0C8, prH2, gcLo}, // HANGUL SYLLABLE SAE + {0xC0C9, 0xC0E3, prH3, gcLo}, // [27] HANGUL SYLLABLE SAEG..HANGUL SYLLABLE SAEH + {0xC0E4, 0xC0E4, prH2, gcLo}, // HANGUL SYLLABLE SYA + {0xC0E5, 0xC0FF, prH3, gcLo}, // [27] HANGUL SYLLABLE SYAG..HANGUL SYLLABLE SYAH + {0xC100, 0xC100, prH2, gcLo}, // HANGUL SYLLABLE SYAE + {0xC101, 0xC11B, prH3, gcLo}, // [27] HANGUL SYLLABLE SYAEG..HANGUL SYLLABLE SYAEH + {0xC11C, 0xC11C, prH2, gcLo}, // HANGUL SYLLABLE SEO + {0xC11D, 0xC137, prH3, gcLo}, // [27] HANGUL SYLLABLE SEOG..HANGUL SYLLABLE SEOH + {0xC138, 0xC138, prH2, gcLo}, // HANGUL SYLLABLE SE + {0xC139, 0xC153, prH3, gcLo}, // [27] HANGUL SYLLABLE SEG..HANGUL SYLLABLE SEH + {0xC154, 0xC154, prH2, gcLo}, // HANGUL SYLLABLE SYEO + {0xC155, 0xC16F, prH3, gcLo}, // [27] HANGUL SYLLABLE SYEOG..HANGUL SYLLABLE SYEOH + {0xC170, 0xC170, prH2, gcLo}, // HANGUL SYLLABLE SYE + {0xC171, 0xC18B, prH3, gcLo}, // [27] HANGUL SYLLABLE SYEG..HANGUL SYLLABLE SYEH + {0xC18C, 0xC18C, prH2, gcLo}, // HANGUL SYLLABLE SO + {0xC18D, 0xC1A7, prH3, gcLo}, // [27] HANGUL SYLLABLE SOG..HANGUL SYLLABLE SOH + {0xC1A8, 0xC1A8, prH2, gcLo}, // HANGUL SYLLABLE SWA + {0xC1A9, 0xC1C3, prH3, gcLo}, // [27] HANGUL SYLLABLE SWAG..HANGUL SYLLABLE SWAH + {0xC1C4, 0xC1C4, prH2, gcLo}, // HANGUL SYLLABLE SWAE + {0xC1C5, 0xC1DF, prH3, gcLo}, // [27] HANGUL SYLLABLE SWAEG..HANGUL SYLLABLE SWAEH + {0xC1E0, 0xC1E0, prH2, gcLo}, // HANGUL SYLLABLE SOE + {0xC1E1, 0xC1FB, prH3, gcLo}, // [27] HANGUL SYLLABLE SOEG..HANGUL SYLLABLE SOEH + {0xC1FC, 0xC1FC, prH2, gcLo}, // HANGUL SYLLABLE SYO + {0xC1FD, 0xC217, prH3, gcLo}, // [27] HANGUL SYLLABLE SYOG..HANGUL SYLLABLE SYOH + {0xC218, 0xC218, prH2, gcLo}, // HANGUL SYLLABLE SU + {0xC219, 0xC233, prH3, gcLo}, // [27] HANGUL SYLLABLE SUG..HANGUL SYLLABLE SUH + {0xC234, 0xC234, prH2, gcLo}, // HANGUL SYLLABLE SWEO + {0xC235, 0xC24F, prH3, gcLo}, // [27] HANGUL SYLLABLE SWEOG..HANGUL SYLLABLE SWEOH + {0xC250, 0xC250, prH2, gcLo}, // HANGUL SYLLABLE SWE + {0xC251, 0xC26B, prH3, gcLo}, // [27] HANGUL SYLLABLE SWEG..HANGUL SYLLABLE SWEH + {0xC26C, 0xC26C, prH2, gcLo}, // HANGUL SYLLABLE SWI + {0xC26D, 0xC287, prH3, gcLo}, // [27] HANGUL SYLLABLE SWIG..HANGUL SYLLABLE SWIH + {0xC288, 0xC288, prH2, gcLo}, // HANGUL SYLLABLE SYU + {0xC289, 0xC2A3, prH3, gcLo}, // [27] HANGUL SYLLABLE SYUG..HANGUL SYLLABLE SYUH + {0xC2A4, 0xC2A4, prH2, gcLo}, // HANGUL SYLLABLE SEU + {0xC2A5, 0xC2BF, prH3, gcLo}, // [27] HANGUL SYLLABLE SEUG..HANGUL SYLLABLE SEUH + {0xC2C0, 0xC2C0, prH2, gcLo}, // HANGUL SYLLABLE SYI + {0xC2C1, 0xC2DB, prH3, gcLo}, // [27] HANGUL SYLLABLE SYIG..HANGUL SYLLABLE SYIH + {0xC2DC, 0xC2DC, prH2, gcLo}, // HANGUL SYLLABLE SI + {0xC2DD, 0xC2F7, prH3, gcLo}, // [27] HANGUL SYLLABLE SIG..HANGUL SYLLABLE SIH + {0xC2F8, 0xC2F8, prH2, gcLo}, // HANGUL SYLLABLE SSA + {0xC2F9, 0xC313, prH3, gcLo}, // [27] HANGUL SYLLABLE SSAG..HANGUL SYLLABLE SSAH + {0xC314, 0xC314, prH2, gcLo}, // HANGUL SYLLABLE SSAE + {0xC315, 0xC32F, prH3, gcLo}, // [27] HANGUL SYLLABLE SSAEG..HANGUL SYLLABLE SSAEH + {0xC330, 0xC330, prH2, gcLo}, // HANGUL SYLLABLE SSYA + {0xC331, 0xC34B, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYAG..HANGUL SYLLABLE SSYAH + {0xC34C, 0xC34C, prH2, gcLo}, // HANGUL SYLLABLE SSYAE + {0xC34D, 0xC367, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYAEG..HANGUL SYLLABLE SSYAEH + {0xC368, 0xC368, prH2, gcLo}, // HANGUL SYLLABLE SSEO + {0xC369, 0xC383, prH3, gcLo}, // [27] HANGUL SYLLABLE SSEOG..HANGUL SYLLABLE SSEOH + {0xC384, 0xC384, prH2, gcLo}, // HANGUL SYLLABLE SSE + {0xC385, 0xC39F, prH3, gcLo}, // [27] HANGUL SYLLABLE SSEG..HANGUL SYLLABLE SSEH + {0xC3A0, 0xC3A0, prH2, gcLo}, // HANGUL SYLLABLE SSYEO + {0xC3A1, 0xC3BB, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYEOG..HANGUL SYLLABLE SSYEOH + {0xC3BC, 0xC3BC, prH2, gcLo}, // HANGUL SYLLABLE SSYE + {0xC3BD, 0xC3D7, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYEG..HANGUL SYLLABLE SSYEH + {0xC3D8, 0xC3D8, prH2, gcLo}, // HANGUL SYLLABLE SSO + {0xC3D9, 0xC3F3, prH3, gcLo}, // [27] HANGUL SYLLABLE SSOG..HANGUL SYLLABLE SSOH + {0xC3F4, 0xC3F4, prH2, gcLo}, // HANGUL SYLLABLE SSWA + {0xC3F5, 0xC40F, prH3, gcLo}, // [27] HANGUL SYLLABLE SSWAG..HANGUL SYLLABLE SSWAH + {0xC410, 0xC410, prH2, gcLo}, // HANGUL SYLLABLE SSWAE + {0xC411, 0xC42B, prH3, gcLo}, // [27] HANGUL SYLLABLE SSWAEG..HANGUL SYLLABLE SSWAEH + {0xC42C, 0xC42C, prH2, gcLo}, // HANGUL SYLLABLE SSOE + {0xC42D, 0xC447, prH3, gcLo}, // [27] HANGUL SYLLABLE SSOEG..HANGUL SYLLABLE SSOEH + {0xC448, 0xC448, prH2, gcLo}, // HANGUL SYLLABLE SSYO + {0xC449, 0xC463, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYOG..HANGUL SYLLABLE SSYOH + {0xC464, 0xC464, prH2, gcLo}, // HANGUL SYLLABLE SSU + {0xC465, 0xC47F, prH3, gcLo}, // [27] HANGUL SYLLABLE SSUG..HANGUL SYLLABLE SSUH + {0xC480, 0xC480, prH2, gcLo}, // HANGUL SYLLABLE SSWEO + {0xC481, 0xC49B, prH3, gcLo}, // [27] HANGUL SYLLABLE SSWEOG..HANGUL SYLLABLE SSWEOH + {0xC49C, 0xC49C, prH2, gcLo}, // HANGUL SYLLABLE SSWE + {0xC49D, 0xC4B7, prH3, gcLo}, // [27] HANGUL SYLLABLE SSWEG..HANGUL SYLLABLE SSWEH + {0xC4B8, 0xC4B8, prH2, gcLo}, // HANGUL SYLLABLE SSWI + {0xC4B9, 0xC4D3, prH3, gcLo}, // [27] HANGUL SYLLABLE SSWIG..HANGUL SYLLABLE SSWIH + {0xC4D4, 0xC4D4, prH2, gcLo}, // HANGUL SYLLABLE SSYU + {0xC4D5, 0xC4EF, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYUG..HANGUL SYLLABLE SSYUH + {0xC4F0, 0xC4F0, prH2, gcLo}, // HANGUL SYLLABLE SSEU + {0xC4F1, 0xC50B, prH3, gcLo}, // [27] HANGUL SYLLABLE SSEUG..HANGUL SYLLABLE SSEUH + {0xC50C, 0xC50C, prH2, gcLo}, // HANGUL SYLLABLE SSYI + {0xC50D, 0xC527, prH3, gcLo}, // [27] HANGUL SYLLABLE SSYIG..HANGUL SYLLABLE SSYIH + {0xC528, 0xC528, prH2, gcLo}, // HANGUL SYLLABLE SSI + {0xC529, 0xC543, prH3, gcLo}, // [27] HANGUL SYLLABLE SSIG..HANGUL SYLLABLE SSIH + {0xC544, 0xC544, prH2, gcLo}, // HANGUL SYLLABLE A + {0xC545, 0xC55F, prH3, gcLo}, // [27] HANGUL SYLLABLE AG..HANGUL SYLLABLE AH + {0xC560, 0xC560, prH2, gcLo}, // HANGUL SYLLABLE AE + {0xC561, 0xC57B, prH3, gcLo}, // [27] HANGUL SYLLABLE AEG..HANGUL SYLLABLE AEH + {0xC57C, 0xC57C, prH2, gcLo}, // HANGUL SYLLABLE YA + {0xC57D, 0xC597, prH3, gcLo}, // [27] HANGUL SYLLABLE YAG..HANGUL SYLLABLE YAH + {0xC598, 0xC598, prH2, gcLo}, // HANGUL SYLLABLE YAE + {0xC599, 0xC5B3, prH3, gcLo}, // [27] HANGUL SYLLABLE YAEG..HANGUL SYLLABLE YAEH + {0xC5B4, 0xC5B4, prH2, gcLo}, // HANGUL SYLLABLE EO + {0xC5B5, 0xC5CF, prH3, gcLo}, // [27] HANGUL SYLLABLE EOG..HANGUL SYLLABLE EOH + {0xC5D0, 0xC5D0, prH2, gcLo}, // HANGUL SYLLABLE E + {0xC5D1, 0xC5EB, prH3, gcLo}, // [27] HANGUL SYLLABLE EG..HANGUL SYLLABLE EH + {0xC5EC, 0xC5EC, prH2, gcLo}, // HANGUL SYLLABLE YEO + {0xC5ED, 0xC607, prH3, gcLo}, // [27] HANGUL SYLLABLE YEOG..HANGUL SYLLABLE YEOH + {0xC608, 0xC608, prH2, gcLo}, // HANGUL SYLLABLE YE + {0xC609, 0xC623, prH3, gcLo}, // [27] HANGUL SYLLABLE YEG..HANGUL SYLLABLE YEH + {0xC624, 0xC624, prH2, gcLo}, // HANGUL SYLLABLE O + {0xC625, 0xC63F, prH3, gcLo}, // [27] HANGUL SYLLABLE OG..HANGUL SYLLABLE OH + {0xC640, 0xC640, prH2, gcLo}, // HANGUL SYLLABLE WA + {0xC641, 0xC65B, prH3, gcLo}, // [27] HANGUL SYLLABLE WAG..HANGUL SYLLABLE WAH + {0xC65C, 0xC65C, prH2, gcLo}, // HANGUL SYLLABLE WAE + {0xC65D, 0xC677, prH3, gcLo}, // [27] HANGUL SYLLABLE WAEG..HANGUL SYLLABLE WAEH + {0xC678, 0xC678, prH2, gcLo}, // HANGUL SYLLABLE OE + {0xC679, 0xC693, prH3, gcLo}, // [27] HANGUL SYLLABLE OEG..HANGUL SYLLABLE OEH + {0xC694, 0xC694, prH2, gcLo}, // HANGUL SYLLABLE YO + {0xC695, 0xC6AF, prH3, gcLo}, // [27] HANGUL SYLLABLE YOG..HANGUL SYLLABLE YOH + {0xC6B0, 0xC6B0, prH2, gcLo}, // HANGUL SYLLABLE U + {0xC6B1, 0xC6CB, prH3, gcLo}, // [27] HANGUL SYLLABLE UG..HANGUL SYLLABLE UH + {0xC6CC, 0xC6CC, prH2, gcLo}, // HANGUL SYLLABLE WEO + {0xC6CD, 0xC6E7, prH3, gcLo}, // [27] HANGUL SYLLABLE WEOG..HANGUL SYLLABLE WEOH + {0xC6E8, 0xC6E8, prH2, gcLo}, // HANGUL SYLLABLE WE + {0xC6E9, 0xC703, prH3, gcLo}, // [27] HANGUL SYLLABLE WEG..HANGUL SYLLABLE WEH + {0xC704, 0xC704, prH2, gcLo}, // HANGUL SYLLABLE WI + {0xC705, 0xC71F, prH3, gcLo}, // [27] HANGUL SYLLABLE WIG..HANGUL SYLLABLE WIH + {0xC720, 0xC720, prH2, gcLo}, // HANGUL SYLLABLE YU + {0xC721, 0xC73B, prH3, gcLo}, // [27] HANGUL SYLLABLE YUG..HANGUL SYLLABLE YUH + {0xC73C, 0xC73C, prH2, gcLo}, // HANGUL SYLLABLE EU + {0xC73D, 0xC757, prH3, gcLo}, // [27] HANGUL SYLLABLE EUG..HANGUL SYLLABLE EUH + {0xC758, 0xC758, prH2, gcLo}, // HANGUL SYLLABLE YI + {0xC759, 0xC773, prH3, gcLo}, // [27] HANGUL SYLLABLE YIG..HANGUL SYLLABLE YIH + {0xC774, 0xC774, prH2, gcLo}, // HANGUL SYLLABLE I + {0xC775, 0xC78F, prH3, gcLo}, // [27] HANGUL SYLLABLE IG..HANGUL SYLLABLE IH + {0xC790, 0xC790, prH2, gcLo}, // HANGUL SYLLABLE JA + {0xC791, 0xC7AB, prH3, gcLo}, // [27] HANGUL SYLLABLE JAG..HANGUL SYLLABLE JAH + {0xC7AC, 0xC7AC, prH2, gcLo}, // HANGUL SYLLABLE JAE + {0xC7AD, 0xC7C7, prH3, gcLo}, // [27] HANGUL SYLLABLE JAEG..HANGUL SYLLABLE JAEH + {0xC7C8, 0xC7C8, prH2, gcLo}, // HANGUL SYLLABLE JYA + {0xC7C9, 0xC7E3, prH3, gcLo}, // [27] HANGUL SYLLABLE JYAG..HANGUL SYLLABLE JYAH + {0xC7E4, 0xC7E4, prH2, gcLo}, // HANGUL SYLLABLE JYAE + {0xC7E5, 0xC7FF, prH3, gcLo}, // [27] HANGUL SYLLABLE JYAEG..HANGUL SYLLABLE JYAEH + {0xC800, 0xC800, prH2, gcLo}, // HANGUL SYLLABLE JEO + {0xC801, 0xC81B, prH3, gcLo}, // [27] HANGUL SYLLABLE JEOG..HANGUL SYLLABLE JEOH + {0xC81C, 0xC81C, prH2, gcLo}, // HANGUL SYLLABLE JE + {0xC81D, 0xC837, prH3, gcLo}, // [27] HANGUL SYLLABLE JEG..HANGUL SYLLABLE JEH + {0xC838, 0xC838, prH2, gcLo}, // HANGUL SYLLABLE JYEO + {0xC839, 0xC853, prH3, gcLo}, // [27] HANGUL SYLLABLE JYEOG..HANGUL SYLLABLE JYEOH + {0xC854, 0xC854, prH2, gcLo}, // HANGUL SYLLABLE JYE + {0xC855, 0xC86F, prH3, gcLo}, // [27] HANGUL SYLLABLE JYEG..HANGUL SYLLABLE JYEH + {0xC870, 0xC870, prH2, gcLo}, // HANGUL SYLLABLE JO + {0xC871, 0xC88B, prH3, gcLo}, // [27] HANGUL SYLLABLE JOG..HANGUL SYLLABLE JOH + {0xC88C, 0xC88C, prH2, gcLo}, // HANGUL SYLLABLE JWA + {0xC88D, 0xC8A7, prH3, gcLo}, // [27] HANGUL SYLLABLE JWAG..HANGUL SYLLABLE JWAH + {0xC8A8, 0xC8A8, prH2, gcLo}, // HANGUL SYLLABLE JWAE + {0xC8A9, 0xC8C3, prH3, gcLo}, // [27] HANGUL SYLLABLE JWAEG..HANGUL SYLLABLE JWAEH + {0xC8C4, 0xC8C4, prH2, gcLo}, // HANGUL SYLLABLE JOE + {0xC8C5, 0xC8DF, prH3, gcLo}, // [27] HANGUL SYLLABLE JOEG..HANGUL SYLLABLE JOEH + {0xC8E0, 0xC8E0, prH2, gcLo}, // HANGUL SYLLABLE JYO + {0xC8E1, 0xC8FB, prH3, gcLo}, // [27] HANGUL SYLLABLE JYOG..HANGUL SYLLABLE JYOH + {0xC8FC, 0xC8FC, prH2, gcLo}, // HANGUL SYLLABLE JU + {0xC8FD, 0xC917, prH3, gcLo}, // [27] HANGUL SYLLABLE JUG..HANGUL SYLLABLE JUH + {0xC918, 0xC918, prH2, gcLo}, // HANGUL SYLLABLE JWEO + {0xC919, 0xC933, prH3, gcLo}, // [27] HANGUL SYLLABLE JWEOG..HANGUL SYLLABLE JWEOH + {0xC934, 0xC934, prH2, gcLo}, // HANGUL SYLLABLE JWE + {0xC935, 0xC94F, prH3, gcLo}, // [27] HANGUL SYLLABLE JWEG..HANGUL SYLLABLE JWEH + {0xC950, 0xC950, prH2, gcLo}, // HANGUL SYLLABLE JWI + {0xC951, 0xC96B, prH3, gcLo}, // [27] HANGUL SYLLABLE JWIG..HANGUL SYLLABLE JWIH + {0xC96C, 0xC96C, prH2, gcLo}, // HANGUL SYLLABLE JYU + {0xC96D, 0xC987, prH3, gcLo}, // [27] HANGUL SYLLABLE JYUG..HANGUL SYLLABLE JYUH + {0xC988, 0xC988, prH2, gcLo}, // HANGUL SYLLABLE JEU + {0xC989, 0xC9A3, prH3, gcLo}, // [27] HANGUL SYLLABLE JEUG..HANGUL SYLLABLE JEUH + {0xC9A4, 0xC9A4, prH2, gcLo}, // HANGUL SYLLABLE JYI + {0xC9A5, 0xC9BF, prH3, gcLo}, // [27] HANGUL SYLLABLE JYIG..HANGUL SYLLABLE JYIH + {0xC9C0, 0xC9C0, prH2, gcLo}, // HANGUL SYLLABLE JI + {0xC9C1, 0xC9DB, prH3, gcLo}, // [27] HANGUL SYLLABLE JIG..HANGUL SYLLABLE JIH + {0xC9DC, 0xC9DC, prH2, gcLo}, // HANGUL SYLLABLE JJA + {0xC9DD, 0xC9F7, prH3, gcLo}, // [27] HANGUL SYLLABLE JJAG..HANGUL SYLLABLE JJAH + {0xC9F8, 0xC9F8, prH2, gcLo}, // HANGUL SYLLABLE JJAE + {0xC9F9, 0xCA13, prH3, gcLo}, // [27] HANGUL SYLLABLE JJAEG..HANGUL SYLLABLE JJAEH + {0xCA14, 0xCA14, prH2, gcLo}, // HANGUL SYLLABLE JJYA + {0xCA15, 0xCA2F, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYAG..HANGUL SYLLABLE JJYAH + {0xCA30, 0xCA30, prH2, gcLo}, // HANGUL SYLLABLE JJYAE + {0xCA31, 0xCA4B, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYAEG..HANGUL SYLLABLE JJYAEH + {0xCA4C, 0xCA4C, prH2, gcLo}, // HANGUL SYLLABLE JJEO + {0xCA4D, 0xCA67, prH3, gcLo}, // [27] HANGUL SYLLABLE JJEOG..HANGUL SYLLABLE JJEOH + {0xCA68, 0xCA68, prH2, gcLo}, // HANGUL SYLLABLE JJE + {0xCA69, 0xCA83, prH3, gcLo}, // [27] HANGUL SYLLABLE JJEG..HANGUL SYLLABLE JJEH + {0xCA84, 0xCA84, prH2, gcLo}, // HANGUL SYLLABLE JJYEO + {0xCA85, 0xCA9F, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYEOG..HANGUL SYLLABLE JJYEOH + {0xCAA0, 0xCAA0, prH2, gcLo}, // HANGUL SYLLABLE JJYE + {0xCAA1, 0xCABB, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYEG..HANGUL SYLLABLE JJYEH + {0xCABC, 0xCABC, prH2, gcLo}, // HANGUL SYLLABLE JJO + {0xCABD, 0xCAD7, prH3, gcLo}, // [27] HANGUL SYLLABLE JJOG..HANGUL SYLLABLE JJOH + {0xCAD8, 0xCAD8, prH2, gcLo}, // HANGUL SYLLABLE JJWA + {0xCAD9, 0xCAF3, prH3, gcLo}, // [27] HANGUL SYLLABLE JJWAG..HANGUL SYLLABLE JJWAH + {0xCAF4, 0xCAF4, prH2, gcLo}, // HANGUL SYLLABLE JJWAE + {0xCAF5, 0xCB0F, prH3, gcLo}, // [27] HANGUL SYLLABLE JJWAEG..HANGUL SYLLABLE JJWAEH + {0xCB10, 0xCB10, prH2, gcLo}, // HANGUL SYLLABLE JJOE + {0xCB11, 0xCB2B, prH3, gcLo}, // [27] HANGUL SYLLABLE JJOEG..HANGUL SYLLABLE JJOEH + {0xCB2C, 0xCB2C, prH2, gcLo}, // HANGUL SYLLABLE JJYO + {0xCB2D, 0xCB47, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYOG..HANGUL SYLLABLE JJYOH + {0xCB48, 0xCB48, prH2, gcLo}, // HANGUL SYLLABLE JJU + {0xCB49, 0xCB63, prH3, gcLo}, // [27] HANGUL SYLLABLE JJUG..HANGUL SYLLABLE JJUH + {0xCB64, 0xCB64, prH2, gcLo}, // HANGUL SYLLABLE JJWEO + {0xCB65, 0xCB7F, prH3, gcLo}, // [27] HANGUL SYLLABLE JJWEOG..HANGUL SYLLABLE JJWEOH + {0xCB80, 0xCB80, prH2, gcLo}, // HANGUL SYLLABLE JJWE + {0xCB81, 0xCB9B, prH3, gcLo}, // [27] HANGUL SYLLABLE JJWEG..HANGUL SYLLABLE JJWEH + {0xCB9C, 0xCB9C, prH2, gcLo}, // HANGUL SYLLABLE JJWI + {0xCB9D, 0xCBB7, prH3, gcLo}, // [27] HANGUL SYLLABLE JJWIG..HANGUL SYLLABLE JJWIH + {0xCBB8, 0xCBB8, prH2, gcLo}, // HANGUL SYLLABLE JJYU + {0xCBB9, 0xCBD3, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYUG..HANGUL SYLLABLE JJYUH + {0xCBD4, 0xCBD4, prH2, gcLo}, // HANGUL SYLLABLE JJEU + {0xCBD5, 0xCBEF, prH3, gcLo}, // [27] HANGUL SYLLABLE JJEUG..HANGUL SYLLABLE JJEUH + {0xCBF0, 0xCBF0, prH2, gcLo}, // HANGUL SYLLABLE JJYI + {0xCBF1, 0xCC0B, prH3, gcLo}, // [27] HANGUL SYLLABLE JJYIG..HANGUL SYLLABLE JJYIH + {0xCC0C, 0xCC0C, prH2, gcLo}, // HANGUL SYLLABLE JJI + {0xCC0D, 0xCC27, prH3, gcLo}, // [27] HANGUL SYLLABLE JJIG..HANGUL SYLLABLE JJIH + {0xCC28, 0xCC28, prH2, gcLo}, // HANGUL SYLLABLE CA + {0xCC29, 0xCC43, prH3, gcLo}, // [27] HANGUL SYLLABLE CAG..HANGUL SYLLABLE CAH + {0xCC44, 0xCC44, prH2, gcLo}, // HANGUL SYLLABLE CAE + {0xCC45, 0xCC5F, prH3, gcLo}, // [27] HANGUL SYLLABLE CAEG..HANGUL SYLLABLE CAEH + {0xCC60, 0xCC60, prH2, gcLo}, // HANGUL SYLLABLE CYA + {0xCC61, 0xCC7B, prH3, gcLo}, // [27] HANGUL SYLLABLE CYAG..HANGUL SYLLABLE CYAH + {0xCC7C, 0xCC7C, prH2, gcLo}, // HANGUL SYLLABLE CYAE + {0xCC7D, 0xCC97, prH3, gcLo}, // [27] HANGUL SYLLABLE CYAEG..HANGUL SYLLABLE CYAEH + {0xCC98, 0xCC98, prH2, gcLo}, // HANGUL SYLLABLE CEO + {0xCC99, 0xCCB3, prH3, gcLo}, // [27] HANGUL SYLLABLE CEOG..HANGUL SYLLABLE CEOH + {0xCCB4, 0xCCB4, prH2, gcLo}, // HANGUL SYLLABLE CE + {0xCCB5, 0xCCCF, prH3, gcLo}, // [27] HANGUL SYLLABLE CEG..HANGUL SYLLABLE CEH + {0xCCD0, 0xCCD0, prH2, gcLo}, // HANGUL SYLLABLE CYEO + {0xCCD1, 0xCCEB, prH3, gcLo}, // [27] HANGUL SYLLABLE CYEOG..HANGUL SYLLABLE CYEOH + {0xCCEC, 0xCCEC, prH2, gcLo}, // HANGUL SYLLABLE CYE + {0xCCED, 0xCD07, prH3, gcLo}, // [27] HANGUL SYLLABLE CYEG..HANGUL SYLLABLE CYEH + {0xCD08, 0xCD08, prH2, gcLo}, // HANGUL SYLLABLE CO + {0xCD09, 0xCD23, prH3, gcLo}, // [27] HANGUL SYLLABLE COG..HANGUL SYLLABLE COH + {0xCD24, 0xCD24, prH2, gcLo}, // HANGUL SYLLABLE CWA + {0xCD25, 0xCD3F, prH3, gcLo}, // [27] HANGUL SYLLABLE CWAG..HANGUL SYLLABLE CWAH + {0xCD40, 0xCD40, prH2, gcLo}, // HANGUL SYLLABLE CWAE + {0xCD41, 0xCD5B, prH3, gcLo}, // [27] HANGUL SYLLABLE CWAEG..HANGUL SYLLABLE CWAEH + {0xCD5C, 0xCD5C, prH2, gcLo}, // HANGUL SYLLABLE COE + {0xCD5D, 0xCD77, prH3, gcLo}, // [27] HANGUL SYLLABLE COEG..HANGUL SYLLABLE COEH + {0xCD78, 0xCD78, prH2, gcLo}, // HANGUL SYLLABLE CYO + {0xCD79, 0xCD93, prH3, gcLo}, // [27] HANGUL SYLLABLE CYOG..HANGUL SYLLABLE CYOH + {0xCD94, 0xCD94, prH2, gcLo}, // HANGUL SYLLABLE CU + {0xCD95, 0xCDAF, prH3, gcLo}, // [27] HANGUL SYLLABLE CUG..HANGUL SYLLABLE CUH + {0xCDB0, 0xCDB0, prH2, gcLo}, // HANGUL SYLLABLE CWEO + {0xCDB1, 0xCDCB, prH3, gcLo}, // [27] HANGUL SYLLABLE CWEOG..HANGUL SYLLABLE CWEOH + {0xCDCC, 0xCDCC, prH2, gcLo}, // HANGUL SYLLABLE CWE + {0xCDCD, 0xCDE7, prH3, gcLo}, // [27] HANGUL SYLLABLE CWEG..HANGUL SYLLABLE CWEH + {0xCDE8, 0xCDE8, prH2, gcLo}, // HANGUL SYLLABLE CWI + {0xCDE9, 0xCE03, prH3, gcLo}, // [27] HANGUL SYLLABLE CWIG..HANGUL SYLLABLE CWIH + {0xCE04, 0xCE04, prH2, gcLo}, // HANGUL SYLLABLE CYU + {0xCE05, 0xCE1F, prH3, gcLo}, // [27] HANGUL SYLLABLE CYUG..HANGUL SYLLABLE CYUH + {0xCE20, 0xCE20, prH2, gcLo}, // HANGUL SYLLABLE CEU + {0xCE21, 0xCE3B, prH3, gcLo}, // [27] HANGUL SYLLABLE CEUG..HANGUL SYLLABLE CEUH + {0xCE3C, 0xCE3C, prH2, gcLo}, // HANGUL SYLLABLE CYI + {0xCE3D, 0xCE57, prH3, gcLo}, // [27] HANGUL SYLLABLE CYIG..HANGUL SYLLABLE CYIH + {0xCE58, 0xCE58, prH2, gcLo}, // HANGUL SYLLABLE CI + {0xCE59, 0xCE73, prH3, gcLo}, // [27] HANGUL SYLLABLE CIG..HANGUL SYLLABLE CIH + {0xCE74, 0xCE74, prH2, gcLo}, // HANGUL SYLLABLE KA + {0xCE75, 0xCE8F, prH3, gcLo}, // [27] HANGUL SYLLABLE KAG..HANGUL SYLLABLE KAH + {0xCE90, 0xCE90, prH2, gcLo}, // HANGUL SYLLABLE KAE + {0xCE91, 0xCEAB, prH3, gcLo}, // [27] HANGUL SYLLABLE KAEG..HANGUL SYLLABLE KAEH + {0xCEAC, 0xCEAC, prH2, gcLo}, // HANGUL SYLLABLE KYA + {0xCEAD, 0xCEC7, prH3, gcLo}, // [27] HANGUL SYLLABLE KYAG..HANGUL SYLLABLE KYAH + {0xCEC8, 0xCEC8, prH2, gcLo}, // HANGUL SYLLABLE KYAE + {0xCEC9, 0xCEE3, prH3, gcLo}, // [27] HANGUL SYLLABLE KYAEG..HANGUL SYLLABLE KYAEH + {0xCEE4, 0xCEE4, prH2, gcLo}, // HANGUL SYLLABLE KEO + {0xCEE5, 0xCEFF, prH3, gcLo}, // [27] HANGUL SYLLABLE KEOG..HANGUL SYLLABLE KEOH + {0xCF00, 0xCF00, prH2, gcLo}, // HANGUL SYLLABLE KE + {0xCF01, 0xCF1B, prH3, gcLo}, // [27] HANGUL SYLLABLE KEG..HANGUL SYLLABLE KEH + {0xCF1C, 0xCF1C, prH2, gcLo}, // HANGUL SYLLABLE KYEO + {0xCF1D, 0xCF37, prH3, gcLo}, // [27] HANGUL SYLLABLE KYEOG..HANGUL SYLLABLE KYEOH + {0xCF38, 0xCF38, prH2, gcLo}, // HANGUL SYLLABLE KYE + {0xCF39, 0xCF53, prH3, gcLo}, // [27] HANGUL SYLLABLE KYEG..HANGUL SYLLABLE KYEH + {0xCF54, 0xCF54, prH2, gcLo}, // HANGUL SYLLABLE KO + {0xCF55, 0xCF6F, prH3, gcLo}, // [27] HANGUL SYLLABLE KOG..HANGUL SYLLABLE KOH + {0xCF70, 0xCF70, prH2, gcLo}, // HANGUL SYLLABLE KWA + {0xCF71, 0xCF8B, prH3, gcLo}, // [27] HANGUL SYLLABLE KWAG..HANGUL SYLLABLE KWAH + {0xCF8C, 0xCF8C, prH2, gcLo}, // HANGUL SYLLABLE KWAE + {0xCF8D, 0xCFA7, prH3, gcLo}, // [27] HANGUL SYLLABLE KWAEG..HANGUL SYLLABLE KWAEH + {0xCFA8, 0xCFA8, prH2, gcLo}, // HANGUL SYLLABLE KOE + {0xCFA9, 0xCFC3, prH3, gcLo}, // [27] HANGUL SYLLABLE KOEG..HANGUL SYLLABLE KOEH + {0xCFC4, 0xCFC4, prH2, gcLo}, // HANGUL SYLLABLE KYO + {0xCFC5, 0xCFDF, prH3, gcLo}, // [27] HANGUL SYLLABLE KYOG..HANGUL SYLLABLE KYOH + {0xCFE0, 0xCFE0, prH2, gcLo}, // HANGUL SYLLABLE KU + {0xCFE1, 0xCFFB, prH3, gcLo}, // [27] HANGUL SYLLABLE KUG..HANGUL SYLLABLE KUH + {0xCFFC, 0xCFFC, prH2, gcLo}, // HANGUL SYLLABLE KWEO + {0xCFFD, 0xD017, prH3, gcLo}, // [27] HANGUL SYLLABLE KWEOG..HANGUL SYLLABLE KWEOH + {0xD018, 0xD018, prH2, gcLo}, // HANGUL SYLLABLE KWE + {0xD019, 0xD033, prH3, gcLo}, // [27] HANGUL SYLLABLE KWEG..HANGUL SYLLABLE KWEH + {0xD034, 0xD034, prH2, gcLo}, // HANGUL SYLLABLE KWI + {0xD035, 0xD04F, prH3, gcLo}, // [27] HANGUL SYLLABLE KWIG..HANGUL SYLLABLE KWIH + {0xD050, 0xD050, prH2, gcLo}, // HANGUL SYLLABLE KYU + {0xD051, 0xD06B, prH3, gcLo}, // [27] HANGUL SYLLABLE KYUG..HANGUL SYLLABLE KYUH + {0xD06C, 0xD06C, prH2, gcLo}, // HANGUL SYLLABLE KEU + {0xD06D, 0xD087, prH3, gcLo}, // [27] HANGUL SYLLABLE KEUG..HANGUL SYLLABLE KEUH + {0xD088, 0xD088, prH2, gcLo}, // HANGUL SYLLABLE KYI + {0xD089, 0xD0A3, prH3, gcLo}, // [27] HANGUL SYLLABLE KYIG..HANGUL SYLLABLE KYIH + {0xD0A4, 0xD0A4, prH2, gcLo}, // HANGUL SYLLABLE KI + {0xD0A5, 0xD0BF, prH3, gcLo}, // [27] HANGUL SYLLABLE KIG..HANGUL SYLLABLE KIH + {0xD0C0, 0xD0C0, prH2, gcLo}, // HANGUL SYLLABLE TA + {0xD0C1, 0xD0DB, prH3, gcLo}, // [27] HANGUL SYLLABLE TAG..HANGUL SYLLABLE TAH + {0xD0DC, 0xD0DC, prH2, gcLo}, // HANGUL SYLLABLE TAE + {0xD0DD, 0xD0F7, prH3, gcLo}, // [27] HANGUL SYLLABLE TAEG..HANGUL SYLLABLE TAEH + {0xD0F8, 0xD0F8, prH2, gcLo}, // HANGUL SYLLABLE TYA + {0xD0F9, 0xD113, prH3, gcLo}, // [27] HANGUL SYLLABLE TYAG..HANGUL SYLLABLE TYAH + {0xD114, 0xD114, prH2, gcLo}, // HANGUL SYLLABLE TYAE + {0xD115, 0xD12F, prH3, gcLo}, // [27] HANGUL SYLLABLE TYAEG..HANGUL SYLLABLE TYAEH + {0xD130, 0xD130, prH2, gcLo}, // HANGUL SYLLABLE TEO + {0xD131, 0xD14B, prH3, gcLo}, // [27] HANGUL SYLLABLE TEOG..HANGUL SYLLABLE TEOH + {0xD14C, 0xD14C, prH2, gcLo}, // HANGUL SYLLABLE TE + {0xD14D, 0xD167, prH3, gcLo}, // [27] HANGUL SYLLABLE TEG..HANGUL SYLLABLE TEH + {0xD168, 0xD168, prH2, gcLo}, // HANGUL SYLLABLE TYEO + {0xD169, 0xD183, prH3, gcLo}, // [27] HANGUL SYLLABLE TYEOG..HANGUL SYLLABLE TYEOH + {0xD184, 0xD184, prH2, gcLo}, // HANGUL SYLLABLE TYE + {0xD185, 0xD19F, prH3, gcLo}, // [27] HANGUL SYLLABLE TYEG..HANGUL SYLLABLE TYEH + {0xD1A0, 0xD1A0, prH2, gcLo}, // HANGUL SYLLABLE TO + {0xD1A1, 0xD1BB, prH3, gcLo}, // [27] HANGUL SYLLABLE TOG..HANGUL SYLLABLE TOH + {0xD1BC, 0xD1BC, prH2, gcLo}, // HANGUL SYLLABLE TWA + {0xD1BD, 0xD1D7, prH3, gcLo}, // [27] HANGUL SYLLABLE TWAG..HANGUL SYLLABLE TWAH + {0xD1D8, 0xD1D8, prH2, gcLo}, // HANGUL SYLLABLE TWAE + {0xD1D9, 0xD1F3, prH3, gcLo}, // [27] HANGUL SYLLABLE TWAEG..HANGUL SYLLABLE TWAEH + {0xD1F4, 0xD1F4, prH2, gcLo}, // HANGUL SYLLABLE TOE + {0xD1F5, 0xD20F, prH3, gcLo}, // [27] HANGUL SYLLABLE TOEG..HANGUL SYLLABLE TOEH + {0xD210, 0xD210, prH2, gcLo}, // HANGUL SYLLABLE TYO + {0xD211, 0xD22B, prH3, gcLo}, // [27] HANGUL SYLLABLE TYOG..HANGUL SYLLABLE TYOH + {0xD22C, 0xD22C, prH2, gcLo}, // HANGUL SYLLABLE TU + {0xD22D, 0xD247, prH3, gcLo}, // [27] HANGUL SYLLABLE TUG..HANGUL SYLLABLE TUH + {0xD248, 0xD248, prH2, gcLo}, // HANGUL SYLLABLE TWEO + {0xD249, 0xD263, prH3, gcLo}, // [27] HANGUL SYLLABLE TWEOG..HANGUL SYLLABLE TWEOH + {0xD264, 0xD264, prH2, gcLo}, // HANGUL SYLLABLE TWE + {0xD265, 0xD27F, prH3, gcLo}, // [27] HANGUL SYLLABLE TWEG..HANGUL SYLLABLE TWEH + {0xD280, 0xD280, prH2, gcLo}, // HANGUL SYLLABLE TWI + {0xD281, 0xD29B, prH3, gcLo}, // [27] HANGUL SYLLABLE TWIG..HANGUL SYLLABLE TWIH + {0xD29C, 0xD29C, prH2, gcLo}, // HANGUL SYLLABLE TYU + {0xD29D, 0xD2B7, prH3, gcLo}, // [27] HANGUL SYLLABLE TYUG..HANGUL SYLLABLE TYUH + {0xD2B8, 0xD2B8, prH2, gcLo}, // HANGUL SYLLABLE TEU + {0xD2B9, 0xD2D3, prH3, gcLo}, // [27] HANGUL SYLLABLE TEUG..HANGUL SYLLABLE TEUH + {0xD2D4, 0xD2D4, prH2, gcLo}, // HANGUL SYLLABLE TYI + {0xD2D5, 0xD2EF, prH3, gcLo}, // [27] HANGUL SYLLABLE TYIG..HANGUL SYLLABLE TYIH + {0xD2F0, 0xD2F0, prH2, gcLo}, // HANGUL SYLLABLE TI + {0xD2F1, 0xD30B, prH3, gcLo}, // [27] HANGUL SYLLABLE TIG..HANGUL SYLLABLE TIH + {0xD30C, 0xD30C, prH2, gcLo}, // HANGUL SYLLABLE PA + {0xD30D, 0xD327, prH3, gcLo}, // [27] HANGUL SYLLABLE PAG..HANGUL SYLLABLE PAH + {0xD328, 0xD328, prH2, gcLo}, // HANGUL SYLLABLE PAE + {0xD329, 0xD343, prH3, gcLo}, // [27] HANGUL SYLLABLE PAEG..HANGUL SYLLABLE PAEH + {0xD344, 0xD344, prH2, gcLo}, // HANGUL SYLLABLE PYA + {0xD345, 0xD35F, prH3, gcLo}, // [27] HANGUL SYLLABLE PYAG..HANGUL SYLLABLE PYAH + {0xD360, 0xD360, prH2, gcLo}, // HANGUL SYLLABLE PYAE + {0xD361, 0xD37B, prH3, gcLo}, // [27] HANGUL SYLLABLE PYAEG..HANGUL SYLLABLE PYAEH + {0xD37C, 0xD37C, prH2, gcLo}, // HANGUL SYLLABLE PEO + {0xD37D, 0xD397, prH3, gcLo}, // [27] HANGUL SYLLABLE PEOG..HANGUL SYLLABLE PEOH + {0xD398, 0xD398, prH2, gcLo}, // HANGUL SYLLABLE PE + {0xD399, 0xD3B3, prH3, gcLo}, // [27] HANGUL SYLLABLE PEG..HANGUL SYLLABLE PEH + {0xD3B4, 0xD3B4, prH2, gcLo}, // HANGUL SYLLABLE PYEO + {0xD3B5, 0xD3CF, prH3, gcLo}, // [27] HANGUL SYLLABLE PYEOG..HANGUL SYLLABLE PYEOH + {0xD3D0, 0xD3D0, prH2, gcLo}, // HANGUL SYLLABLE PYE + {0xD3D1, 0xD3EB, prH3, gcLo}, // [27] HANGUL SYLLABLE PYEG..HANGUL SYLLABLE PYEH + {0xD3EC, 0xD3EC, prH2, gcLo}, // HANGUL SYLLABLE PO + {0xD3ED, 0xD407, prH3, gcLo}, // [27] HANGUL SYLLABLE POG..HANGUL SYLLABLE POH + {0xD408, 0xD408, prH2, gcLo}, // HANGUL SYLLABLE PWA + {0xD409, 0xD423, prH3, gcLo}, // [27] HANGUL SYLLABLE PWAG..HANGUL SYLLABLE PWAH + {0xD424, 0xD424, prH2, gcLo}, // HANGUL SYLLABLE PWAE + {0xD425, 0xD43F, prH3, gcLo}, // [27] HANGUL SYLLABLE PWAEG..HANGUL SYLLABLE PWAEH + {0xD440, 0xD440, prH2, gcLo}, // HANGUL SYLLABLE POE + {0xD441, 0xD45B, prH3, gcLo}, // [27] HANGUL SYLLABLE POEG..HANGUL SYLLABLE POEH + {0xD45C, 0xD45C, prH2, gcLo}, // HANGUL SYLLABLE PYO + {0xD45D, 0xD477, prH3, gcLo}, // [27] HANGUL SYLLABLE PYOG..HANGUL SYLLABLE PYOH + {0xD478, 0xD478, prH2, gcLo}, // HANGUL SYLLABLE PU + {0xD479, 0xD493, prH3, gcLo}, // [27] HANGUL SYLLABLE PUG..HANGUL SYLLABLE PUH + {0xD494, 0xD494, prH2, gcLo}, // HANGUL SYLLABLE PWEO + {0xD495, 0xD4AF, prH3, gcLo}, // [27] HANGUL SYLLABLE PWEOG..HANGUL SYLLABLE PWEOH + {0xD4B0, 0xD4B0, prH2, gcLo}, // HANGUL SYLLABLE PWE + {0xD4B1, 0xD4CB, prH3, gcLo}, // [27] HANGUL SYLLABLE PWEG..HANGUL SYLLABLE PWEH + {0xD4CC, 0xD4CC, prH2, gcLo}, // HANGUL SYLLABLE PWI + {0xD4CD, 0xD4E7, prH3, gcLo}, // [27] HANGUL SYLLABLE PWIG..HANGUL SYLLABLE PWIH + {0xD4E8, 0xD4E8, prH2, gcLo}, // HANGUL SYLLABLE PYU + {0xD4E9, 0xD503, prH3, gcLo}, // [27] HANGUL SYLLABLE PYUG..HANGUL SYLLABLE PYUH + {0xD504, 0xD504, prH2, gcLo}, // HANGUL SYLLABLE PEU + {0xD505, 0xD51F, prH3, gcLo}, // [27] HANGUL SYLLABLE PEUG..HANGUL SYLLABLE PEUH + {0xD520, 0xD520, prH2, gcLo}, // HANGUL SYLLABLE PYI + {0xD521, 0xD53B, prH3, gcLo}, // [27] HANGUL SYLLABLE PYIG..HANGUL SYLLABLE PYIH + {0xD53C, 0xD53C, prH2, gcLo}, // HANGUL SYLLABLE PI + {0xD53D, 0xD557, prH3, gcLo}, // [27] HANGUL SYLLABLE PIG..HANGUL SYLLABLE PIH + {0xD558, 0xD558, prH2, gcLo}, // HANGUL SYLLABLE HA + {0xD559, 0xD573, prH3, gcLo}, // [27] HANGUL SYLLABLE HAG..HANGUL SYLLABLE HAH + {0xD574, 0xD574, prH2, gcLo}, // HANGUL SYLLABLE HAE + {0xD575, 0xD58F, prH3, gcLo}, // [27] HANGUL SYLLABLE HAEG..HANGUL SYLLABLE HAEH + {0xD590, 0xD590, prH2, gcLo}, // HANGUL SYLLABLE HYA + {0xD591, 0xD5AB, prH3, gcLo}, // [27] HANGUL SYLLABLE HYAG..HANGUL SYLLABLE HYAH + {0xD5AC, 0xD5AC, prH2, gcLo}, // HANGUL SYLLABLE HYAE + {0xD5AD, 0xD5C7, prH3, gcLo}, // [27] HANGUL SYLLABLE HYAEG..HANGUL SYLLABLE HYAEH + {0xD5C8, 0xD5C8, prH2, gcLo}, // HANGUL SYLLABLE HEO + {0xD5C9, 0xD5E3, prH3, gcLo}, // [27] HANGUL SYLLABLE HEOG..HANGUL SYLLABLE HEOH + {0xD5E4, 0xD5E4, prH2, gcLo}, // HANGUL SYLLABLE HE + {0xD5E5, 0xD5FF, prH3, gcLo}, // [27] HANGUL SYLLABLE HEG..HANGUL SYLLABLE HEH + {0xD600, 0xD600, prH2, gcLo}, // HANGUL SYLLABLE HYEO + {0xD601, 0xD61B, prH3, gcLo}, // [27] HANGUL SYLLABLE HYEOG..HANGUL SYLLABLE HYEOH + {0xD61C, 0xD61C, prH2, gcLo}, // HANGUL SYLLABLE HYE + {0xD61D, 0xD637, prH3, gcLo}, // [27] HANGUL SYLLABLE HYEG..HANGUL SYLLABLE HYEH + {0xD638, 0xD638, prH2, gcLo}, // HANGUL SYLLABLE HO + {0xD639, 0xD653, prH3, gcLo}, // [27] HANGUL SYLLABLE HOG..HANGUL SYLLABLE HOH + {0xD654, 0xD654, prH2, gcLo}, // HANGUL SYLLABLE HWA + {0xD655, 0xD66F, prH3, gcLo}, // [27] HANGUL SYLLABLE HWAG..HANGUL SYLLABLE HWAH + {0xD670, 0xD670, prH2, gcLo}, // HANGUL SYLLABLE HWAE + {0xD671, 0xD68B, prH3, gcLo}, // [27] HANGUL SYLLABLE HWAEG..HANGUL SYLLABLE HWAEH + {0xD68C, 0xD68C, prH2, gcLo}, // HANGUL SYLLABLE HOE + {0xD68D, 0xD6A7, prH3, gcLo}, // [27] HANGUL SYLLABLE HOEG..HANGUL SYLLABLE HOEH + {0xD6A8, 0xD6A8, prH2, gcLo}, // HANGUL SYLLABLE HYO + {0xD6A9, 0xD6C3, prH3, gcLo}, // [27] HANGUL SYLLABLE HYOG..HANGUL SYLLABLE HYOH + {0xD6C4, 0xD6C4, prH2, gcLo}, // HANGUL SYLLABLE HU + {0xD6C5, 0xD6DF, prH3, gcLo}, // [27] HANGUL SYLLABLE HUG..HANGUL SYLLABLE HUH + {0xD6E0, 0xD6E0, prH2, gcLo}, // HANGUL SYLLABLE HWEO + {0xD6E1, 0xD6FB, prH3, gcLo}, // [27] HANGUL SYLLABLE HWEOG..HANGUL SYLLABLE HWEOH + {0xD6FC, 0xD6FC, prH2, gcLo}, // HANGUL SYLLABLE HWE + {0xD6FD, 0xD717, prH3, gcLo}, // [27] HANGUL SYLLABLE HWEG..HANGUL SYLLABLE HWEH + {0xD718, 0xD718, prH2, gcLo}, // HANGUL SYLLABLE HWI + {0xD719, 0xD733, prH3, gcLo}, // [27] HANGUL SYLLABLE HWIG..HANGUL SYLLABLE HWIH + {0xD734, 0xD734, prH2, gcLo}, // HANGUL SYLLABLE HYU + {0xD735, 0xD74F, prH3, gcLo}, // [27] HANGUL SYLLABLE HYUG..HANGUL SYLLABLE HYUH + {0xD750, 0xD750, prH2, gcLo}, // HANGUL SYLLABLE HEU + {0xD751, 0xD76B, prH3, gcLo}, // [27] HANGUL SYLLABLE HEUG..HANGUL SYLLABLE HEUH + {0xD76C, 0xD76C, prH2, gcLo}, // HANGUL SYLLABLE HYI + {0xD76D, 0xD787, prH3, gcLo}, // [27] HANGUL SYLLABLE HYIG..HANGUL SYLLABLE HYIH + {0xD788, 0xD788, prH2, gcLo}, // HANGUL SYLLABLE HI + {0xD789, 0xD7A3, prH3, gcLo}, // [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH + {0xD7B0, 0xD7C6, prJV, gcLo}, // [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E + {0xD7CB, 0xD7FB, prJT, gcLo}, // [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH + {0xD800, 0xDB7F, prSG, gcCs}, // [896] .. + {0xDB80, 0xDBFF, prSG, gcCs}, // [128] .. + {0xDC00, 0xDFFF, prSG, gcCs}, // [1024] .. + {0xE000, 0xF8FF, prXX, gcCo}, // [6400] .. + {0xF900, 0xFA6D, prID, gcLo}, // [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D + {0xFA6E, 0xFA6F, prID, gcCn}, // [2] .. + {0xFA70, 0xFAD9, prID, gcLo}, // [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 + {0xFADA, 0xFAFF, prID, gcCn}, // [38] .. + {0xFB00, 0xFB06, prAL, gcLl}, // [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST + {0xFB13, 0xFB17, prAL, gcLl}, // [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH + {0xFB1D, 0xFB1D, prHL, gcLo}, // HEBREW LETTER YOD WITH HIRIQ + {0xFB1E, 0xFB1E, prCM, gcMn}, // HEBREW POINT JUDEO-SPANISH VARIKA + {0xFB1F, 0xFB28, prHL, gcLo}, // [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV + {0xFB29, 0xFB29, prAL, gcSm}, // HEBREW LETTER ALTERNATIVE PLUS SIGN + {0xFB2A, 0xFB36, prHL, gcLo}, // [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH + {0xFB38, 0xFB3C, prHL, gcLo}, // [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH + {0xFB3E, 0xFB3E, prHL, gcLo}, // HEBREW LETTER MEM WITH DAGESH + {0xFB40, 0xFB41, prHL, gcLo}, // [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH + {0xFB43, 0xFB44, prHL, gcLo}, // [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH + {0xFB46, 0xFB4F, prHL, gcLo}, // [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATURE ALEF LAMED + {0xFB50, 0xFBB1, prAL, gcLo}, // [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM + {0xFBB2, 0xFBC2, prAL, gcSk}, // [17] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL WASLA ABOVE + {0xFBD3, 0xFD3D, prAL, gcLo}, // [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM + {0xFD3E, 0xFD3E, prCL, gcPe}, // ORNATE LEFT PARENTHESIS + {0xFD3F, 0xFD3F, prOP, gcPs}, // ORNATE RIGHT PARENTHESIS + {0xFD40, 0xFD4F, prAL, gcSo}, // [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH + {0xFD50, 0xFD8F, prAL, gcLo}, // [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM + {0xFD92, 0xFDC7, prAL, gcLo}, // [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM + {0xFDCF, 0xFDCF, prAL, gcSo}, // ARABIC LIGATURE SALAAMUHU ALAYNAA + {0xFDF0, 0xFDFB, prAL, gcLo}, // [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU + {0xFDFC, 0xFDFC, prPO, gcSc}, // RIAL SIGN + {0xFDFD, 0xFDFF, prAL, gcSo}, // [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL + {0xFE00, 0xFE0F, prCM, gcMn}, // [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 + {0xFE10, 0xFE10, prIS, gcPo}, // PRESENTATION FORM FOR VERTICAL COMMA + {0xFE11, 0xFE12, prCL, gcPo}, // [2] PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC COMMA..PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC FULL STOP + {0xFE13, 0xFE14, prIS, gcPo}, // [2] PRESENTATION FORM FOR VERTICAL COLON..PRESENTATION FORM FOR VERTICAL SEMICOLON + {0xFE15, 0xFE16, prEX, gcPo}, // [2] PRESENTATION FORM FOR VERTICAL EXCLAMATION MARK..PRESENTATION FORM FOR VERTICAL QUESTION MARK + {0xFE17, 0xFE17, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET + {0xFE18, 0xFE18, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET + {0xFE19, 0xFE19, prIN, gcPo}, // PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS + {0xFE20, 0xFE2F, prCM, gcMn}, // [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + {0xFE30, 0xFE30, prID, gcPo}, // PRESENTATION FORM FOR VERTICAL TWO DOT LEADER + {0xFE31, 0xFE32, prID, gcPd}, // [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH + {0xFE33, 0xFE34, prID, gcPc}, // [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE + {0xFE35, 0xFE35, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS + {0xFE36, 0xFE36, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS + {0xFE37, 0xFE37, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET + {0xFE38, 0xFE38, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET + {0xFE39, 0xFE39, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET + {0xFE3A, 0xFE3A, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET + {0xFE3B, 0xFE3B, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET + {0xFE3C, 0xFE3C, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET + {0xFE3D, 0xFE3D, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET + {0xFE3E, 0xFE3E, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET + {0xFE3F, 0xFE3F, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET + {0xFE40, 0xFE40, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET + {0xFE41, 0xFE41, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET + {0xFE42, 0xFE42, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET + {0xFE43, 0xFE43, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET + {0xFE44, 0xFE44, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET + {0xFE45, 0xFE46, prID, gcPo}, // [2] SESAME DOT..WHITE SESAME DOT + {0xFE47, 0xFE47, prOP, gcPs}, // PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET + {0xFE48, 0xFE48, prCL, gcPe}, // PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET + {0xFE49, 0xFE4C, prID, gcPo}, // [4] DASHED OVERLINE..DOUBLE WAVY OVERLINE + {0xFE4D, 0xFE4F, prID, gcPc}, // [3] DASHED LOW LINE..WAVY LOW LINE + {0xFE50, 0xFE50, prCL, gcPo}, // SMALL COMMA + {0xFE51, 0xFE51, prID, gcPo}, // SMALL IDEOGRAPHIC COMMA + {0xFE52, 0xFE52, prCL, gcPo}, // SMALL FULL STOP + {0xFE54, 0xFE55, prNS, gcPo}, // [2] SMALL SEMICOLON..SMALL COLON + {0xFE56, 0xFE57, prEX, gcPo}, // [2] SMALL QUESTION MARK..SMALL EXCLAMATION MARK + {0xFE58, 0xFE58, prID, gcPd}, // SMALL EM DASH + {0xFE59, 0xFE59, prOP, gcPs}, // SMALL LEFT PARENTHESIS + {0xFE5A, 0xFE5A, prCL, gcPe}, // SMALL RIGHT PARENTHESIS + {0xFE5B, 0xFE5B, prOP, gcPs}, // SMALL LEFT CURLY BRACKET + {0xFE5C, 0xFE5C, prCL, gcPe}, // SMALL RIGHT CURLY BRACKET + {0xFE5D, 0xFE5D, prOP, gcPs}, // SMALL LEFT TORTOISE SHELL BRACKET + {0xFE5E, 0xFE5E, prCL, gcPe}, // SMALL RIGHT TORTOISE SHELL BRACKET + {0xFE5F, 0xFE61, prID, gcPo}, // [3] SMALL NUMBER SIGN..SMALL ASTERISK + {0xFE62, 0xFE62, prID, gcSm}, // SMALL PLUS SIGN + {0xFE63, 0xFE63, prID, gcPd}, // SMALL HYPHEN-MINUS + {0xFE64, 0xFE66, prID, gcSm}, // [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN + {0xFE68, 0xFE68, prID, gcPo}, // SMALL REVERSE SOLIDUS + {0xFE69, 0xFE69, prPR, gcSc}, // SMALL DOLLAR SIGN + {0xFE6A, 0xFE6A, prPO, gcPo}, // SMALL PERCENT SIGN + {0xFE6B, 0xFE6B, prID, gcPo}, // SMALL COMMERCIAL AT + {0xFE70, 0xFE74, prAL, gcLo}, // [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM + {0xFE76, 0xFEFC, prAL, gcLo}, // [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM + {0xFEFF, 0xFEFF, prWJ, gcCf}, // ZERO WIDTH NO-BREAK SPACE + {0xFF01, 0xFF01, prEX, gcPo}, // FULLWIDTH EXCLAMATION MARK + {0xFF02, 0xFF03, prID, gcPo}, // [2] FULLWIDTH QUOTATION MARK..FULLWIDTH NUMBER SIGN + {0xFF04, 0xFF04, prPR, gcSc}, // FULLWIDTH DOLLAR SIGN + {0xFF05, 0xFF05, prPO, gcPo}, // FULLWIDTH PERCENT SIGN + {0xFF06, 0xFF07, prID, gcPo}, // [2] FULLWIDTH AMPERSAND..FULLWIDTH APOSTROPHE + {0xFF08, 0xFF08, prOP, gcPs}, // FULLWIDTH LEFT PARENTHESIS + {0xFF09, 0xFF09, prCL, gcPe}, // FULLWIDTH RIGHT PARENTHESIS + {0xFF0A, 0xFF0A, prID, gcPo}, // FULLWIDTH ASTERISK + {0xFF0B, 0xFF0B, prID, gcSm}, // FULLWIDTH PLUS SIGN + {0xFF0C, 0xFF0C, prCL, gcPo}, // FULLWIDTH COMMA + {0xFF0D, 0xFF0D, prID, gcPd}, // FULLWIDTH HYPHEN-MINUS + {0xFF0E, 0xFF0E, prCL, gcPo}, // FULLWIDTH FULL STOP + {0xFF0F, 0xFF0F, prID, gcPo}, // FULLWIDTH SOLIDUS + {0xFF10, 0xFF19, prID, gcNd}, // [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE + {0xFF1A, 0xFF1B, prNS, gcPo}, // [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON + {0xFF1C, 0xFF1E, prID, gcSm}, // [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN + {0xFF1F, 0xFF1F, prEX, gcPo}, // FULLWIDTH QUESTION MARK + {0xFF20, 0xFF20, prID, gcPo}, // FULLWIDTH COMMERCIAL AT + {0xFF21, 0xFF3A, prID, gcLu}, // [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z + {0xFF3B, 0xFF3B, prOP, gcPs}, // FULLWIDTH LEFT SQUARE BRACKET + {0xFF3C, 0xFF3C, prID, gcPo}, // FULLWIDTH REVERSE SOLIDUS + {0xFF3D, 0xFF3D, prCL, gcPe}, // FULLWIDTH RIGHT SQUARE BRACKET + {0xFF3E, 0xFF3E, prID, gcSk}, // FULLWIDTH CIRCUMFLEX ACCENT + {0xFF3F, 0xFF3F, prID, gcPc}, // FULLWIDTH LOW LINE + {0xFF40, 0xFF40, prID, gcSk}, // FULLWIDTH GRAVE ACCENT + {0xFF41, 0xFF5A, prID, gcLl}, // [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z + {0xFF5B, 0xFF5B, prOP, gcPs}, // FULLWIDTH LEFT CURLY BRACKET + {0xFF5C, 0xFF5C, prID, gcSm}, // FULLWIDTH VERTICAL LINE + {0xFF5D, 0xFF5D, prCL, gcPe}, // FULLWIDTH RIGHT CURLY BRACKET + {0xFF5E, 0xFF5E, prID, gcSm}, // FULLWIDTH TILDE + {0xFF5F, 0xFF5F, prOP, gcPs}, // FULLWIDTH LEFT WHITE PARENTHESIS + {0xFF60, 0xFF60, prCL, gcPe}, // FULLWIDTH RIGHT WHITE PARENTHESIS + {0xFF61, 0xFF61, prCL, gcPo}, // HALFWIDTH IDEOGRAPHIC FULL STOP + {0xFF62, 0xFF62, prOP, gcPs}, // HALFWIDTH LEFT CORNER BRACKET + {0xFF63, 0xFF63, prCL, gcPe}, // HALFWIDTH RIGHT CORNER BRACKET + {0xFF64, 0xFF64, prCL, gcPo}, // HALFWIDTH IDEOGRAPHIC COMMA + {0xFF65, 0xFF65, prNS, gcPo}, // HALFWIDTH KATAKANA MIDDLE DOT + {0xFF66, 0xFF66, prID, gcLo}, // HALFWIDTH KATAKANA LETTER WO + {0xFF67, 0xFF6F, prCJ, gcLo}, // [9] HALFWIDTH KATAKANA LETTER SMALL A..HALFWIDTH KATAKANA LETTER SMALL TU + {0xFF70, 0xFF70, prCJ, gcLm}, // HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK + {0xFF71, 0xFF9D, prID, gcLo}, // [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N + {0xFF9E, 0xFF9F, prNS, gcLm}, // [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK + {0xFFA0, 0xFFBE, prID, gcLo}, // [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH + {0xFFC2, 0xFFC7, prID, gcLo}, // [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E + {0xFFCA, 0xFFCF, prID, gcLo}, // [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE + {0xFFD2, 0xFFD7, prID, gcLo}, // [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU + {0xFFDA, 0xFFDC, prID, gcLo}, // [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I + {0xFFE0, 0xFFE0, prPO, gcSc}, // FULLWIDTH CENT SIGN + {0xFFE1, 0xFFE1, prPR, gcSc}, // FULLWIDTH POUND SIGN + {0xFFE2, 0xFFE2, prID, gcSm}, // FULLWIDTH NOT SIGN + {0xFFE3, 0xFFE3, prID, gcSk}, // FULLWIDTH MACRON + {0xFFE4, 0xFFE4, prID, gcSo}, // FULLWIDTH BROKEN BAR + {0xFFE5, 0xFFE6, prPR, gcSc}, // [2] FULLWIDTH YEN SIGN..FULLWIDTH WON SIGN + {0xFFE8, 0xFFE8, prAL, gcSo}, // HALFWIDTH FORMS LIGHT VERTICAL + {0xFFE9, 0xFFEC, prAL, gcSm}, // [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW + {0xFFED, 0xFFEE, prAL, gcSo}, // [2] HALFWIDTH BLACK SQUARE..HALFWIDTH WHITE CIRCLE + {0xFFF9, 0xFFFB, prCM, gcCf}, // [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR + {0xFFFC, 0xFFFC, prCB, gcSo}, // OBJECT REPLACEMENT CHARACTER + {0xFFFD, 0xFFFD, prAI, gcSo}, // REPLACEMENT CHARACTER + {0x10000, 0x1000B, prAL, gcLo}, // [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE + {0x1000D, 0x10026, prAL, gcLo}, // [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO + {0x10028, 0x1003A, prAL, gcLo}, // [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO + {0x1003C, 0x1003D, prAL, gcLo}, // [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE + {0x1003F, 0x1004D, prAL, gcLo}, // [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO + {0x10050, 0x1005D, prAL, gcLo}, // [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 + {0x10080, 0x100FA, prAL, gcLo}, // [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 + {0x10100, 0x10102, prBA, gcPo}, // [3] AEGEAN WORD SEPARATOR LINE..AEGEAN CHECK MARK + {0x10107, 0x10133, prAL, gcNo}, // [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND + {0x10137, 0x1013F, prAL, gcSo}, // [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT + {0x10140, 0x10174, prAL, gcNl}, // [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS + {0x10175, 0x10178, prAL, gcNo}, // [4] GREEK ONE HALF SIGN..GREEK THREE QUARTERS SIGN + {0x10179, 0x10189, prAL, gcSo}, // [17] GREEK YEAR SIGN..GREEK TRYBLION BASE SIGN + {0x1018A, 0x1018B, prAL, gcNo}, // [2] GREEK ZERO SIGN..GREEK ONE QUARTER SIGN + {0x1018C, 0x1018E, prAL, gcSo}, // [3] GREEK SINUSOID SIGN..NOMISMA SIGN + {0x10190, 0x1019C, prAL, gcSo}, // [13] ROMAN SEXTANS SIGN..ASCIA SYMBOL + {0x101A0, 0x101A0, prAL, gcSo}, // GREEK SYMBOL TAU RHO + {0x101D0, 0x101FC, prAL, gcSo}, // [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND + {0x101FD, 0x101FD, prCM, gcMn}, // PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE + {0x10280, 0x1029C, prAL, gcLo}, // [29] LYCIAN LETTER A..LYCIAN LETTER X + {0x102A0, 0x102D0, prAL, gcLo}, // [49] CARIAN LETTER A..CARIAN LETTER UUU3 + {0x102E0, 0x102E0, prCM, gcMn}, // COPTIC EPACT THOUSANDS MARK + {0x102E1, 0x102FB, prAL, gcNo}, // [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED + {0x10300, 0x1031F, prAL, gcLo}, // [32] OLD ITALIC LETTER A..OLD ITALIC LETTER ESS + {0x10320, 0x10323, prAL, gcNo}, // [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY + {0x1032D, 0x1032F, prAL, gcLo}, // [3] OLD ITALIC LETTER YE..OLD ITALIC LETTER SOUTHERN TSE + {0x10330, 0x10340, prAL, gcLo}, // [17] GOTHIC LETTER AHSA..GOTHIC LETTER PAIRTHRA + {0x10341, 0x10341, prAL, gcNl}, // GOTHIC LETTER NINETY + {0x10342, 0x10349, prAL, gcLo}, // [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL + {0x1034A, 0x1034A, prAL, gcNl}, // GOTHIC LETTER NINE HUNDRED + {0x10350, 0x10375, prAL, gcLo}, // [38] OLD PERMIC LETTER AN..OLD PERMIC LETTER IA + {0x10376, 0x1037A, prCM, gcMn}, // [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII + {0x10380, 0x1039D, prAL, gcLo}, // [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU + {0x1039F, 0x1039F, prBA, gcPo}, // UGARITIC WORD DIVIDER + {0x103A0, 0x103C3, prAL, gcLo}, // [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA + {0x103C8, 0x103CF, prAL, gcLo}, // [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH + {0x103D0, 0x103D0, prBA, gcPo}, // OLD PERSIAN WORD DIVIDER + {0x103D1, 0x103D5, prAL, gcNl}, // [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED + {0x10400, 0x1044F, prAL, gcLC}, // [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW + {0x10450, 0x1047F, prAL, gcLo}, // [48] SHAVIAN LETTER PEEP..SHAVIAN LETTER YEW + {0x10480, 0x1049D, prAL, gcLo}, // [30] OSMANYA LETTER ALEF..OSMANYA LETTER OO + {0x104A0, 0x104A9, prNU, gcNd}, // [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE + {0x104B0, 0x104D3, prAL, gcLu}, // [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA + {0x104D8, 0x104FB, prAL, gcLl}, // [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA + {0x10500, 0x10527, prAL, gcLo}, // [40] ELBASAN LETTER A..ELBASAN LETTER KHE + {0x10530, 0x10563, prAL, gcLo}, // [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW + {0x1056F, 0x1056F, prAL, gcPo}, // CAUCASIAN ALBANIAN CITATION MARK + {0x10570, 0x1057A, prAL, gcLu}, // [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA + {0x1057C, 0x1058A, prAL, gcLu}, // [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE + {0x1058C, 0x10592, prAL, gcLu}, // [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE + {0x10594, 0x10595, prAL, gcLu}, // [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE + {0x10597, 0x105A1, prAL, gcLl}, // [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA + {0x105A3, 0x105B1, prAL, gcLl}, // [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE + {0x105B3, 0x105B9, prAL, gcLl}, // [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE + {0x105BB, 0x105BC, prAL, gcLl}, // [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE + {0x10600, 0x10736, prAL, gcLo}, // [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 + {0x10740, 0x10755, prAL, gcLo}, // [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE + {0x10760, 0x10767, prAL, gcLo}, // [8] LINEAR A SIGN A800..LINEAR A SIGN A807 + {0x10780, 0x10785, prAL, gcLm}, // [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK + {0x10787, 0x107B0, prAL, gcLm}, // [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK + {0x107B2, 0x107BA, prAL, gcLm}, // [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL + {0x10800, 0x10805, prAL, gcLo}, // [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA + {0x10808, 0x10808, prAL, gcLo}, // CYPRIOT SYLLABLE JO + {0x1080A, 0x10835, prAL, gcLo}, // [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO + {0x10837, 0x10838, prAL, gcLo}, // [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE + {0x1083C, 0x1083C, prAL, gcLo}, // CYPRIOT SYLLABLE ZA + {0x1083F, 0x1083F, prAL, gcLo}, // CYPRIOT SYLLABLE ZO + {0x10840, 0x10855, prAL, gcLo}, // [22] IMPERIAL ARAMAIC LETTER ALEPH..IMPERIAL ARAMAIC LETTER TAW + {0x10857, 0x10857, prBA, gcPo}, // IMPERIAL ARAMAIC SECTION SIGN + {0x10858, 0x1085F, prAL, gcNo}, // [8] IMPERIAL ARAMAIC NUMBER ONE..IMPERIAL ARAMAIC NUMBER TEN THOUSAND + {0x10860, 0x10876, prAL, gcLo}, // [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW + {0x10877, 0x10878, prAL, gcSo}, // [2] PALMYRENE LEFT-POINTING FLEURON..PALMYRENE RIGHT-POINTING FLEURON + {0x10879, 0x1087F, prAL, gcNo}, // [7] PALMYRENE NUMBER ONE..PALMYRENE NUMBER TWENTY + {0x10880, 0x1089E, prAL, gcLo}, // [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW + {0x108A7, 0x108AF, prAL, gcNo}, // [9] NABATAEAN NUMBER ONE..NABATAEAN NUMBER ONE HUNDRED + {0x108E0, 0x108F2, prAL, gcLo}, // [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH + {0x108F4, 0x108F5, prAL, gcLo}, // [2] HATRAN LETTER SHIN..HATRAN LETTER TAW + {0x108FB, 0x108FF, prAL, gcNo}, // [5] HATRAN NUMBER ONE..HATRAN NUMBER ONE HUNDRED + {0x10900, 0x10915, prAL, gcLo}, // [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU + {0x10916, 0x1091B, prAL, gcNo}, // [6] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER THREE + {0x1091F, 0x1091F, prBA, gcPo}, // PHOENICIAN WORD SEPARATOR + {0x10920, 0x10939, prAL, gcLo}, // [26] LYDIAN LETTER A..LYDIAN LETTER C + {0x1093F, 0x1093F, prAL, gcPo}, // LYDIAN TRIANGULAR MARK + {0x10980, 0x1099F, prAL, gcLo}, // [32] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC HIEROGLYPHIC SYMBOL VIDJ-2 + {0x109A0, 0x109B7, prAL, gcLo}, // [24] MEROITIC CURSIVE LETTER A..MEROITIC CURSIVE LETTER DA + {0x109BC, 0x109BD, prAL, gcNo}, // [2] MEROITIC CURSIVE FRACTION ELEVEN TWELFTHS..MEROITIC CURSIVE FRACTION ONE HALF + {0x109BE, 0x109BF, prAL, gcLo}, // [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN + {0x109C0, 0x109CF, prAL, gcNo}, // [16] MEROITIC CURSIVE NUMBER ONE..MEROITIC CURSIVE NUMBER SEVENTY + {0x109D2, 0x109FF, prAL, gcNo}, // [46] MEROITIC CURSIVE NUMBER ONE HUNDRED..MEROITIC CURSIVE FRACTION TEN TWELFTHS + {0x10A00, 0x10A00, prAL, gcLo}, // KHAROSHTHI LETTER A + {0x10A01, 0x10A03, prCM, gcMn}, // [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R + {0x10A05, 0x10A06, prCM, gcMn}, // [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O + {0x10A0C, 0x10A0F, prCM, gcMn}, // [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA + {0x10A10, 0x10A13, prAL, gcLo}, // [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA + {0x10A15, 0x10A17, prAL, gcLo}, // [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA + {0x10A19, 0x10A35, prAL, gcLo}, // [29] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER VHA + {0x10A38, 0x10A3A, prCM, gcMn}, // [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW + {0x10A3F, 0x10A3F, prCM, gcMn}, // KHAROSHTHI VIRAMA + {0x10A40, 0x10A48, prAL, gcNo}, // [9] KHAROSHTHI DIGIT ONE..KHAROSHTHI FRACTION ONE HALF + {0x10A50, 0x10A57, prBA, gcPo}, // [8] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION DOUBLE DANDA + {0x10A58, 0x10A58, prAL, gcPo}, // KHAROSHTHI PUNCTUATION LINES + {0x10A60, 0x10A7C, prAL, gcLo}, // [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH + {0x10A7D, 0x10A7E, prAL, gcNo}, // [2] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMBER FIFTY + {0x10A7F, 0x10A7F, prAL, gcPo}, // OLD SOUTH ARABIAN NUMERIC INDICATOR + {0x10A80, 0x10A9C, prAL, gcLo}, // [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH + {0x10A9D, 0x10A9F, prAL, gcNo}, // [3] OLD NORTH ARABIAN NUMBER ONE..OLD NORTH ARABIAN NUMBER TWENTY + {0x10AC0, 0x10AC7, prAL, gcLo}, // [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW + {0x10AC8, 0x10AC8, prAL, gcSo}, // MANICHAEAN SIGN UD + {0x10AC9, 0x10AE4, prAL, gcLo}, // [28] MANICHAEAN LETTER ZAYIN..MANICHAEAN LETTER TAW + {0x10AE5, 0x10AE6, prCM, gcMn}, // [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW + {0x10AEB, 0x10AEF, prAL, gcNo}, // [5] MANICHAEAN NUMBER ONE..MANICHAEAN NUMBER ONE HUNDRED + {0x10AF0, 0x10AF5, prBA, gcPo}, // [6] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION TWO DOTS + {0x10AF6, 0x10AF6, prIN, gcPo}, // MANICHAEAN PUNCTUATION LINE FILLER + {0x10B00, 0x10B35, prAL, gcLo}, // [54] AVESTAN LETTER A..AVESTAN LETTER HE + {0x10B39, 0x10B3F, prBA, gcPo}, // [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION + {0x10B40, 0x10B55, prAL, gcLo}, // [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW + {0x10B58, 0x10B5F, prAL, gcNo}, // [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND + {0x10B60, 0x10B72, prAL, gcLo}, // [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW + {0x10B78, 0x10B7F, prAL, gcNo}, // [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND + {0x10B80, 0x10B91, prAL, gcLo}, // [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW + {0x10B99, 0x10B9C, prAL, gcPo}, // [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT + {0x10BA9, 0x10BAF, prAL, gcNo}, // [7] PSALTER PAHLAVI NUMBER ONE..PSALTER PAHLAVI NUMBER ONE HUNDRED + {0x10C00, 0x10C48, prAL, gcLo}, // [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH + {0x10C80, 0x10CB2, prAL, gcLu}, // [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US + {0x10CC0, 0x10CF2, prAL, gcLl}, // [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US + {0x10CFA, 0x10CFF, prAL, gcNo}, // [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND + {0x10D00, 0x10D23, prAL, gcLo}, // [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA + {0x10D24, 0x10D27, prCM, gcMn}, // [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI + {0x10D30, 0x10D39, prNU, gcNd}, // [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE + {0x10E60, 0x10E7E, prAL, gcNo}, // [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS + {0x10E80, 0x10EA9, prAL, gcLo}, // [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET + {0x10EAB, 0x10EAC, prCM, gcMn}, // [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK + {0x10EAD, 0x10EAD, prBA, gcPd}, // YEZIDI HYPHENATION MARK + {0x10EB0, 0x10EB1, prAL, gcLo}, // [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE + {0x10EFD, 0x10EFF, prCM, gcMn}, // [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA + {0x10F00, 0x10F1C, prAL, gcLo}, // [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL + {0x10F1D, 0x10F26, prAL, gcNo}, // [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF + {0x10F27, 0x10F27, prAL, gcLo}, // OLD SOGDIAN LIGATURE AYIN-DALETH + {0x10F30, 0x10F45, prAL, gcLo}, // [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN + {0x10F46, 0x10F50, prCM, gcMn}, // [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW + {0x10F51, 0x10F54, prAL, gcNo}, // [4] SOGDIAN NUMBER ONE..SOGDIAN NUMBER ONE HUNDRED + {0x10F55, 0x10F59, prAL, gcPo}, // [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT + {0x10F70, 0x10F81, prAL, gcLo}, // [18] OLD UYGHUR LETTER ALEPH..OLD UYGHUR LETTER LESH + {0x10F82, 0x10F85, prCM, gcMn}, // [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW + {0x10F86, 0x10F89, prAL, gcPo}, // [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS + {0x10FB0, 0x10FC4, prAL, gcLo}, // [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW + {0x10FC5, 0x10FCB, prAL, gcNo}, // [7] CHORASMIAN NUMBER ONE..CHORASMIAN NUMBER ONE HUNDRED + {0x10FE0, 0x10FF6, prAL, gcLo}, // [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH + {0x11000, 0x11000, prCM, gcMc}, // BRAHMI SIGN CANDRABINDU + {0x11001, 0x11001, prCM, gcMn}, // BRAHMI SIGN ANUSVARA + {0x11002, 0x11002, prCM, gcMc}, // BRAHMI SIGN VISARGA + {0x11003, 0x11037, prAL, gcLo}, // [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI LETTER OLD TAMIL NNNA + {0x11038, 0x11046, prCM, gcMn}, // [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA + {0x11047, 0x11048, prBA, gcPo}, // [2] BRAHMI DANDA..BRAHMI DOUBLE DANDA + {0x11049, 0x1104D, prAL, gcPo}, // [5] BRAHMI PUNCTUATION DOT..BRAHMI PUNCTUATION LOTUS + {0x11052, 0x11065, prAL, gcNo}, // [20] BRAHMI NUMBER ONE..BRAHMI NUMBER ONE THOUSAND + {0x11066, 0x1106F, prNU, gcNd}, // [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE + {0x11070, 0x11070, prCM, gcMn}, // BRAHMI SIGN OLD TAMIL VIRAMA + {0x11071, 0x11072, prAL, gcLo}, // [2] BRAHMI LETTER OLD TAMIL SHORT E..BRAHMI LETTER OLD TAMIL SHORT O + {0x11073, 0x11074, prCM, gcMn}, // [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O + {0x11075, 0x11075, prAL, gcLo}, // BRAHMI LETTER OLD TAMIL LLA + {0x1107F, 0x1107F, prCM, gcMn}, // BRAHMI NUMBER JOINER + {0x11080, 0x11081, prCM, gcMn}, // [2] KAITHI SIGN CANDRABINDU..KAITHI SIGN ANUSVARA + {0x11082, 0x11082, prCM, gcMc}, // KAITHI SIGN VISARGA + {0x11083, 0x110AF, prAL, gcLo}, // [45] KAITHI LETTER A..KAITHI LETTER HA + {0x110B0, 0x110B2, prCM, gcMc}, // [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II + {0x110B3, 0x110B6, prCM, gcMn}, // [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI + {0x110B7, 0x110B8, prCM, gcMc}, // [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU + {0x110B9, 0x110BA, prCM, gcMn}, // [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA + {0x110BB, 0x110BC, prAL, gcPo}, // [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN + {0x110BD, 0x110BD, prAL, gcCf}, // KAITHI NUMBER SIGN + {0x110BE, 0x110C1, prBA, gcPo}, // [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA + {0x110C2, 0x110C2, prCM, gcMn}, // KAITHI VOWEL SIGN VOCALIC R + {0x110CD, 0x110CD, prAL, gcCf}, // KAITHI NUMBER SIGN ABOVE + {0x110D0, 0x110E8, prAL, gcLo}, // [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE + {0x110F0, 0x110F9, prNU, gcNd}, // [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE + {0x11100, 0x11102, prCM, gcMn}, // [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA + {0x11103, 0x11126, prAL, gcLo}, // [36] CHAKMA LETTER AA..CHAKMA LETTER HAA + {0x11127, 0x1112B, prCM, gcMn}, // [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU + {0x1112C, 0x1112C, prCM, gcMc}, // CHAKMA VOWEL SIGN E + {0x1112D, 0x11134, prCM, gcMn}, // [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA + {0x11136, 0x1113F, prNU, gcNd}, // [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE + {0x11140, 0x11143, prBA, gcPo}, // [4] CHAKMA SECTION MARK..CHAKMA QUESTION MARK + {0x11144, 0x11144, prAL, gcLo}, // CHAKMA LETTER LHAA + {0x11145, 0x11146, prCM, gcMc}, // [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI + {0x11147, 0x11147, prAL, gcLo}, // CHAKMA LETTER VAA + {0x11150, 0x11172, prAL, gcLo}, // [35] MAHAJANI LETTER A..MAHAJANI LETTER RRA + {0x11173, 0x11173, prCM, gcMn}, // MAHAJANI SIGN NUKTA + {0x11174, 0x11174, prAL, gcPo}, // MAHAJANI ABBREVIATION SIGN + {0x11175, 0x11175, prBB, gcPo}, // MAHAJANI SECTION MARK + {0x11176, 0x11176, prAL, gcLo}, // MAHAJANI LIGATURE SHRI + {0x11180, 0x11181, prCM, gcMn}, // [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA + {0x11182, 0x11182, prCM, gcMc}, // SHARADA SIGN VISARGA + {0x11183, 0x111B2, prAL, gcLo}, // [48] SHARADA LETTER A..SHARADA LETTER HA + {0x111B3, 0x111B5, prCM, gcMc}, // [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II + {0x111B6, 0x111BE, prCM, gcMn}, // [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O + {0x111BF, 0x111C0, prCM, gcMc}, // [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA + {0x111C1, 0x111C4, prAL, gcLo}, // [4] SHARADA SIGN AVAGRAHA..SHARADA OM + {0x111C5, 0x111C6, prBA, gcPo}, // [2] SHARADA DANDA..SHARADA DOUBLE DANDA + {0x111C7, 0x111C7, prAL, gcPo}, // SHARADA ABBREVIATION SIGN + {0x111C8, 0x111C8, prBA, gcPo}, // SHARADA SEPARATOR + {0x111C9, 0x111CC, prCM, gcMn}, // [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK + {0x111CD, 0x111CD, prAL, gcPo}, // SHARADA SUTRA MARK + {0x111CE, 0x111CE, prCM, gcMc}, // SHARADA VOWEL SIGN PRISHTHAMATRA E + {0x111CF, 0x111CF, prCM, gcMn}, // SHARADA SIGN INVERTED CANDRABINDU + {0x111D0, 0x111D9, prNU, gcNd}, // [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE + {0x111DA, 0x111DA, prAL, gcLo}, // SHARADA EKAM + {0x111DB, 0x111DB, prBB, gcPo}, // SHARADA SIGN SIDDHAM + {0x111DC, 0x111DC, prAL, gcLo}, // SHARADA HEADSTROKE + {0x111DD, 0x111DF, prBA, gcPo}, // [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 + {0x111E1, 0x111F4, prAL, gcNo}, // [20] SINHALA ARCHAIC DIGIT ONE..SINHALA ARCHAIC NUMBER ONE THOUSAND + {0x11200, 0x11211, prAL, gcLo}, // [18] KHOJKI LETTER A..KHOJKI LETTER JJA + {0x11213, 0x1122B, prAL, gcLo}, // [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA + {0x1122C, 0x1122E, prCM, gcMc}, // [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II + {0x1122F, 0x11231, prCM, gcMn}, // [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI + {0x11232, 0x11233, prCM, gcMc}, // [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU + {0x11234, 0x11234, prCM, gcMn}, // KHOJKI SIGN ANUSVARA + {0x11235, 0x11235, prCM, gcMc}, // KHOJKI SIGN VIRAMA + {0x11236, 0x11237, prCM, gcMn}, // [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA + {0x11238, 0x11239, prBA, gcPo}, // [2] KHOJKI DANDA..KHOJKI DOUBLE DANDA + {0x1123A, 0x1123A, prAL, gcPo}, // KHOJKI WORD SEPARATOR + {0x1123B, 0x1123C, prBA, gcPo}, // [2] KHOJKI SECTION MARK..KHOJKI DOUBLE SECTION MARK + {0x1123D, 0x1123D, prAL, gcPo}, // KHOJKI ABBREVIATION SIGN + {0x1123E, 0x1123E, prCM, gcMn}, // KHOJKI SIGN SUKUN + {0x1123F, 0x11240, prAL, gcLo}, // [2] KHOJKI LETTER QA..KHOJKI LETTER SHORT I + {0x11241, 0x11241, prCM, gcMn}, // KHOJKI VOWEL SIGN VOCALIC R + {0x11280, 0x11286, prAL, gcLo}, // [7] MULTANI LETTER A..MULTANI LETTER GA + {0x11288, 0x11288, prAL, gcLo}, // MULTANI LETTER GHA + {0x1128A, 0x1128D, prAL, gcLo}, // [4] MULTANI LETTER CA..MULTANI LETTER JJA + {0x1128F, 0x1129D, prAL, gcLo}, // [15] MULTANI LETTER NYA..MULTANI LETTER BA + {0x1129F, 0x112A8, prAL, gcLo}, // [10] MULTANI LETTER BHA..MULTANI LETTER RHA + {0x112A9, 0x112A9, prBA, gcPo}, // MULTANI SECTION MARK + {0x112B0, 0x112DE, prAL, gcLo}, // [47] KHUDAWADI LETTER A..KHUDAWADI LETTER HA + {0x112DF, 0x112DF, prCM, gcMn}, // KHUDAWADI SIGN ANUSVARA + {0x112E0, 0x112E2, prCM, gcMc}, // [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II + {0x112E3, 0x112EA, prCM, gcMn}, // [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA + {0x112F0, 0x112F9, prNU, gcNd}, // [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE + {0x11300, 0x11301, prCM, gcMn}, // [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU + {0x11302, 0x11303, prCM, gcMc}, // [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA + {0x11305, 0x1130C, prAL, gcLo}, // [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L + {0x1130F, 0x11310, prAL, gcLo}, // [2] GRANTHA LETTER EE..GRANTHA LETTER AI + {0x11313, 0x11328, prAL, gcLo}, // [22] GRANTHA LETTER OO..GRANTHA LETTER NA + {0x1132A, 0x11330, prAL, gcLo}, // [7] GRANTHA LETTER PA..GRANTHA LETTER RA + {0x11332, 0x11333, prAL, gcLo}, // [2] GRANTHA LETTER LA..GRANTHA LETTER LLA + {0x11335, 0x11339, prAL, gcLo}, // [5] GRANTHA LETTER VA..GRANTHA LETTER HA + {0x1133B, 0x1133C, prCM, gcMn}, // [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA + {0x1133D, 0x1133D, prAL, gcLo}, // GRANTHA SIGN AVAGRAHA + {0x1133E, 0x1133F, prCM, gcMc}, // [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I + {0x11340, 0x11340, prCM, gcMn}, // GRANTHA VOWEL SIGN II + {0x11341, 0x11344, prCM, gcMc}, // [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR + {0x11347, 0x11348, prCM, gcMc}, // [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI + {0x1134B, 0x1134D, prCM, gcMc}, // [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA + {0x11350, 0x11350, prAL, gcLo}, // GRANTHA OM + {0x11357, 0x11357, prCM, gcMc}, // GRANTHA AU LENGTH MARK + {0x1135D, 0x11361, prAL, gcLo}, // [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL + {0x11362, 0x11363, prCM, gcMc}, // [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL + {0x11366, 0x1136C, prCM, gcMn}, // [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX + {0x11370, 0x11374, prCM, gcMn}, // [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA + {0x11400, 0x11434, prAL, gcLo}, // [53] NEWA LETTER A..NEWA LETTER HA + {0x11435, 0x11437, prCM, gcMc}, // [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II + {0x11438, 0x1143F, prCM, gcMn}, // [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI + {0x11440, 0x11441, prCM, gcMc}, // [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU + {0x11442, 0x11444, prCM, gcMn}, // [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA + {0x11445, 0x11445, prCM, gcMc}, // NEWA SIGN VISARGA + {0x11446, 0x11446, prCM, gcMn}, // NEWA SIGN NUKTA + {0x11447, 0x1144A, prAL, gcLo}, // [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI + {0x1144B, 0x1144E, prBA, gcPo}, // [4] NEWA DANDA..NEWA GAP FILLER + {0x1144F, 0x1144F, prAL, gcPo}, // NEWA ABBREVIATION SIGN + {0x11450, 0x11459, prNU, gcNd}, // [10] NEWA DIGIT ZERO..NEWA DIGIT NINE + {0x1145A, 0x1145B, prBA, gcPo}, // [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK + {0x1145D, 0x1145D, prAL, gcPo}, // NEWA INSERTION SIGN + {0x1145E, 0x1145E, prCM, gcMn}, // NEWA SANDHI MARK + {0x1145F, 0x11461, prAL, gcLo}, // [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA + {0x11480, 0x114AF, prAL, gcLo}, // [48] TIRHUTA ANJI..TIRHUTA LETTER HA + {0x114B0, 0x114B2, prCM, gcMc}, // [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II + {0x114B3, 0x114B8, prCM, gcMn}, // [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL + {0x114B9, 0x114B9, prCM, gcMc}, // TIRHUTA VOWEL SIGN E + {0x114BA, 0x114BA, prCM, gcMn}, // TIRHUTA VOWEL SIGN SHORT E + {0x114BB, 0x114BE, prCM, gcMc}, // [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU + {0x114BF, 0x114C0, prCM, gcMn}, // [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA + {0x114C1, 0x114C1, prCM, gcMc}, // TIRHUTA SIGN VISARGA + {0x114C2, 0x114C3, prCM, gcMn}, // [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA + {0x114C4, 0x114C5, prAL, gcLo}, // [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA GVANG + {0x114C6, 0x114C6, prAL, gcPo}, // TIRHUTA ABBREVIATION SIGN + {0x114C7, 0x114C7, prAL, gcLo}, // TIRHUTA OM + {0x114D0, 0x114D9, prNU, gcNd}, // [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE + {0x11580, 0x115AE, prAL, gcLo}, // [47] SIDDHAM LETTER A..SIDDHAM LETTER HA + {0x115AF, 0x115B1, prCM, gcMc}, // [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II + {0x115B2, 0x115B5, prCM, gcMn}, // [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR + {0x115B8, 0x115BB, prCM, gcMc}, // [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU + {0x115BC, 0x115BD, prCM, gcMn}, // [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA + {0x115BE, 0x115BE, prCM, gcMc}, // SIDDHAM SIGN VISARGA + {0x115BF, 0x115C0, prCM, gcMn}, // [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA + {0x115C1, 0x115C1, prBB, gcPo}, // SIDDHAM SIGN SIDDHAM + {0x115C2, 0x115C3, prBA, gcPo}, // [2] SIDDHAM DANDA..SIDDHAM DOUBLE DANDA + {0x115C4, 0x115C5, prEX, gcPo}, // [2] SIDDHAM SEPARATOR DOT..SIDDHAM SEPARATOR BAR + {0x115C6, 0x115C8, prAL, gcPo}, // [3] SIDDHAM REPETITION MARK-1..SIDDHAM REPETITION MARK-3 + {0x115C9, 0x115D7, prBA, gcPo}, // [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES + {0x115D8, 0x115DB, prAL, gcLo}, // [4] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM LETTER ALTERNATE U + {0x115DC, 0x115DD, prCM, gcMn}, // [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU + {0x11600, 0x1162F, prAL, gcLo}, // [48] MODI LETTER A..MODI LETTER LLA + {0x11630, 0x11632, prCM, gcMc}, // [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II + {0x11633, 0x1163A, prCM, gcMn}, // [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI + {0x1163B, 0x1163C, prCM, gcMc}, // [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU + {0x1163D, 0x1163D, prCM, gcMn}, // MODI SIGN ANUSVARA + {0x1163E, 0x1163E, prCM, gcMc}, // MODI SIGN VISARGA + {0x1163F, 0x11640, prCM, gcMn}, // [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA + {0x11641, 0x11642, prBA, gcPo}, // [2] MODI DANDA..MODI DOUBLE DANDA + {0x11643, 0x11643, prAL, gcPo}, // MODI ABBREVIATION SIGN + {0x11644, 0x11644, prAL, gcLo}, // MODI SIGN HUVA + {0x11650, 0x11659, prNU, gcNd}, // [10] MODI DIGIT ZERO..MODI DIGIT NINE + {0x11660, 0x1166C, prBB, gcPo}, // [13] MONGOLIAN BIRGA WITH ORNAMENT..MONGOLIAN TURNED SWIRL BIRGA WITH DOUBLE ORNAMENT + {0x11680, 0x116AA, prAL, gcLo}, // [43] TAKRI LETTER A..TAKRI LETTER RRA + {0x116AB, 0x116AB, prCM, gcMn}, // TAKRI SIGN ANUSVARA + {0x116AC, 0x116AC, prCM, gcMc}, // TAKRI SIGN VISARGA + {0x116AD, 0x116AD, prCM, gcMn}, // TAKRI VOWEL SIGN AA + {0x116AE, 0x116AF, prCM, gcMc}, // [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II + {0x116B0, 0x116B5, prCM, gcMn}, // [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU + {0x116B6, 0x116B6, prCM, gcMc}, // TAKRI SIGN VIRAMA + {0x116B7, 0x116B7, prCM, gcMn}, // TAKRI SIGN NUKTA + {0x116B8, 0x116B8, prAL, gcLo}, // TAKRI LETTER ARCHAIC KHA + {0x116B9, 0x116B9, prAL, gcPo}, // TAKRI ABBREVIATION SIGN + {0x116C0, 0x116C9, prNU, gcNd}, // [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE + {0x11700, 0x1171A, prSA, gcLo}, // [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA + {0x1171D, 0x1171F, prSA, gcMn}, // [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA + {0x11720, 0x11721, prSA, gcMc}, // [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA + {0x11722, 0x11725, prSA, gcMn}, // [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU + {0x11726, 0x11726, prSA, gcMc}, // AHOM VOWEL SIGN E + {0x11727, 0x1172B, prSA, gcMn}, // [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER + {0x11730, 0x11739, prNU, gcNd}, // [10] AHOM DIGIT ZERO..AHOM DIGIT NINE + {0x1173A, 0x1173B, prSA, gcNo}, // [2] AHOM NUMBER TEN..AHOM NUMBER TWENTY + {0x1173C, 0x1173E, prBA, gcPo}, // [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI + {0x1173F, 0x1173F, prSA, gcSo}, // AHOM SYMBOL VI + {0x11740, 0x11746, prSA, gcLo}, // [7] AHOM LETTER CA..AHOM LETTER LLA + {0x11800, 0x1182B, prAL, gcLo}, // [44] DOGRA LETTER A..DOGRA LETTER RRA + {0x1182C, 0x1182E, prCM, gcMc}, // [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II + {0x1182F, 0x11837, prCM, gcMn}, // [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA + {0x11838, 0x11838, prCM, gcMc}, // DOGRA SIGN VISARGA + {0x11839, 0x1183A, prCM, gcMn}, // [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA + {0x1183B, 0x1183B, prAL, gcPo}, // DOGRA ABBREVIATION SIGN + {0x118A0, 0x118DF, prAL, gcLC}, // [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO + {0x118E0, 0x118E9, prNU, gcNd}, // [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE + {0x118EA, 0x118F2, prAL, gcNo}, // [9] WARANG CITI NUMBER TEN..WARANG CITI NUMBER NINETY + {0x118FF, 0x118FF, prAL, gcLo}, // WARANG CITI OM + {0x11900, 0x11906, prAL, gcLo}, // [7] DIVES AKURU LETTER A..DIVES AKURU LETTER E + {0x11909, 0x11909, prAL, gcLo}, // DIVES AKURU LETTER O + {0x1190C, 0x11913, prAL, gcLo}, // [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA + {0x11915, 0x11916, prAL, gcLo}, // [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA + {0x11918, 0x1192F, prAL, gcLo}, // [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA + {0x11930, 0x11935, prCM, gcMc}, // [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E + {0x11937, 0x11938, prCM, gcMc}, // [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O + {0x1193B, 0x1193C, prCM, gcMn}, // [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU + {0x1193D, 0x1193D, prCM, gcMc}, // DIVES AKURU SIGN HALANTA + {0x1193E, 0x1193E, prCM, gcMn}, // DIVES AKURU VIRAMA + {0x1193F, 0x1193F, prAL, gcLo}, // DIVES AKURU PREFIXED NASAL SIGN + {0x11940, 0x11940, prCM, gcMc}, // DIVES AKURU MEDIAL YA + {0x11941, 0x11941, prAL, gcLo}, // DIVES AKURU INITIAL RA + {0x11942, 0x11942, prCM, gcMc}, // DIVES AKURU MEDIAL RA + {0x11943, 0x11943, prCM, gcMn}, // DIVES AKURU SIGN NUKTA + {0x11944, 0x11946, prBA, gcPo}, // [3] DIVES AKURU DOUBLE DANDA..DIVES AKURU END OF TEXT MARK + {0x11950, 0x11959, prNU, gcNd}, // [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE + {0x119A0, 0x119A7, prAL, gcLo}, // [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR + {0x119AA, 0x119D0, prAL, gcLo}, // [39] NANDINAGARI LETTER E..NANDINAGARI LETTER RRA + {0x119D1, 0x119D3, prCM, gcMc}, // [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II + {0x119D4, 0x119D7, prCM, gcMn}, // [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR + {0x119DA, 0x119DB, prCM, gcMn}, // [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI + {0x119DC, 0x119DF, prCM, gcMc}, // [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA + {0x119E0, 0x119E0, prCM, gcMn}, // NANDINAGARI SIGN VIRAMA + {0x119E1, 0x119E1, prAL, gcLo}, // NANDINAGARI SIGN AVAGRAHA + {0x119E2, 0x119E2, prBB, gcPo}, // NANDINAGARI SIGN SIDDHAM + {0x119E3, 0x119E3, prAL, gcLo}, // NANDINAGARI HEADSTROKE + {0x119E4, 0x119E4, prCM, gcMc}, // NANDINAGARI VOWEL SIGN PRISHTHAMATRA E + {0x11A00, 0x11A00, prAL, gcLo}, // ZANABAZAR SQUARE LETTER A + {0x11A01, 0x11A0A, prCM, gcMn}, // [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK + {0x11A0B, 0x11A32, prAL, gcLo}, // [40] ZANABAZAR SQUARE LETTER KA..ZANABAZAR SQUARE LETTER KSSA + {0x11A33, 0x11A38, prCM, gcMn}, // [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA + {0x11A39, 0x11A39, prCM, gcMc}, // ZANABAZAR SQUARE SIGN VISARGA + {0x11A3A, 0x11A3A, prAL, gcLo}, // ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA + {0x11A3B, 0x11A3E, prCM, gcMn}, // [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA + {0x11A3F, 0x11A3F, prBB, gcPo}, // ZANABAZAR SQUARE INITIAL HEAD MARK + {0x11A40, 0x11A40, prAL, gcPo}, // ZANABAZAR SQUARE CLOSING HEAD MARK + {0x11A41, 0x11A44, prBA, gcPo}, // [4] ZANABAZAR SQUARE MARK TSHEG..ZANABAZAR SQUARE MARK LONG TSHEG + {0x11A45, 0x11A45, prBB, gcPo}, // ZANABAZAR SQUARE INITIAL DOUBLE-LINED HEAD MARK + {0x11A46, 0x11A46, prAL, gcPo}, // ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK + {0x11A47, 0x11A47, prCM, gcMn}, // ZANABAZAR SQUARE SUBJOINER + {0x11A50, 0x11A50, prAL, gcLo}, // SOYOMBO LETTER A + {0x11A51, 0x11A56, prCM, gcMn}, // [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE + {0x11A57, 0x11A58, prCM, gcMc}, // [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU + {0x11A59, 0x11A5B, prCM, gcMn}, // [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK + {0x11A5C, 0x11A89, prAL, gcLo}, // [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA + {0x11A8A, 0x11A96, prCM, gcMn}, // [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA + {0x11A97, 0x11A97, prCM, gcMc}, // SOYOMBO SIGN VISARGA + {0x11A98, 0x11A99, prCM, gcMn}, // [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER + {0x11A9A, 0x11A9C, prBA, gcPo}, // [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD + {0x11A9D, 0x11A9D, prAL, gcLo}, // SOYOMBO MARK PLUTA + {0x11A9E, 0x11AA0, prBB, gcPo}, // [3] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO HEAD MARK WITH MOON AND SUN + {0x11AA1, 0x11AA2, prBA, gcPo}, // [2] SOYOMBO TERMINAL MARK-1..SOYOMBO TERMINAL MARK-2 + {0x11AB0, 0x11ABF, prAL, gcLo}, // [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA + {0x11AC0, 0x11AF8, prAL, gcLo}, // [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL + {0x11B00, 0x11B09, prBB, gcPo}, // [10] DEVANAGARI HEAD MARK..DEVANAGARI SIGN MINDU + {0x11C00, 0x11C08, prAL, gcLo}, // [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L + {0x11C0A, 0x11C2E, prAL, gcLo}, // [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA + {0x11C2F, 0x11C2F, prCM, gcMc}, // BHAIKSUKI VOWEL SIGN AA + {0x11C30, 0x11C36, prCM, gcMn}, // [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L + {0x11C38, 0x11C3D, prCM, gcMn}, // [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA + {0x11C3E, 0x11C3E, prCM, gcMc}, // BHAIKSUKI SIGN VISARGA + {0x11C3F, 0x11C3F, prCM, gcMn}, // BHAIKSUKI SIGN VIRAMA + {0x11C40, 0x11C40, prAL, gcLo}, // BHAIKSUKI SIGN AVAGRAHA + {0x11C41, 0x11C45, prBA, gcPo}, // [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2 + {0x11C50, 0x11C59, prNU, gcNd}, // [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE + {0x11C5A, 0x11C6C, prAL, gcNo}, // [19] BHAIKSUKI NUMBER ONE..BHAIKSUKI HUNDREDS UNIT MARK + {0x11C70, 0x11C70, prBB, gcPo}, // MARCHEN HEAD MARK + {0x11C71, 0x11C71, prEX, gcPo}, // MARCHEN MARK SHAD + {0x11C72, 0x11C8F, prAL, gcLo}, // [30] MARCHEN LETTER KA..MARCHEN LETTER A + {0x11C92, 0x11CA7, prCM, gcMn}, // [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA + {0x11CA9, 0x11CA9, prCM, gcMc}, // MARCHEN SUBJOINED LETTER YA + {0x11CAA, 0x11CB0, prCM, gcMn}, // [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA + {0x11CB1, 0x11CB1, prCM, gcMc}, // MARCHEN VOWEL SIGN I + {0x11CB2, 0x11CB3, prCM, gcMn}, // [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E + {0x11CB4, 0x11CB4, prCM, gcMc}, // MARCHEN VOWEL SIGN O + {0x11CB5, 0x11CB6, prCM, gcMn}, // [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU + {0x11D00, 0x11D06, prAL, gcLo}, // [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E + {0x11D08, 0x11D09, prAL, gcLo}, // [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O + {0x11D0B, 0x11D30, prAL, gcLo}, // [38] MASARAM GONDI LETTER AU..MASARAM GONDI LETTER TRA + {0x11D31, 0x11D36, prCM, gcMn}, // [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R + {0x11D3A, 0x11D3A, prCM, gcMn}, // MASARAM GONDI VOWEL SIGN E + {0x11D3C, 0x11D3D, prCM, gcMn}, // [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O + {0x11D3F, 0x11D45, prCM, gcMn}, // [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA + {0x11D46, 0x11D46, prAL, gcLo}, // MASARAM GONDI REPHA + {0x11D47, 0x11D47, prCM, gcMn}, // MASARAM GONDI RA-KARA + {0x11D50, 0x11D59, prNU, gcNd}, // [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE + {0x11D60, 0x11D65, prAL, gcLo}, // [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU + {0x11D67, 0x11D68, prAL, gcLo}, // [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI + {0x11D6A, 0x11D89, prAL, gcLo}, // [32] GUNJALA GONDI LETTER OO..GUNJALA GONDI LETTER SA + {0x11D8A, 0x11D8E, prCM, gcMc}, // [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU + {0x11D90, 0x11D91, prCM, gcMn}, // [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI + {0x11D93, 0x11D94, prCM, gcMc}, // [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU + {0x11D95, 0x11D95, prCM, gcMn}, // GUNJALA GONDI SIGN ANUSVARA + {0x11D96, 0x11D96, prCM, gcMc}, // GUNJALA GONDI SIGN VISARGA + {0x11D97, 0x11D97, prCM, gcMn}, // GUNJALA GONDI VIRAMA + {0x11D98, 0x11D98, prAL, gcLo}, // GUNJALA GONDI OM + {0x11DA0, 0x11DA9, prNU, gcNd}, // [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE + {0x11EE0, 0x11EF2, prAL, gcLo}, // [19] MAKASAR LETTER KA..MAKASAR ANGKA + {0x11EF3, 0x11EF4, prCM, gcMn}, // [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U + {0x11EF5, 0x11EF6, prCM, gcMc}, // [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O + {0x11EF7, 0x11EF8, prAL, gcPo}, // [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION + {0x11F00, 0x11F01, prCM, gcMn}, // [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA + {0x11F02, 0x11F02, prAL, gcLo}, // KAWI SIGN REPHA + {0x11F03, 0x11F03, prCM, gcMc}, // KAWI SIGN VISARGA + {0x11F04, 0x11F10, prAL, gcLo}, // [13] KAWI LETTER A..KAWI LETTER O + {0x11F12, 0x11F33, prAL, gcLo}, // [34] KAWI LETTER KA..KAWI LETTER JNYA + {0x11F34, 0x11F35, prCM, gcMc}, // [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA + {0x11F36, 0x11F3A, prCM, gcMn}, // [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R + {0x11F3E, 0x11F3F, prCM, gcMc}, // [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI + {0x11F40, 0x11F40, prCM, gcMn}, // KAWI VOWEL SIGN EU + {0x11F41, 0x11F41, prCM, gcMc}, // KAWI SIGN KILLER + {0x11F42, 0x11F42, prCM, gcMn}, // KAWI CONJOINER + {0x11F43, 0x11F44, prBA, gcPo}, // [2] KAWI DANDA..KAWI DOUBLE DANDA + {0x11F45, 0x11F4F, prID, gcPo}, // [11] KAWI PUNCTUATION SECTION MARKER..KAWI PUNCTUATION CLOSING SPIRAL + {0x11F50, 0x11F59, prNU, gcNd}, // [10] KAWI DIGIT ZERO..KAWI DIGIT NINE + {0x11FB0, 0x11FB0, prAL, gcLo}, // LISU LETTER YHA + {0x11FC0, 0x11FD4, prAL, gcNo}, // [21] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL FRACTION DOWNSCALING FACTOR KIIZH + {0x11FD5, 0x11FDC, prAL, gcSo}, // [8] TAMIL SIGN NEL..TAMIL SIGN MUKKURUNI + {0x11FDD, 0x11FE0, prPO, gcSc}, // [4] TAMIL SIGN KAACU..TAMIL SIGN VARAAKAN + {0x11FE1, 0x11FF1, prAL, gcSo}, // [17] TAMIL SIGN PAARAM..TAMIL SIGN VAKAIYARAA + {0x11FFF, 0x11FFF, prBA, gcPo}, // TAMIL PUNCTUATION END OF TEXT + {0x12000, 0x12399, prAL, gcLo}, // [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U + {0x12400, 0x1246E, prAL, gcNl}, // [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM + {0x12470, 0x12474, prBA, gcPo}, // [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON + {0x12480, 0x12543, prAL, gcLo}, // [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU + {0x12F90, 0x12FF0, prAL, gcLo}, // [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 + {0x12FF1, 0x12FF2, prAL, gcPo}, // [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302 + {0x13000, 0x13257, prAL, gcLo}, // [600] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH O006 + {0x13258, 0x1325A, prOP, gcLo}, // [3] EGYPTIAN HIEROGLYPH O006A..EGYPTIAN HIEROGLYPH O006C + {0x1325B, 0x1325D, prCL, gcLo}, // [3] EGYPTIAN HIEROGLYPH O006D..EGYPTIAN HIEROGLYPH O006F + {0x1325E, 0x13281, prAL, gcLo}, // [36] EGYPTIAN HIEROGLYPH O007..EGYPTIAN HIEROGLYPH O033 + {0x13282, 0x13282, prCL, gcLo}, // EGYPTIAN HIEROGLYPH O033A + {0x13283, 0x13285, prAL, gcLo}, // [3] EGYPTIAN HIEROGLYPH O034..EGYPTIAN HIEROGLYPH O036 + {0x13286, 0x13286, prOP, gcLo}, // EGYPTIAN HIEROGLYPH O036A + {0x13287, 0x13287, prCL, gcLo}, // EGYPTIAN HIEROGLYPH O036B + {0x13288, 0x13288, prOP, gcLo}, // EGYPTIAN HIEROGLYPH O036C + {0x13289, 0x13289, prCL, gcLo}, // EGYPTIAN HIEROGLYPH O036D + {0x1328A, 0x13378, prAL, gcLo}, // [239] EGYPTIAN HIEROGLYPH O037..EGYPTIAN HIEROGLYPH V011 + {0x13379, 0x13379, prOP, gcLo}, // EGYPTIAN HIEROGLYPH V011A + {0x1337A, 0x1337B, prCL, gcLo}, // [2] EGYPTIAN HIEROGLYPH V011B..EGYPTIAN HIEROGLYPH V011C + {0x1337C, 0x1342F, prAL, gcLo}, // [180] EGYPTIAN HIEROGLYPH V012..EGYPTIAN HIEROGLYPH V011D + {0x13430, 0x13436, prGL, gcCf}, // [7] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH OVERLAY MIDDLE + {0x13437, 0x13437, prOP, gcCf}, // EGYPTIAN HIEROGLYPH BEGIN SEGMENT + {0x13438, 0x13438, prCL, gcCf}, // EGYPTIAN HIEROGLYPH END SEGMENT + {0x13439, 0x1343B, prGL, gcCf}, // [3] EGYPTIAN HIEROGLYPH INSERT AT MIDDLE..EGYPTIAN HIEROGLYPH INSERT AT BOTTOM + {0x1343C, 0x1343C, prOP, gcCf}, // EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE + {0x1343D, 0x1343D, prCL, gcCf}, // EGYPTIAN HIEROGLYPH END ENCLOSURE + {0x1343E, 0x1343E, prOP, gcCf}, // EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE + {0x1343F, 0x1343F, prCL, gcCf}, // EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE + {0x13440, 0x13440, prCM, gcMn}, // EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY + {0x13441, 0x13446, prAL, gcLo}, // [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN + {0x13447, 0x13455, prCM, gcMn}, // [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED + {0x14400, 0x145CD, prAL, gcLo}, // [462] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A409 + {0x145CE, 0x145CE, prOP, gcLo}, // ANATOLIAN HIEROGLYPH A410 BEGIN LOGOGRAM MARK + {0x145CF, 0x145CF, prCL, gcLo}, // ANATOLIAN HIEROGLYPH A410A END LOGOGRAM MARK + {0x145D0, 0x14646, prAL, gcLo}, // [119] ANATOLIAN HIEROGLYPH A411..ANATOLIAN HIEROGLYPH A530 + {0x16800, 0x16A38, prAL, gcLo}, // [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ + {0x16A40, 0x16A5E, prAL, gcLo}, // [31] MRO LETTER TA..MRO LETTER TEK + {0x16A60, 0x16A69, prNU, gcNd}, // [10] MRO DIGIT ZERO..MRO DIGIT NINE + {0x16A6E, 0x16A6F, prBA, gcPo}, // [2] MRO DANDA..MRO DOUBLE DANDA + {0x16A70, 0x16ABE, prAL, gcLo}, // [79] TANGSA LETTER OZ..TANGSA LETTER ZA + {0x16AC0, 0x16AC9, prNU, gcNd}, // [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE + {0x16AD0, 0x16AED, prAL, gcLo}, // [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I + {0x16AF0, 0x16AF4, prCM, gcMn}, // [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE + {0x16AF5, 0x16AF5, prBA, gcPo}, // BASSA VAH FULL STOP + {0x16B00, 0x16B2F, prAL, gcLo}, // [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU + {0x16B30, 0x16B36, prCM, gcMn}, // [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM + {0x16B37, 0x16B39, prBA, gcPo}, // [3] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN CIM CHEEM + {0x16B3A, 0x16B3B, prAL, gcPo}, // [2] PAHAWH HMONG SIGN VOS THIAB..PAHAWH HMONG SIGN VOS FEEM + {0x16B3C, 0x16B3F, prAL, gcSo}, // [4] PAHAWH HMONG SIGN XYEEM NTXIV..PAHAWH HMONG SIGN XYEEM FAIB + {0x16B40, 0x16B43, prAL, gcLm}, // [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM + {0x16B44, 0x16B44, prBA, gcPo}, // PAHAWH HMONG SIGN XAUS + {0x16B45, 0x16B45, prAL, gcSo}, // PAHAWH HMONG SIGN CIM TSOV ROG + {0x16B50, 0x16B59, prNU, gcNd}, // [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE + {0x16B5B, 0x16B61, prAL, gcNo}, // [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS + {0x16B63, 0x16B77, prAL, gcLo}, // [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS + {0x16B7D, 0x16B8F, prAL, gcLo}, // [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ + {0x16E40, 0x16E7F, prAL, gcLC}, // [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y + {0x16E80, 0x16E96, prAL, gcNo}, // [23] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN DIGIT THREE ALTERNATE FORM + {0x16E97, 0x16E98, prBA, gcPo}, // [2] MEDEFAIDRIN COMMA..MEDEFAIDRIN FULL STOP + {0x16E99, 0x16E9A, prAL, gcPo}, // [2] MEDEFAIDRIN SYMBOL AIVA..MEDEFAIDRIN EXCLAMATION OH + {0x16F00, 0x16F4A, prAL, gcLo}, // [75] MIAO LETTER PA..MIAO LETTER RTE + {0x16F4F, 0x16F4F, prCM, gcMn}, // MIAO SIGN CONSONANT MODIFIER BAR + {0x16F50, 0x16F50, prAL, gcLo}, // MIAO LETTER NASALIZATION + {0x16F51, 0x16F87, prCM, gcMc}, // [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI + {0x16F8F, 0x16F92, prCM, gcMn}, // [4] MIAO TONE RIGHT..MIAO TONE BELOW + {0x16F93, 0x16F9F, prAL, gcLm}, // [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 + {0x16FE0, 0x16FE1, prNS, gcLm}, // [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK + {0x16FE2, 0x16FE2, prNS, gcPo}, // OLD CHINESE HOOK MARK + {0x16FE3, 0x16FE3, prNS, gcLm}, // OLD CHINESE ITERATION MARK + {0x16FE4, 0x16FE4, prGL, gcMn}, // KHITAN SMALL SCRIPT FILLER + {0x16FF0, 0x16FF1, prCM, gcMc}, // [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY + {0x17000, 0x187F7, prID, gcLo}, // [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 + {0x18800, 0x18AFF, prID, gcLo}, // [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768 + {0x18B00, 0x18CD5, prAL, gcLo}, // [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5 + {0x18D00, 0x18D08, prID, gcLo}, // [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 + {0x1AFF0, 0x1AFF3, prAL, gcLm}, // [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 + {0x1AFF5, 0x1AFFB, prAL, gcLm}, // [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 + {0x1AFFD, 0x1AFFE, prAL, gcLm}, // [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 + {0x1B000, 0x1B0FF, prID, gcLo}, // [256] KATAKANA LETTER ARCHAIC E..HENTAIGANA LETTER RE-2 + {0x1B100, 0x1B122, prID, gcLo}, // [35] HENTAIGANA LETTER RE-3..KATAKANA LETTER ARCHAIC WU + {0x1B132, 0x1B132, prCJ, gcLo}, // HIRAGANA LETTER SMALL KO + {0x1B150, 0x1B152, prCJ, gcLo}, // [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO + {0x1B155, 0x1B155, prCJ, gcLo}, // KATAKANA LETTER SMALL KO + {0x1B164, 0x1B167, prCJ, gcLo}, // [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N + {0x1B170, 0x1B2FB, prID, gcLo}, // [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB + {0x1BC00, 0x1BC6A, prAL, gcLo}, // [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M + {0x1BC70, 0x1BC7C, prAL, gcLo}, // [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK + {0x1BC80, 0x1BC88, prAL, gcLo}, // [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL + {0x1BC90, 0x1BC99, prAL, gcLo}, // [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW + {0x1BC9C, 0x1BC9C, prAL, gcSo}, // DUPLOYAN SIGN O WITH CROSS + {0x1BC9D, 0x1BC9E, prCM, gcMn}, // [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK + {0x1BC9F, 0x1BC9F, prBA, gcPo}, // DUPLOYAN PUNCTUATION CHINOOK FULL STOP + {0x1BCA0, 0x1BCA3, prCM, gcCf}, // [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP + {0x1CF00, 0x1CF2D, prCM, gcMn}, // [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT + {0x1CF30, 0x1CF46, prCM, gcMn}, // [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG + {0x1CF50, 0x1CFC3, prAL, gcSo}, // [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK + {0x1D000, 0x1D0F5, prAL, gcSo}, // [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO + {0x1D100, 0x1D126, prAL, gcSo}, // [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 + {0x1D129, 0x1D164, prAL, gcSo}, // [60] MUSICAL SYMBOL MULTIPLE MEASURE REST..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE + {0x1D165, 0x1D166, prCM, gcMc}, // [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM + {0x1D167, 0x1D169, prCM, gcMn}, // [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 + {0x1D16A, 0x1D16C, prAL, gcSo}, // [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 + {0x1D16D, 0x1D172, prCM, gcMc}, // [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 + {0x1D173, 0x1D17A, prCM, gcCf}, // [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE + {0x1D17B, 0x1D182, prCM, gcMn}, // [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE + {0x1D183, 0x1D184, prAL, gcSo}, // [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN + {0x1D185, 0x1D18B, prCM, gcMn}, // [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE + {0x1D18C, 0x1D1A9, prAL, gcSo}, // [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH + {0x1D1AA, 0x1D1AD, prCM, gcMn}, // [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO + {0x1D1AE, 0x1D1EA, prAL, gcSo}, // [61] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL KORON + {0x1D200, 0x1D241, prAL, gcSo}, // [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 + {0x1D242, 0x1D244, prCM, gcMn}, // [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + {0x1D245, 0x1D245, prAL, gcSo}, // GREEK MUSICAL LEIMMA + {0x1D2C0, 0x1D2D3, prAL, gcNo}, // [20] KAKTOVIK NUMERAL ZERO..KAKTOVIK NUMERAL NINETEEN + {0x1D2E0, 0x1D2F3, prAL, gcNo}, // [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN + {0x1D300, 0x1D356, prAL, gcSo}, // [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING + {0x1D360, 0x1D378, prAL, gcNo}, // [25] COUNTING ROD UNIT DIGIT ONE..TALLY MARK FIVE + {0x1D400, 0x1D454, prAL, gcLC}, // [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G + {0x1D456, 0x1D49C, prAL, gcLC}, // [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A + {0x1D49E, 0x1D49F, prAL, gcLu}, // [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D + {0x1D4A2, 0x1D4A2, prAL, gcLu}, // MATHEMATICAL SCRIPT CAPITAL G + {0x1D4A5, 0x1D4A6, prAL, gcLu}, // [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K + {0x1D4A9, 0x1D4AC, prAL, gcLu}, // [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q + {0x1D4AE, 0x1D4B9, prAL, gcLC}, // [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D + {0x1D4BB, 0x1D4BB, prAL, gcLl}, // MATHEMATICAL SCRIPT SMALL F + {0x1D4BD, 0x1D4C3, prAL, gcLl}, // [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N + {0x1D4C5, 0x1D505, prAL, gcLC}, // [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B + {0x1D507, 0x1D50A, prAL, gcLu}, // [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G + {0x1D50D, 0x1D514, prAL, gcLu}, // [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q + {0x1D516, 0x1D51C, prAL, gcLu}, // [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y + {0x1D51E, 0x1D539, prAL, gcLC}, // [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B + {0x1D53B, 0x1D53E, prAL, gcLu}, // [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G + {0x1D540, 0x1D544, prAL, gcLu}, // [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M + {0x1D546, 0x1D546, prAL, gcLu}, // MATHEMATICAL DOUBLE-STRUCK CAPITAL O + {0x1D54A, 0x1D550, prAL, gcLu}, // [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y + {0x1D552, 0x1D6A5, prAL, gcLC}, // [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J + {0x1D6A8, 0x1D6C0, prAL, gcLu}, // [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA + {0x1D6C1, 0x1D6C1, prAL, gcSm}, // MATHEMATICAL BOLD NABLA + {0x1D6C2, 0x1D6DA, prAL, gcLl}, // [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA + {0x1D6DB, 0x1D6DB, prAL, gcSm}, // MATHEMATICAL BOLD PARTIAL DIFFERENTIAL + {0x1D6DC, 0x1D6FA, prAL, gcLC}, // [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA + {0x1D6FB, 0x1D6FB, prAL, gcSm}, // MATHEMATICAL ITALIC NABLA + {0x1D6FC, 0x1D714, prAL, gcLl}, // [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA + {0x1D715, 0x1D715, prAL, gcSm}, // MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL + {0x1D716, 0x1D734, prAL, gcLC}, // [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA + {0x1D735, 0x1D735, prAL, gcSm}, // MATHEMATICAL BOLD ITALIC NABLA + {0x1D736, 0x1D74E, prAL, gcLl}, // [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA + {0x1D74F, 0x1D74F, prAL, gcSm}, // MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL + {0x1D750, 0x1D76E, prAL, gcLC}, // [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA + {0x1D76F, 0x1D76F, prAL, gcSm}, // MATHEMATICAL SANS-SERIF BOLD NABLA + {0x1D770, 0x1D788, prAL, gcLl}, // [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA + {0x1D789, 0x1D789, prAL, gcSm}, // MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL + {0x1D78A, 0x1D7A8, prAL, gcLC}, // [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA + {0x1D7A9, 0x1D7A9, prAL, gcSm}, // MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA + {0x1D7AA, 0x1D7C2, prAL, gcLl}, // [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA + {0x1D7C3, 0x1D7C3, prAL, gcSm}, // MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL + {0x1D7C4, 0x1D7CB, prAL, gcLC}, // [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA + {0x1D7CE, 0x1D7FF, prNU, gcNd}, // [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + {0x1D800, 0x1D9FF, prAL, gcSo}, // [512] SIGNWRITING HAND-FIST INDEX..SIGNWRITING HEAD + {0x1DA00, 0x1DA36, prCM, gcMn}, // [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN + {0x1DA37, 0x1DA3A, prAL, gcSo}, // [4] SIGNWRITING AIR BLOW SMALL ROTATIONS..SIGNWRITING BREATH EXHALE + {0x1DA3B, 0x1DA6C, prCM, gcMn}, // [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT + {0x1DA6D, 0x1DA74, prAL, gcSo}, // [8] SIGNWRITING SHOULDER HIP SPINE..SIGNWRITING TORSO-FLOORPLANE TWISTING + {0x1DA75, 0x1DA75, prCM, gcMn}, // SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS + {0x1DA76, 0x1DA83, prAL, gcSo}, // [14] SIGNWRITING LIMB COMBINATION..SIGNWRITING LOCATION DEPTH + {0x1DA84, 0x1DA84, prCM, gcMn}, // SIGNWRITING LOCATION HEAD NECK + {0x1DA85, 0x1DA86, prAL, gcSo}, // [2] SIGNWRITING LOCATION TORSO..SIGNWRITING LOCATION LIMBS DIGITS + {0x1DA87, 0x1DA8A, prBA, gcPo}, // [4] SIGNWRITING COMMA..SIGNWRITING COLON + {0x1DA8B, 0x1DA8B, prAL, gcPo}, // SIGNWRITING PARENTHESIS + {0x1DA9B, 0x1DA9F, prCM, gcMn}, // [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 + {0x1DAA1, 0x1DAAF, prCM, gcMn}, // [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 + {0x1DF00, 0x1DF09, prAL, gcLl}, // [10] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK + {0x1DF0A, 0x1DF0A, prAL, gcLo}, // LATIN LETTER RETROFLEX CLICK WITH RETROFLEX HOOK + {0x1DF0B, 0x1DF1E, prAL, gcLl}, // [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL + {0x1DF25, 0x1DF2A, prAL, gcLl}, // [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK + {0x1E000, 0x1E006, prCM, gcMn}, // [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE + {0x1E008, 0x1E018, prCM, gcMn}, // [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU + {0x1E01B, 0x1E021, prCM, gcMn}, // [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI + {0x1E023, 0x1E024, prCM, gcMn}, // [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS + {0x1E026, 0x1E02A, prCM, gcMn}, // [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA + {0x1E030, 0x1E06D, prAL, gcLm}, // [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE + {0x1E08F, 0x1E08F, prCM, gcMn}, // COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + {0x1E100, 0x1E12C, prAL, gcLo}, // [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W + {0x1E130, 0x1E136, prCM, gcMn}, // [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D + {0x1E137, 0x1E13D, prAL, gcLm}, // [7] NYIAKENG PUACHUE HMONG SIGN FOR PERSON..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER + {0x1E140, 0x1E149, prNU, gcNd}, // [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE + {0x1E14E, 0x1E14E, prAL, gcLo}, // NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ + {0x1E14F, 0x1E14F, prAL, gcSo}, // NYIAKENG PUACHUE HMONG CIRCLED CA + {0x1E290, 0x1E2AD, prAL, gcLo}, // [30] TOTO LETTER PA..TOTO LETTER A + {0x1E2AE, 0x1E2AE, prCM, gcMn}, // TOTO SIGN RISING TONE + {0x1E2C0, 0x1E2EB, prAL, gcLo}, // [44] WANCHO LETTER AA..WANCHO LETTER YIH + {0x1E2EC, 0x1E2EF, prCM, gcMn}, // [4] WANCHO TONE TUP..WANCHO TONE KOINI + {0x1E2F0, 0x1E2F9, prNU, gcNd}, // [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE + {0x1E2FF, 0x1E2FF, prPR, gcSc}, // WANCHO NGUN SIGN + {0x1E4D0, 0x1E4EA, prAL, gcLo}, // [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL + {0x1E4EB, 0x1E4EB, prAL, gcLm}, // NAG MUNDARI SIGN OJOD + {0x1E4EC, 0x1E4EF, prCM, gcMn}, // [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH + {0x1E4F0, 0x1E4F9, prNU, gcNd}, // [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE + {0x1E7E0, 0x1E7E6, prAL, gcLo}, // [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO + {0x1E7E8, 0x1E7EB, prAL, gcLo}, // [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE + {0x1E7ED, 0x1E7EE, prAL, gcLo}, // [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE + {0x1E7F0, 0x1E7FE, prAL, gcLo}, // [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE + {0x1E800, 0x1E8C4, prAL, gcLo}, // [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON + {0x1E8C7, 0x1E8CF, prAL, gcNo}, // [9] MENDE KIKAKUI DIGIT ONE..MENDE KIKAKUI DIGIT NINE + {0x1E8D0, 0x1E8D6, prCM, gcMn}, // [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + {0x1E900, 0x1E943, prAL, gcLC}, // [68] ADLAM CAPITAL LETTER ALIF..ADLAM SMALL LETTER SHA + {0x1E944, 0x1E94A, prCM, gcMn}, // [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA + {0x1E94B, 0x1E94B, prAL, gcLm}, // ADLAM NASALIZATION MARK + {0x1E950, 0x1E959, prNU, gcNd}, // [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE + {0x1E95E, 0x1E95F, prOP, gcPo}, // [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK + {0x1EC71, 0x1ECAB, prAL, gcNo}, // [59] INDIC SIYAQ NUMBER ONE..INDIC SIYAQ NUMBER PREFIXED NINE + {0x1ECAC, 0x1ECAC, prPO, gcSo}, // INDIC SIYAQ PLACEHOLDER + {0x1ECAD, 0x1ECAF, prAL, gcNo}, // [3] INDIC SIYAQ FRACTION ONE QUARTER..INDIC SIYAQ FRACTION THREE QUARTERS + {0x1ECB0, 0x1ECB0, prPO, gcSc}, // INDIC SIYAQ RUPEE MARK + {0x1ECB1, 0x1ECB4, prAL, gcNo}, // [4] INDIC SIYAQ NUMBER ALTERNATE ONE..INDIC SIYAQ ALTERNATE LAKH MARK + {0x1ED01, 0x1ED2D, prAL, gcNo}, // [45] OTTOMAN SIYAQ NUMBER ONE..OTTOMAN SIYAQ NUMBER NINETY THOUSAND + {0x1ED2E, 0x1ED2E, prAL, gcSo}, // OTTOMAN SIYAQ MARRATAN + {0x1ED2F, 0x1ED3D, prAL, gcNo}, // [15] OTTOMAN SIYAQ ALTERNATE NUMBER TWO..OTTOMAN SIYAQ FRACTION ONE SIXTH + {0x1EE00, 0x1EE03, prAL, gcLo}, // [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL + {0x1EE05, 0x1EE1F, prAL, gcLo}, // [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF + {0x1EE21, 0x1EE22, prAL, gcLo}, // [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM + {0x1EE24, 0x1EE24, prAL, gcLo}, // ARABIC MATHEMATICAL INITIAL HEH + {0x1EE27, 0x1EE27, prAL, gcLo}, // ARABIC MATHEMATICAL INITIAL HAH + {0x1EE29, 0x1EE32, prAL, gcLo}, // [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF + {0x1EE34, 0x1EE37, prAL, gcLo}, // [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH + {0x1EE39, 0x1EE39, prAL, gcLo}, // ARABIC MATHEMATICAL INITIAL DAD + {0x1EE3B, 0x1EE3B, prAL, gcLo}, // ARABIC MATHEMATICAL INITIAL GHAIN + {0x1EE42, 0x1EE42, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED JEEM + {0x1EE47, 0x1EE47, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED HAH + {0x1EE49, 0x1EE49, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED YEH + {0x1EE4B, 0x1EE4B, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED LAM + {0x1EE4D, 0x1EE4F, prAL, gcLo}, // [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN + {0x1EE51, 0x1EE52, prAL, gcLo}, // [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF + {0x1EE54, 0x1EE54, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED SHEEN + {0x1EE57, 0x1EE57, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED KHAH + {0x1EE59, 0x1EE59, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED DAD + {0x1EE5B, 0x1EE5B, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED GHAIN + {0x1EE5D, 0x1EE5D, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED DOTLESS NOON + {0x1EE5F, 0x1EE5F, prAL, gcLo}, // ARABIC MATHEMATICAL TAILED DOTLESS QAF + {0x1EE61, 0x1EE62, prAL, gcLo}, // [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM + {0x1EE64, 0x1EE64, prAL, gcLo}, // ARABIC MATHEMATICAL STRETCHED HEH + {0x1EE67, 0x1EE6A, prAL, gcLo}, // [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF + {0x1EE6C, 0x1EE72, prAL, gcLo}, // [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF + {0x1EE74, 0x1EE77, prAL, gcLo}, // [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH + {0x1EE79, 0x1EE7C, prAL, gcLo}, // [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH + {0x1EE7E, 0x1EE7E, prAL, gcLo}, // ARABIC MATHEMATICAL STRETCHED DOTLESS FEH + {0x1EE80, 0x1EE89, prAL, gcLo}, // [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH + {0x1EE8B, 0x1EE9B, prAL, gcLo}, // [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN + {0x1EEA1, 0x1EEA3, prAL, gcLo}, // [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL + {0x1EEA5, 0x1EEA9, prAL, gcLo}, // [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH + {0x1EEAB, 0x1EEBB, prAL, gcLo}, // [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN + {0x1EEF0, 0x1EEF1, prAL, gcSm}, // [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL + {0x1F000, 0x1F02B, prID, gcSo}, // [44] MAHJONG TILE EAST WIND..MAHJONG TILE BACK + {0x1F02C, 0x1F02F, prID, gcCn}, // [4] .. + {0x1F030, 0x1F093, prID, gcSo}, // [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 + {0x1F094, 0x1F09F, prID, gcCn}, // [12] .. + {0x1F0A0, 0x1F0AE, prID, gcSo}, // [15] PLAYING CARD BACK..PLAYING CARD KING OF SPADES + {0x1F0AF, 0x1F0B0, prID, gcCn}, // [2] .. + {0x1F0B1, 0x1F0BF, prID, gcSo}, // [15] PLAYING CARD ACE OF HEARTS..PLAYING CARD RED JOKER + {0x1F0C0, 0x1F0C0, prID, gcCn}, // + {0x1F0C1, 0x1F0CF, prID, gcSo}, // [15] PLAYING CARD ACE OF DIAMONDS..PLAYING CARD BLACK JOKER + {0x1F0D0, 0x1F0D0, prID, gcCn}, // + {0x1F0D1, 0x1F0F5, prID, gcSo}, // [37] PLAYING CARD ACE OF CLUBS..PLAYING CARD TRUMP-21 + {0x1F0F6, 0x1F0FF, prID, gcCn}, // [10] .. + {0x1F100, 0x1F10C, prAI, gcNo}, // [13] DIGIT ZERO FULL STOP..DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ZERO + {0x1F10D, 0x1F10F, prID, gcSo}, // [3] CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH + {0x1F110, 0x1F12D, prAI, gcSo}, // [30] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED CD + {0x1F12E, 0x1F12F, prAL, gcSo}, // [2] CIRCLED WZ..COPYLEFT SYMBOL + {0x1F130, 0x1F169, prAI, gcSo}, // [58] SQUARED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z + {0x1F16A, 0x1F16C, prAL, gcSo}, // [3] RAISED MC SIGN..RAISED MR SIGN + {0x1F16D, 0x1F16F, prID, gcSo}, // [3] CIRCLED CC..CIRCLED HUMAN FIGURE + {0x1F170, 0x1F1AC, prAI, gcSo}, // [61] NEGATIVE SQUARED LATIN CAPITAL LETTER A..SQUARED VOD + {0x1F1AD, 0x1F1AD, prID, gcSo}, // MASK WORK SYMBOL + {0x1F1AE, 0x1F1E5, prID, gcCn}, // [56] .. + {0x1F1E6, 0x1F1FF, prRI, gcSo}, // [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + {0x1F200, 0x1F202, prID, gcSo}, // [3] SQUARE HIRAGANA HOKA..SQUARED KATAKANA SA + {0x1F203, 0x1F20F, prID, gcCn}, // [13] .. + {0x1F210, 0x1F23B, prID, gcSo}, // [44] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-914D + {0x1F23C, 0x1F23F, prID, gcCn}, // [4] .. + {0x1F240, 0x1F248, prID, gcSo}, // [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 + {0x1F249, 0x1F24F, prID, gcCn}, // [7] .. + {0x1F250, 0x1F251, prID, gcSo}, // [2] CIRCLED IDEOGRAPH ADVANTAGE..CIRCLED IDEOGRAPH ACCEPT + {0x1F252, 0x1F25F, prID, gcCn}, // [14] .. + {0x1F260, 0x1F265, prID, gcSo}, // [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI + {0x1F266, 0x1F2FF, prID, gcCn}, // [154] .. + {0x1F300, 0x1F384, prID, gcSo}, // [133] CYCLONE..CHRISTMAS TREE + {0x1F385, 0x1F385, prEB, gcSo}, // FATHER CHRISTMAS + {0x1F386, 0x1F39B, prID, gcSo}, // [22] FIREWORKS..CONTROL KNOBS + {0x1F39C, 0x1F39D, prAL, gcSo}, // [2] BEAMED ASCENDING MUSICAL NOTES..BEAMED DESCENDING MUSICAL NOTES + {0x1F39E, 0x1F3B4, prID, gcSo}, // [23] FILM FRAMES..FLOWER PLAYING CARDS + {0x1F3B5, 0x1F3B6, prAL, gcSo}, // [2] MUSICAL NOTE..MULTIPLE MUSICAL NOTES + {0x1F3B7, 0x1F3BB, prID, gcSo}, // [5] SAXOPHONE..VIOLIN + {0x1F3BC, 0x1F3BC, prAL, gcSo}, // MUSICAL SCORE + {0x1F3BD, 0x1F3C1, prID, gcSo}, // [5] RUNNING SHIRT WITH SASH..CHEQUERED FLAG + {0x1F3C2, 0x1F3C4, prEB, gcSo}, // [3] SNOWBOARDER..SURFER + {0x1F3C5, 0x1F3C6, prID, gcSo}, // [2] SPORTS MEDAL..TROPHY + {0x1F3C7, 0x1F3C7, prEB, gcSo}, // HORSE RACING + {0x1F3C8, 0x1F3C9, prID, gcSo}, // [2] AMERICAN FOOTBALL..RUGBY FOOTBALL + {0x1F3CA, 0x1F3CC, prEB, gcSo}, // [3] SWIMMER..GOLFER + {0x1F3CD, 0x1F3FA, prID, gcSo}, // [46] RACING MOTORCYCLE..AMPHORA + {0x1F3FB, 0x1F3FF, prEM, gcSk}, // [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 + {0x1F400, 0x1F441, prID, gcSo}, // [66] RAT..EYE + {0x1F442, 0x1F443, prEB, gcSo}, // [2] EAR..NOSE + {0x1F444, 0x1F445, prID, gcSo}, // [2] MOUTH..TONGUE + {0x1F446, 0x1F450, prEB, gcSo}, // [11] WHITE UP POINTING BACKHAND INDEX..OPEN HANDS SIGN + {0x1F451, 0x1F465, prID, gcSo}, // [21] CROWN..BUSTS IN SILHOUETTE + {0x1F466, 0x1F478, prEB, gcSo}, // [19] BOY..PRINCESS + {0x1F479, 0x1F47B, prID, gcSo}, // [3] JAPANESE OGRE..GHOST + {0x1F47C, 0x1F47C, prEB, gcSo}, // BABY ANGEL + {0x1F47D, 0x1F480, prID, gcSo}, // [4] EXTRATERRESTRIAL ALIEN..SKULL + {0x1F481, 0x1F483, prEB, gcSo}, // [3] INFORMATION DESK PERSON..DANCER + {0x1F484, 0x1F484, prID, gcSo}, // LIPSTICK + {0x1F485, 0x1F487, prEB, gcSo}, // [3] NAIL POLISH..HAIRCUT + {0x1F488, 0x1F48E, prID, gcSo}, // [7] BARBER POLE..GEM STONE + {0x1F48F, 0x1F48F, prEB, gcSo}, // KISS + {0x1F490, 0x1F490, prID, gcSo}, // BOUQUET + {0x1F491, 0x1F491, prEB, gcSo}, // COUPLE WITH HEART + {0x1F492, 0x1F49F, prID, gcSo}, // [14] WEDDING..HEART DECORATION + {0x1F4A0, 0x1F4A0, prAL, gcSo}, // DIAMOND SHAPE WITH A DOT INSIDE + {0x1F4A1, 0x1F4A1, prID, gcSo}, // ELECTRIC LIGHT BULB + {0x1F4A2, 0x1F4A2, prAL, gcSo}, // ANGER SYMBOL + {0x1F4A3, 0x1F4A3, prID, gcSo}, // BOMB + {0x1F4A4, 0x1F4A4, prAL, gcSo}, // SLEEPING SYMBOL + {0x1F4A5, 0x1F4A9, prID, gcSo}, // [5] COLLISION SYMBOL..PILE OF POO + {0x1F4AA, 0x1F4AA, prEB, gcSo}, // FLEXED BICEPS + {0x1F4AB, 0x1F4AE, prID, gcSo}, // [4] DIZZY SYMBOL..WHITE FLOWER + {0x1F4AF, 0x1F4AF, prAL, gcSo}, // HUNDRED POINTS SYMBOL + {0x1F4B0, 0x1F4B0, prID, gcSo}, // MONEY BAG + {0x1F4B1, 0x1F4B2, prAL, gcSo}, // [2] CURRENCY EXCHANGE..HEAVY DOLLAR SIGN + {0x1F4B3, 0x1F4FF, prID, gcSo}, // [77] CREDIT CARD..PRAYER BEADS + {0x1F500, 0x1F506, prAL, gcSo}, // [7] TWISTED RIGHTWARDS ARROWS..HIGH BRIGHTNESS SYMBOL + {0x1F507, 0x1F516, prID, gcSo}, // [16] SPEAKER WITH CANCELLATION STROKE..BOOKMARK + {0x1F517, 0x1F524, prAL, gcSo}, // [14] LINK SYMBOL..INPUT SYMBOL FOR LATIN LETTERS + {0x1F525, 0x1F531, prID, gcSo}, // [13] FIRE..TRIDENT EMBLEM + {0x1F532, 0x1F549, prAL, gcSo}, // [24] BLACK SQUARE BUTTON..OM SYMBOL + {0x1F54A, 0x1F573, prID, gcSo}, // [42] DOVE OF PEACE..HOLE + {0x1F574, 0x1F575, prEB, gcSo}, // [2] MAN IN BUSINESS SUIT LEVITATING..SLEUTH OR SPY + {0x1F576, 0x1F579, prID, gcSo}, // [4] DARK SUNGLASSES..JOYSTICK + {0x1F57A, 0x1F57A, prEB, gcSo}, // MAN DANCING + {0x1F57B, 0x1F58F, prID, gcSo}, // [21] LEFT HAND TELEPHONE RECEIVER..TURNED OK HAND SIGN + {0x1F590, 0x1F590, prEB, gcSo}, // RAISED HAND WITH FINGERS SPLAYED + {0x1F591, 0x1F594, prID, gcSo}, // [4] REVERSED RAISED HAND WITH FINGERS SPLAYED..REVERSED VICTORY HAND + {0x1F595, 0x1F596, prEB, gcSo}, // [2] REVERSED HAND WITH MIDDLE FINGER EXTENDED..RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS + {0x1F597, 0x1F5D3, prID, gcSo}, // [61] WHITE DOWN POINTING LEFT HAND INDEX..SPIRAL CALENDAR PAD + {0x1F5D4, 0x1F5DB, prAL, gcSo}, // [8] DESKTOP WINDOW..DECREASE FONT SIZE SYMBOL + {0x1F5DC, 0x1F5F3, prID, gcSo}, // [24] COMPRESSION..BALLOT BOX WITH BALLOT + {0x1F5F4, 0x1F5F9, prAL, gcSo}, // [6] BALLOT SCRIPT X..BALLOT BOX WITH BOLD CHECK + {0x1F5FA, 0x1F5FF, prID, gcSo}, // [6] WORLD MAP..MOYAI + {0x1F600, 0x1F644, prID, gcSo}, // [69] GRINNING FACE..FACE WITH ROLLING EYES + {0x1F645, 0x1F647, prEB, gcSo}, // [3] FACE WITH NO GOOD GESTURE..PERSON BOWING DEEPLY + {0x1F648, 0x1F64A, prID, gcSo}, // [3] SEE-NO-EVIL MONKEY..SPEAK-NO-EVIL MONKEY + {0x1F64B, 0x1F64F, prEB, gcSo}, // [5] HAPPY PERSON RAISING ONE HAND..PERSON WITH FOLDED HANDS + {0x1F650, 0x1F675, prAL, gcSo}, // [38] NORTH WEST POINTING LEAF..SWASH AMPERSAND ORNAMENT + {0x1F676, 0x1F678, prQU, gcSo}, // [3] SANS-SERIF HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT..SANS-SERIF HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT + {0x1F679, 0x1F67B, prNS, gcSo}, // [3] HEAVY INTERROBANG ORNAMENT..HEAVY SANS-SERIF INTERROBANG ORNAMENT + {0x1F67C, 0x1F67F, prAL, gcSo}, // [4] VERY HEAVY SOLIDUS..REVERSE CHECKER BOARD + {0x1F680, 0x1F6A2, prID, gcSo}, // [35] ROCKET..SHIP + {0x1F6A3, 0x1F6A3, prEB, gcSo}, // ROWBOAT + {0x1F6A4, 0x1F6B3, prID, gcSo}, // [16] SPEEDBOAT..NO BICYCLES + {0x1F6B4, 0x1F6B6, prEB, gcSo}, // [3] BICYCLIST..PEDESTRIAN + {0x1F6B7, 0x1F6BF, prID, gcSo}, // [9] NO PEDESTRIANS..SHOWER + {0x1F6C0, 0x1F6C0, prEB, gcSo}, // BATH + {0x1F6C1, 0x1F6CB, prID, gcSo}, // [11] BATHTUB..COUCH AND LAMP + {0x1F6CC, 0x1F6CC, prEB, gcSo}, // SLEEPING ACCOMMODATION + {0x1F6CD, 0x1F6D7, prID, gcSo}, // [11] SHOPPING BAGS..ELEVATOR + {0x1F6D8, 0x1F6DB, prID, gcCn}, // [4] .. + {0x1F6DC, 0x1F6EC, prID, gcSo}, // [17] WIRELESS..AIRPLANE ARRIVING + {0x1F6ED, 0x1F6EF, prID, gcCn}, // [3] .. + {0x1F6F0, 0x1F6FC, prID, gcSo}, // [13] SATELLITE..ROLLER SKATE + {0x1F6FD, 0x1F6FF, prID, gcCn}, // [3] .. + {0x1F700, 0x1F773, prAL, gcSo}, // [116] ALCHEMICAL SYMBOL FOR QUINTESSENCE..ALCHEMICAL SYMBOL FOR HALF OUNCE + {0x1F774, 0x1F776, prID, gcSo}, // [3] LOT OF FORTUNE..LUNAR ECLIPSE + {0x1F777, 0x1F77A, prID, gcCn}, // [4] .. + {0x1F77B, 0x1F77F, prID, gcSo}, // [5] HAUMEA..ORCUS + {0x1F780, 0x1F7D4, prAL, gcSo}, // [85] BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE..HEAVY TWELVE POINTED PINWHEEL STAR + {0x1F7D5, 0x1F7D9, prID, gcSo}, // [5] CIRCLED TRIANGLE..NINE POINTED WHITE STAR + {0x1F7DA, 0x1F7DF, prID, gcCn}, // [6] .. + {0x1F7E0, 0x1F7EB, prID, gcSo}, // [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE + {0x1F7EC, 0x1F7EF, prID, gcCn}, // [4] .. + {0x1F7F0, 0x1F7F0, prID, gcSo}, // HEAVY EQUALS SIGN + {0x1F7F1, 0x1F7FF, prID, gcCn}, // [15] .. + {0x1F800, 0x1F80B, prAL, gcSo}, // [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD + {0x1F80C, 0x1F80F, prID, gcCn}, // [4] .. + {0x1F810, 0x1F847, prAL, gcSo}, // [56] LEFTWARDS ARROW WITH SMALL EQUILATERAL ARROWHEAD..DOWNWARDS HEAVY ARROW + {0x1F848, 0x1F84F, prID, gcCn}, // [8] .. + {0x1F850, 0x1F859, prAL, gcSo}, // [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW + {0x1F85A, 0x1F85F, prID, gcCn}, // [6] .. + {0x1F860, 0x1F887, prAL, gcSo}, // [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW + {0x1F888, 0x1F88F, prID, gcCn}, // [8] .. + {0x1F890, 0x1F8AD, prAL, gcSo}, // [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS + {0x1F8AE, 0x1F8AF, prID, gcCn}, // [2] .. + {0x1F8B0, 0x1F8B1, prID, gcSo}, // [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST + {0x1F8B2, 0x1F8FF, prID, gcCn}, // [78] .. + {0x1F900, 0x1F90B, prAL, gcSo}, // [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT + {0x1F90C, 0x1F90C, prEB, gcSo}, // PINCHED FINGERS + {0x1F90D, 0x1F90E, prID, gcSo}, // [2] WHITE HEART..BROWN HEART + {0x1F90F, 0x1F90F, prEB, gcSo}, // PINCHING HAND + {0x1F910, 0x1F917, prID, gcSo}, // [8] ZIPPER-MOUTH FACE..HUGGING FACE + {0x1F918, 0x1F91F, prEB, gcSo}, // [8] SIGN OF THE HORNS..I LOVE YOU HAND SIGN + {0x1F920, 0x1F925, prID, gcSo}, // [6] FACE WITH COWBOY HAT..LYING FACE + {0x1F926, 0x1F926, prEB, gcSo}, // FACE PALM + {0x1F927, 0x1F92F, prID, gcSo}, // [9] SNEEZING FACE..SHOCKED FACE WITH EXPLODING HEAD + {0x1F930, 0x1F939, prEB, gcSo}, // [10] PREGNANT WOMAN..JUGGLING + {0x1F93A, 0x1F93B, prID, gcSo}, // [2] FENCER..MODERN PENTATHLON + {0x1F93C, 0x1F93E, prEB, gcSo}, // [3] WRESTLERS..HANDBALL + {0x1F93F, 0x1F976, prID, gcSo}, // [56] DIVING MASK..FREEZING FACE + {0x1F977, 0x1F977, prEB, gcSo}, // NINJA + {0x1F978, 0x1F9B4, prID, gcSo}, // [61] DISGUISED FACE..BONE + {0x1F9B5, 0x1F9B6, prEB, gcSo}, // [2] LEG..FOOT + {0x1F9B7, 0x1F9B7, prID, gcSo}, // TOOTH + {0x1F9B8, 0x1F9B9, prEB, gcSo}, // [2] SUPERHERO..SUPERVILLAIN + {0x1F9BA, 0x1F9BA, prID, gcSo}, // SAFETY VEST + {0x1F9BB, 0x1F9BB, prEB, gcSo}, // EAR WITH HEARING AID + {0x1F9BC, 0x1F9CC, prID, gcSo}, // [17] MOTORIZED WHEELCHAIR..TROLL + {0x1F9CD, 0x1F9CF, prEB, gcSo}, // [3] STANDING PERSON..DEAF PERSON + {0x1F9D0, 0x1F9D0, prID, gcSo}, // FACE WITH MONOCLE + {0x1F9D1, 0x1F9DD, prEB, gcSo}, // [13] ADULT..ELF + {0x1F9DE, 0x1F9FF, prID, gcSo}, // [34] GENIE..NAZAR AMULET + {0x1FA00, 0x1FA53, prAL, gcSo}, // [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP + {0x1FA54, 0x1FA5F, prID, gcCn}, // [12] .. + {0x1FA60, 0x1FA6D, prID, gcSo}, // [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER + {0x1FA6E, 0x1FA6F, prID, gcCn}, // [2] .. + {0x1FA70, 0x1FA7C, prID, gcSo}, // [13] BALLET SHOES..CRUTCH + {0x1FA7D, 0x1FA7F, prID, gcCn}, // [3] .. + {0x1FA80, 0x1FA88, prID, gcSo}, // [9] YO-YO..FLUTE + {0x1FA89, 0x1FA8F, prID, gcCn}, // [7] .. + {0x1FA90, 0x1FABD, prID, gcSo}, // [46] RINGED PLANET..WING + {0x1FABE, 0x1FABE, prID, gcCn}, // + {0x1FABF, 0x1FAC2, prID, gcSo}, // [4] GOOSE..PEOPLE HUGGING + {0x1FAC3, 0x1FAC5, prEB, gcSo}, // [3] PREGNANT MAN..PERSON WITH CROWN + {0x1FAC6, 0x1FACD, prID, gcCn}, // [8] .. + {0x1FACE, 0x1FADB, prID, gcSo}, // [14] MOOSE..PEA POD + {0x1FADC, 0x1FADF, prID, gcCn}, // [4] .. + {0x1FAE0, 0x1FAE8, prID, gcSo}, // [9] MELTING FACE..SHAKING FACE + {0x1FAE9, 0x1FAEF, prID, gcCn}, // [7] .. + {0x1FAF0, 0x1FAF8, prEB, gcSo}, // [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND + {0x1FAF9, 0x1FAFF, prID, gcCn}, // [7] .. + {0x1FB00, 0x1FB92, prAL, gcSo}, // [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK + {0x1FB94, 0x1FBCA, prAL, gcSo}, // [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON + {0x1FBF0, 0x1FBF9, prNU, gcNd}, // [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE + {0x1FC00, 0x1FFFD, prID, gcCn}, // [1022] .. + {0x20000, 0x2A6DF, prID, gcLo}, // [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF + {0x2A6E0, 0x2A6FF, prID, gcCn}, // [32] .. + {0x2A700, 0x2B739, prID, gcLo}, // [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 + {0x2B73A, 0x2B73F, prID, gcCn}, // [6] .. + {0x2B740, 0x2B81D, prID, gcLo}, // [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D + {0x2B81E, 0x2B81F, prID, gcCn}, // [2] .. + {0x2B820, 0x2CEA1, prID, gcLo}, // [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 + {0x2CEA2, 0x2CEAF, prID, gcCn}, // [14] .. + {0x2CEB0, 0x2EBE0, prID, gcLo}, // [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 + {0x2EBE1, 0x2F7FF, prID, gcCn}, // [3103] .. + {0x2F800, 0x2FA1D, prID, gcLo}, // [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + {0x2FA1E, 0x2FA1F, prID, gcCn}, // [2] .. + {0x2FA20, 0x2FFFD, prID, gcCn}, // [1502] .. + {0x30000, 0x3134A, prID, gcLo}, // [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A + {0x3134B, 0x3134F, prID, gcCn}, // [5] .. + {0x31350, 0x323AF, prID, gcLo}, // [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF + {0x323B0, 0x3FFFD, prID, gcCn}, // [56398] .. + {0xE0001, 0xE0001, prCM, gcCf}, // LANGUAGE TAG + {0xE0020, 0xE007F, prCM, gcCf}, // [96] TAG SPACE..CANCEL TAG + {0xE0100, 0xE01EF, prCM, gcMn}, // [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + {0xF0000, 0xFFFFD, prXX, gcCo}, // [65534] .. + {0x100000, 0x10FFFD, prXX, gcCo}, // [65534] .. +} diff --git a/vendor/github.com/rivo/uniseg/linerules.go b/vendor/github.com/rivo/uniseg/linerules.go new file mode 100644 index 0000000000..7708ae0fbe --- /dev/null +++ b/vendor/github.com/rivo/uniseg/linerules.go @@ -0,0 +1,626 @@ +package uniseg + +import "unicode/utf8" + +// The states of the line break parser. +const ( + lbAny = iota + lbBK + lbCR + lbLF + lbNL + lbSP + lbZW + lbWJ + lbGL + lbBA + lbHY + lbCL + lbCP + lbEX + lbIS + lbSY + lbOP + lbQU + lbQUSP + lbNS + lbCLCPSP + lbB2 + lbB2SP + lbCB + lbBB + lbLB21a + lbHL + lbAL + lbNU + lbPR + lbEB + lbIDEM + lbNUNU + lbNUSY + lbNUIS + lbNUCL + lbNUCP + lbPO + lbJL + lbJV + lbJT + lbH2 + lbH3 + lbOddRI + lbEvenRI + lbExtPicCn + lbZWJBit = 64 + lbCPeaFWHBit = 128 +) + +// These constants define whether a given text may be broken into the next line. +// If the break is optional (LineCanBreak), you may choose to break or not based +// on your own criteria, for example, if the text has reached the available +// width. +const ( + LineDontBreak = iota // You may not break the line here. + LineCanBreak // You may or may not break the line here. + LineMustBreak // You must break the line here. +) + +// lbTransitions implements the line break parser's state transitions. It's +// anologous to [grTransitions], see comments there for details. +// +// Unicode version 15.0.0. +func lbTransitions(state, prop int) (newState, lineBreak, rule int) { + switch uint64(state) | uint64(prop)<<32 { + // LB4. + case lbBK | prAny<<32: + return lbAny, LineMustBreak, 40 + + // LB5. + case lbCR | prLF<<32: + return lbLF, LineDontBreak, 50 + case lbCR | prAny<<32: + return lbAny, LineMustBreak, 50 + case lbLF | prAny<<32: + return lbAny, LineMustBreak, 50 + case lbNL | prAny<<32: + return lbAny, LineMustBreak, 50 + + // LB6. + case lbAny | prBK<<32: + return lbBK, LineDontBreak, 60 + case lbAny | prCR<<32: + return lbCR, LineDontBreak, 60 + case lbAny | prLF<<32: + return lbLF, LineDontBreak, 60 + case lbAny | prNL<<32: + return lbNL, LineDontBreak, 60 + + // LB7. + case lbAny | prSP<<32: + return lbSP, LineDontBreak, 70 + case lbAny | prZW<<32: + return lbZW, LineDontBreak, 70 + + // LB8. + case lbZW | prSP<<32: + return lbZW, LineDontBreak, 70 + case lbZW | prAny<<32: + return lbAny, LineCanBreak, 80 + + // LB11. + case lbAny | prWJ<<32: + return lbWJ, LineDontBreak, 110 + case lbWJ | prAny<<32: + return lbAny, LineDontBreak, 110 + + // LB12. + case lbAny | prGL<<32: + return lbGL, LineCanBreak, 310 + case lbGL | prAny<<32: + return lbAny, LineDontBreak, 120 + + // LB13 (simple transitions). + case lbAny | prCL<<32: + return lbCL, LineCanBreak, 310 + case lbAny | prCP<<32: + return lbCP, LineCanBreak, 310 + case lbAny | prEX<<32: + return lbEX, LineDontBreak, 130 + case lbAny | prIS<<32: + return lbIS, LineCanBreak, 310 + case lbAny | prSY<<32: + return lbSY, LineCanBreak, 310 + + // LB14. + case lbAny | prOP<<32: + return lbOP, LineCanBreak, 310 + case lbOP | prSP<<32: + return lbOP, LineDontBreak, 70 + case lbOP | prAny<<32: + return lbAny, LineDontBreak, 140 + + // LB15. + case lbQU | prSP<<32: + return lbQUSP, LineDontBreak, 70 + case lbQU | prOP<<32: + return lbOP, LineDontBreak, 150 + case lbQUSP | prOP<<32: + return lbOP, LineDontBreak, 150 + + // LB16. + case lbCL | prSP<<32: + return lbCLCPSP, LineDontBreak, 70 + case lbNUCL | prSP<<32: + return lbCLCPSP, LineDontBreak, 70 + case lbCP | prSP<<32: + return lbCLCPSP, LineDontBreak, 70 + case lbNUCP | prSP<<32: + return lbCLCPSP, LineDontBreak, 70 + case lbCL | prNS<<32: + return lbNS, LineDontBreak, 160 + case lbNUCL | prNS<<32: + return lbNS, LineDontBreak, 160 + case lbCP | prNS<<32: + return lbNS, LineDontBreak, 160 + case lbNUCP | prNS<<32: + return lbNS, LineDontBreak, 160 + case lbCLCPSP | prNS<<32: + return lbNS, LineDontBreak, 160 + + // LB17. + case lbAny | prB2<<32: + return lbB2, LineCanBreak, 310 + case lbB2 | prSP<<32: + return lbB2SP, LineDontBreak, 70 + case lbB2 | prB2<<32: + return lbB2, LineDontBreak, 170 + case lbB2SP | prB2<<32: + return lbB2, LineDontBreak, 170 + + // LB18. + case lbSP | prAny<<32: + return lbAny, LineCanBreak, 180 + case lbQUSP | prAny<<32: + return lbAny, LineCanBreak, 180 + case lbCLCPSP | prAny<<32: + return lbAny, LineCanBreak, 180 + case lbB2SP | prAny<<32: + return lbAny, LineCanBreak, 180 + + // LB19. + case lbAny | prQU<<32: + return lbQU, LineDontBreak, 190 + case lbQU | prAny<<32: + return lbAny, LineDontBreak, 190 + + // LB20. + case lbAny | prCB<<32: + return lbCB, LineCanBreak, 200 + case lbCB | prAny<<32: + return lbAny, LineCanBreak, 200 + + // LB21. + case lbAny | prBA<<32: + return lbBA, LineDontBreak, 210 + case lbAny | prHY<<32: + return lbHY, LineDontBreak, 210 + case lbAny | prNS<<32: + return lbNS, LineDontBreak, 210 + case lbAny | prBB<<32: + return lbBB, LineCanBreak, 310 + case lbBB | prAny<<32: + return lbAny, LineDontBreak, 210 + + // LB21a. + case lbAny | prHL<<32: + return lbHL, LineCanBreak, 310 + case lbHL | prHY<<32: + return lbLB21a, LineDontBreak, 210 + case lbHL | prBA<<32: + return lbLB21a, LineDontBreak, 210 + case lbLB21a | prAny<<32: + return lbAny, LineDontBreak, 211 + + // LB21b. + case lbSY | prHL<<32: + return lbHL, LineDontBreak, 212 + case lbNUSY | prHL<<32: + return lbHL, LineDontBreak, 212 + + // LB22. + case lbAny | prIN<<32: + return lbAny, LineDontBreak, 220 + + // LB23. + case lbAny | prAL<<32: + return lbAL, LineCanBreak, 310 + case lbAny | prNU<<32: + return lbNU, LineCanBreak, 310 + case lbAL | prNU<<32: + return lbNU, LineDontBreak, 230 + case lbHL | prNU<<32: + return lbNU, LineDontBreak, 230 + case lbNU | prAL<<32: + return lbAL, LineDontBreak, 230 + case lbNU | prHL<<32: + return lbHL, LineDontBreak, 230 + case lbNUNU | prAL<<32: + return lbAL, LineDontBreak, 230 + case lbNUNU | prHL<<32: + return lbHL, LineDontBreak, 230 + + // LB23a. + case lbAny | prPR<<32: + return lbPR, LineCanBreak, 310 + case lbAny | prID<<32: + return lbIDEM, LineCanBreak, 310 + case lbAny | prEB<<32: + return lbEB, LineCanBreak, 310 + case lbAny | prEM<<32: + return lbIDEM, LineCanBreak, 310 + case lbPR | prID<<32: + return lbIDEM, LineDontBreak, 231 + case lbPR | prEB<<32: + return lbEB, LineDontBreak, 231 + case lbPR | prEM<<32: + return lbIDEM, LineDontBreak, 231 + case lbIDEM | prPO<<32: + return lbPO, LineDontBreak, 231 + case lbEB | prPO<<32: + return lbPO, LineDontBreak, 231 + + // LB24. + case lbAny | prPO<<32: + return lbPO, LineCanBreak, 310 + case lbPR | prAL<<32: + return lbAL, LineDontBreak, 240 + case lbPR | prHL<<32: + return lbHL, LineDontBreak, 240 + case lbPO | prAL<<32: + return lbAL, LineDontBreak, 240 + case lbPO | prHL<<32: + return lbHL, LineDontBreak, 240 + case lbAL | prPR<<32: + return lbPR, LineDontBreak, 240 + case lbAL | prPO<<32: + return lbPO, LineDontBreak, 240 + case lbHL | prPR<<32: + return lbPR, LineDontBreak, 240 + case lbHL | prPO<<32: + return lbPO, LineDontBreak, 240 + + // LB25 (simple transitions). + case lbPR | prNU<<32: + return lbNU, LineDontBreak, 250 + case lbPO | prNU<<32: + return lbNU, LineDontBreak, 250 + case lbOP | prNU<<32: + return lbNU, LineDontBreak, 250 + case lbHY | prNU<<32: + return lbNU, LineDontBreak, 250 + case lbNU | prNU<<32: + return lbNUNU, LineDontBreak, 250 + case lbNU | prSY<<32: + return lbNUSY, LineDontBreak, 250 + case lbNU | prIS<<32: + return lbNUIS, LineDontBreak, 250 + case lbNUNU | prNU<<32: + return lbNUNU, LineDontBreak, 250 + case lbNUNU | prSY<<32: + return lbNUSY, LineDontBreak, 250 + case lbNUNU | prIS<<32: + return lbNUIS, LineDontBreak, 250 + case lbNUSY | prNU<<32: + return lbNUNU, LineDontBreak, 250 + case lbNUSY | prSY<<32: + return lbNUSY, LineDontBreak, 250 + case lbNUSY | prIS<<32: + return lbNUIS, LineDontBreak, 250 + case lbNUIS | prNU<<32: + return lbNUNU, LineDontBreak, 250 + case lbNUIS | prSY<<32: + return lbNUSY, LineDontBreak, 250 + case lbNUIS | prIS<<32: + return lbNUIS, LineDontBreak, 250 + case lbNU | prCL<<32: + return lbNUCL, LineDontBreak, 250 + case lbNU | prCP<<32: + return lbNUCP, LineDontBreak, 250 + case lbNUNU | prCL<<32: + return lbNUCL, LineDontBreak, 250 + case lbNUNU | prCP<<32: + return lbNUCP, LineDontBreak, 250 + case lbNUSY | prCL<<32: + return lbNUCL, LineDontBreak, 250 + case lbNUSY | prCP<<32: + return lbNUCP, LineDontBreak, 250 + case lbNUIS | prCL<<32: + return lbNUCL, LineDontBreak, 250 + case lbNUIS | prCP<<32: + return lbNUCP, LineDontBreak, 250 + case lbNU | prPO<<32: + return lbPO, LineDontBreak, 250 + case lbNUNU | prPO<<32: + return lbPO, LineDontBreak, 250 + case lbNUSY | prPO<<32: + return lbPO, LineDontBreak, 250 + case lbNUIS | prPO<<32: + return lbPO, LineDontBreak, 250 + case lbNUCL | prPO<<32: + return lbPO, LineDontBreak, 250 + case lbNUCP | prPO<<32: + return lbPO, LineDontBreak, 250 + case lbNU | prPR<<32: + return lbPR, LineDontBreak, 250 + case lbNUNU | prPR<<32: + return lbPR, LineDontBreak, 250 + case lbNUSY | prPR<<32: + return lbPR, LineDontBreak, 250 + case lbNUIS | prPR<<32: + return lbPR, LineDontBreak, 250 + case lbNUCL | prPR<<32: + return lbPR, LineDontBreak, 250 + case lbNUCP | prPR<<32: + return lbPR, LineDontBreak, 250 + + // LB26. + case lbAny | prJL<<32: + return lbJL, LineCanBreak, 310 + case lbAny | prJV<<32: + return lbJV, LineCanBreak, 310 + case lbAny | prJT<<32: + return lbJT, LineCanBreak, 310 + case lbAny | prH2<<32: + return lbH2, LineCanBreak, 310 + case lbAny | prH3<<32: + return lbH3, LineCanBreak, 310 + case lbJL | prJL<<32: + return lbJL, LineDontBreak, 260 + case lbJL | prJV<<32: + return lbJV, LineDontBreak, 260 + case lbJL | prH2<<32: + return lbH2, LineDontBreak, 260 + case lbJL | prH3<<32: + return lbH3, LineDontBreak, 260 + case lbJV | prJV<<32: + return lbJV, LineDontBreak, 260 + case lbJV | prJT<<32: + return lbJT, LineDontBreak, 260 + case lbH2 | prJV<<32: + return lbJV, LineDontBreak, 260 + case lbH2 | prJT<<32: + return lbJT, LineDontBreak, 260 + case lbJT | prJT<<32: + return lbJT, LineDontBreak, 260 + case lbH3 | prJT<<32: + return lbJT, LineDontBreak, 260 + + // LB27. + case lbJL | prPO<<32: + return lbPO, LineDontBreak, 270 + case lbJV | prPO<<32: + return lbPO, LineDontBreak, 270 + case lbJT | prPO<<32: + return lbPO, LineDontBreak, 270 + case lbH2 | prPO<<32: + return lbPO, LineDontBreak, 270 + case lbH3 | prPO<<32: + return lbPO, LineDontBreak, 270 + case lbPR | prJL<<32: + return lbJL, LineDontBreak, 270 + case lbPR | prJV<<32: + return lbJV, LineDontBreak, 270 + case lbPR | prJT<<32: + return lbJT, LineDontBreak, 270 + case lbPR | prH2<<32: + return lbH2, LineDontBreak, 270 + case lbPR | prH3<<32: + return lbH3, LineDontBreak, 270 + + // LB28. + case lbAL | prAL<<32: + return lbAL, LineDontBreak, 280 + case lbAL | prHL<<32: + return lbHL, LineDontBreak, 280 + case lbHL | prAL<<32: + return lbAL, LineDontBreak, 280 + case lbHL | prHL<<32: + return lbHL, LineDontBreak, 280 + + // LB29. + case lbIS | prAL<<32: + return lbAL, LineDontBreak, 290 + case lbIS | prHL<<32: + return lbHL, LineDontBreak, 290 + case lbNUIS | prAL<<32: + return lbAL, LineDontBreak, 290 + case lbNUIS | prHL<<32: + return lbHL, LineDontBreak, 290 + + default: + return -1, -1, -1 + } +} + +// transitionLineBreakState determines the new state of the line break parser +// given the current state and the next code point. It also returns the type of +// line break: LineDontBreak, LineCanBreak, or LineMustBreak. If more than one +// code point is needed to determine the new state, the byte slice or the string +// starting after rune "r" can be used (whichever is not nil or empty) for +// further lookups. +func transitionLineBreakState(state int, r rune, b []byte, str string) (newState int, lineBreak int) { + // Determine the property of the next character. + nextProperty, generalCategory := propertyLineBreak(r) + + // Prepare. + var forceNoBreak, isCPeaFWH bool + if state >= 0 && state&lbCPeaFWHBit != 0 { + isCPeaFWH = true // LB30: CP but ea is not F, W, or H. + state = state &^ lbCPeaFWHBit + } + if state >= 0 && state&lbZWJBit != 0 { + state = state &^ lbZWJBit // Extract zero-width joiner bit. + forceNoBreak = true // LB8a. + } + + defer func() { + // Transition into LB30. + if newState == lbCP || newState == lbNUCP { + ea := propertyEastAsianWidth(r) + if ea != prF && ea != prW && ea != prH { + newState |= lbCPeaFWHBit + } + } + + // Override break. + if forceNoBreak { + lineBreak = LineDontBreak + } + }() + + // LB1. + if nextProperty == prAI || nextProperty == prSG || nextProperty == prXX { + nextProperty = prAL + } else if nextProperty == prSA { + if generalCategory == gcMn || generalCategory == gcMc { + nextProperty = prCM + } else { + nextProperty = prAL + } + } else if nextProperty == prCJ { + nextProperty = prNS + } + + // Combining marks. + if nextProperty == prZWJ || nextProperty == prCM { + var bit int + if nextProperty == prZWJ { + bit = lbZWJBit + } + mustBreakState := state < 0 || state == lbBK || state == lbCR || state == lbLF || state == lbNL + if !mustBreakState && state != lbSP && state != lbZW && state != lbQUSP && state != lbCLCPSP && state != lbB2SP { + // LB9. + return state | bit, LineDontBreak + } else { + // LB10. + if mustBreakState { + return lbAL | bit, LineMustBreak + } + return lbAL | bit, LineCanBreak + } + } + + // Find the applicable transition in the table. + var rule int + newState, lineBreak, rule = lbTransitions(state, nextProperty) + if newState < 0 { + // No specific transition found. Try the less specific ones. + anyPropProp, anyPropLineBreak, anyPropRule := lbTransitions(state, prAny) + anyStateProp, anyStateLineBreak, anyStateRule := lbTransitions(lbAny, nextProperty) + if anyPropProp >= 0 && anyStateProp >= 0 { + // Both apply. We'll use a mix (see comments for grTransitions). + newState, lineBreak, rule = anyStateProp, anyStateLineBreak, anyStateRule + if anyPropRule < anyStateRule { + lineBreak, rule = anyPropLineBreak, anyPropRule + } + } else if anyPropProp >= 0 { + // We only have a specific state. + newState, lineBreak, rule = anyPropProp, anyPropLineBreak, anyPropRule + // This branch will probably never be reached because okAnyState will + // always be true given the current transition map. But we keep it here + // for future modifications to the transition map where this may not be + // true anymore. + } else if anyStateProp >= 0 { + // We only have a specific property. + newState, lineBreak, rule = anyStateProp, anyStateLineBreak, anyStateRule + } else { + // No known transition. LB31: ALL รท ALL. + newState, lineBreak, rule = lbAny, LineCanBreak, 310 + } + } + + // LB12a. + if rule > 121 && + nextProperty == prGL && + (state != lbSP && state != lbBA && state != lbHY && state != lbLB21a && state != lbQUSP && state != lbCLCPSP && state != lbB2SP) { + return lbGL, LineDontBreak + } + + // LB13. + if rule > 130 && state != lbNU && state != lbNUNU { + switch nextProperty { + case prCL: + return lbCL, LineDontBreak + case prCP: + return lbCP, LineDontBreak + case prIS: + return lbIS, LineDontBreak + case prSY: + return lbSY, LineDontBreak + } + } + + // LB25 (look ahead). + if rule > 250 && + (state == lbPR || state == lbPO) && + nextProperty == prOP || nextProperty == prHY { + var r rune + if b != nil { // Byte slice version. + r, _ = utf8.DecodeRune(b) + } else { // String version. + r, _ = utf8.DecodeRuneInString(str) + } + if r != utf8.RuneError { + pr, _ := propertyLineBreak(r) + if pr == prNU { + return lbNU, LineDontBreak + } + } + } + + // LB30 (part one). + if rule > 300 { + if (state == lbAL || state == lbHL || state == lbNU || state == lbNUNU) && nextProperty == prOP { + ea := propertyEastAsianWidth(r) + if ea != prF && ea != prW && ea != prH { + return lbOP, LineDontBreak + } + } else if isCPeaFWH { + switch nextProperty { + case prAL: + return lbAL, LineDontBreak + case prHL: + return lbHL, LineDontBreak + case prNU: + return lbNU, LineDontBreak + } + } + } + + // LB30a. + if newState == lbAny && nextProperty == prRI { + if state != lbOddRI && state != lbEvenRI { // Includes state == -1. + // Transition into the first RI. + return lbOddRI, lineBreak + } + if state == lbOddRI { + // Don't break pairs of Regional Indicators. + return lbEvenRI, LineDontBreak + } + return lbOddRI, lineBreak + } + + // LB30b. + if rule > 302 { + if nextProperty == prEM { + if state == lbEB || state == lbExtPicCn { + return prAny, LineDontBreak + } + } + graphemeProperty := propertyGraphemes(r) + if graphemeProperty == prExtendedPictographic && generalCategory == gcCn { + return lbExtPicCn, LineCanBreak + } + } + + return +} diff --git a/vendor/github.com/rivo/uniseg/properties.go b/vendor/github.com/rivo/uniseg/properties.go new file mode 100644 index 0000000000..6290e6810f --- /dev/null +++ b/vendor/github.com/rivo/uniseg/properties.go @@ -0,0 +1,208 @@ +package uniseg + +// The Unicode properties as used in the various parsers. Only the ones needed +// in the context of this package are included. +const ( + prXX = 0 // Same as prAny. + prAny = iota // prAny must be 0. + prPrepend // Grapheme properties must come first, to reduce the number of bits stored in the state vector. + prCR + prLF + prControl + prExtend + prRegionalIndicator + prSpacingMark + prL + prV + prT + prLV + prLVT + prZWJ + prExtendedPictographic + prNewline + prWSegSpace + prDoubleQuote + prSingleQuote + prMidNumLet + prNumeric + prMidLetter + prMidNum + prExtendNumLet + prALetter + prFormat + prHebrewLetter + prKatakana + prSp + prSTerm + prClose + prSContinue + prATerm + prUpper + prLower + prSep + prOLetter + prCM + prBA + prBK + prSP + prEX + prQU + prAL + prPR + prPO + prOP + prCP + prIS + prHY + prSY + prNU + prCL + prNL + prGL + prAI + prBB + prHL + prSA + prJL + prJV + prJT + prNS + prZW + prB2 + prIN + prWJ + prID + prEB + prCJ + prH2 + prH3 + prSG + prCB + prRI + prEM + prN + prNa + prA + prW + prH + prF + prEmojiPresentation +) + +// Unicode General Categories. Only the ones needed in the context of this +// package are included. +const ( + gcNone = iota // gcNone must be 0. + gcCc + gcZs + gcPo + gcSc + gcPs + gcPe + gcSm + gcPd + gcNd + gcLu + gcSk + gcPc + gcLl + gcSo + gcLo + gcPi + gcCf + gcNo + gcPf + gcLC + gcLm + gcMn + gcMe + gcMc + gcNl + gcZl + gcZp + gcCn + gcCs + gcCo +) + +// Special code points. +const ( + vs15 = 0xfe0e // Variation Selector-15 (text presentation) + vs16 = 0xfe0f // Variation Selector-16 (emoji presentation) +) + +// propertySearch performs a binary search on a property slice and returns the +// entry whose range (start = first array element, end = second array element) +// includes r, or an array of 0's if no such entry was found. +func propertySearch[E interface{ [3]int | [4]int }](dictionary []E, r rune) (result E) { + // Run a binary search. + from := 0 + to := len(dictionary) + for to > from { + middle := (from + to) / 2 + cpRange := dictionary[middle] + if int(r) < cpRange[0] { + to = middle + continue + } + if int(r) > cpRange[1] { + from = middle + 1 + continue + } + return cpRange + } + return +} + +// property returns the Unicode property value (see constants above) of the +// given code point. +func property(dictionary [][3]int, r rune) int { + return propertySearch(dictionary, r)[2] +} + +// propertyLineBreak returns the Unicode property value and General Category +// (see constants above) of the given code point, as listed in the line break +// code points table, while fast tracking ASCII digits and letters. +func propertyLineBreak(r rune) (property, generalCategory int) { + if r >= 'a' && r <= 'z' { + return prAL, gcLl + } + if r >= 'A' && r <= 'Z' { + return prAL, gcLu + } + if r >= '0' && r <= '9' { + return prNU, gcNd + } + entry := propertySearch(lineBreakCodePoints, r) + return entry[2], entry[3] +} + +// propertyGraphemes returns the Unicode grapheme cluster property value of the +// given code point while fast tracking ASCII characters. +func propertyGraphemes(r rune) int { + if r >= 0x20 && r <= 0x7e { + return prAny + } + if r == 0x0a { + return prLF + } + if r == 0x0d { + return prCR + } + if r >= 0 && r <= 0x1f || r == 0x7f { + return prControl + } + return property(graphemeCodePoints, r) +} + +// propertyEastAsianWidth returns the Unicode East Asian Width property value of +// the given code point while fast tracking ASCII characters. +func propertyEastAsianWidth(r rune) int { + if r >= 0x20 && r <= 0x7e { + return prNa + } + if r >= 0 && r <= 0x1f || r == 0x7f { + return prN + } + return property(eastAsianWidth, r) +} diff --git a/vendor/github.com/rivo/uniseg/sentence.go b/vendor/github.com/rivo/uniseg/sentence.go new file mode 100644 index 0000000000..adc2a35773 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/sentence.go @@ -0,0 +1,90 @@ +package uniseg + +import "unicode/utf8" + +// FirstSentence returns the first sentence found in the given byte slice +// according to the rules of [Unicode Standard Annex #29, Sentence Boundaries]. +// This function can be called continuously to extract all sentences from a byte +// slice, as illustrated in the example below. +// +// If you don't know the current state, for example when calling the function +// for the first time, you must pass -1. For consecutive calls, pass the state +// and rest slice returned by the previous call. +// +// The "rest" slice is the sub-slice of the original byte slice "b" starting +// after the last byte of the identified sentence. If the length of the "rest" +// slice is 0, the entire byte slice "b" has been processed. The "sentence" byte +// slice is the sub-slice of the input slice containing the identified sentence. +// +// Given an empty byte slice "b", the function returns nil values. +// +// [Unicode Standard Annex #29, Sentence Boundaries]: http://unicode.org/reports/tr29/#Sentence_Boundaries +func FirstSentence(b []byte, state int) (sentence, rest []byte, newState int) { + // An empty byte slice returns nothing. + if len(b) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRune(b) + if len(b) <= length { // If we're already past the end, there is nothing else to parse. + return b, nil, sbAny + } + + // If we don't know the state, determine it now. + if state < 0 { + state, _ = transitionSentenceBreakState(state, r, b[length:], "") + } + + // Transition until we find a boundary. + var boundary bool + for { + r, l := utf8.DecodeRune(b[length:]) + state, boundary = transitionSentenceBreakState(state, r, b[length+l:], "") + + if boundary { + return b[:length], b[length:], state + } + + length += l + if len(b) <= length { + return b, nil, sbAny + } + } +} + +// FirstSentenceInString is like [FirstSentence] but its input and outputs are +// strings. +func FirstSentenceInString(str string, state int) (sentence, rest string, newState int) { + // An empty byte slice returns nothing. + if len(str) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRuneInString(str) + if len(str) <= length { // If we're already past the end, there is nothing else to parse. + return str, "", sbAny + } + + // If we don't know the state, determine it now. + if state < 0 { + state, _ = transitionSentenceBreakState(state, r, nil, str[length:]) + } + + // Transition until we find a boundary. + var boundary bool + for { + r, l := utf8.DecodeRuneInString(str[length:]) + state, boundary = transitionSentenceBreakState(state, r, nil, str[length+l:]) + + if boundary { + return str[:length], str[length:], state + } + + length += l + if len(str) <= length { + return str, "", sbAny + } + } +} diff --git a/vendor/github.com/rivo/uniseg/sentenceproperties.go b/vendor/github.com/rivo/uniseg/sentenceproperties.go new file mode 100644 index 0000000000..67717ec1f3 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/sentenceproperties.go @@ -0,0 +1,2845 @@ +// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// sentenceBreakCodePoints are taken from +// https://www.unicode.org/Public/15.0.0/ucd/auxiliary/SentenceBreakProperty.txt +// and +// https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt +// ("Extended_Pictographic" only) +// on September 5, 2023. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var sentenceBreakCodePoints = [][3]int{ + {0x0009, 0x0009, prSp}, // Cc + {0x000A, 0x000A, prLF}, // Cc + {0x000B, 0x000C, prSp}, // Cc [2] .. + {0x000D, 0x000D, prCR}, // Cc + {0x0020, 0x0020, prSp}, // Zs SPACE + {0x0021, 0x0021, prSTerm}, // Po EXCLAMATION MARK + {0x0022, 0x0022, prClose}, // Po QUOTATION MARK + {0x0027, 0x0027, prClose}, // Po APOSTROPHE + {0x0028, 0x0028, prClose}, // Ps LEFT PARENTHESIS + {0x0029, 0x0029, prClose}, // Pe RIGHT PARENTHESIS + {0x002C, 0x002C, prSContinue}, // Po COMMA + {0x002D, 0x002D, prSContinue}, // Pd HYPHEN-MINUS + {0x002E, 0x002E, prATerm}, // Po FULL STOP + {0x0030, 0x0039, prNumeric}, // Nd [10] DIGIT ZERO..DIGIT NINE + {0x003A, 0x003A, prSContinue}, // Po COLON + {0x003F, 0x003F, prSTerm}, // Po QUESTION MARK + {0x0041, 0x005A, prUpper}, // L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z + {0x005B, 0x005B, prClose}, // Ps LEFT SQUARE BRACKET + {0x005D, 0x005D, prClose}, // Pe RIGHT SQUARE BRACKET + {0x0061, 0x007A, prLower}, // L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z + {0x007B, 0x007B, prClose}, // Ps LEFT CURLY BRACKET + {0x007D, 0x007D, prClose}, // Pe RIGHT CURLY BRACKET + {0x0085, 0x0085, prSep}, // Cc + {0x00A0, 0x00A0, prSp}, // Zs NO-BREAK SPACE + {0x00AA, 0x00AA, prLower}, // Lo FEMININE ORDINAL INDICATOR + {0x00AB, 0x00AB, prClose}, // Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00AD, 0x00AD, prFormat}, // Cf SOFT HYPHEN + {0x00B5, 0x00B5, prLower}, // L& MICRO SIGN + {0x00BA, 0x00BA, prLower}, // Lo MASCULINE ORDINAL INDICATOR + {0x00BB, 0x00BB, prClose}, // Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00C0, 0x00D6, prUpper}, // L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS + {0x00D8, 0x00DE, prUpper}, // L& [7] LATIN CAPITAL LETTER O WITH STROKE..LATIN CAPITAL LETTER THORN + {0x00DF, 0x00F6, prLower}, // L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS + {0x00F8, 0x00FF, prLower}, // L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS + {0x0100, 0x0100, prUpper}, // L& LATIN CAPITAL LETTER A WITH MACRON + {0x0101, 0x0101, prLower}, // L& LATIN SMALL LETTER A WITH MACRON + {0x0102, 0x0102, prUpper}, // L& LATIN CAPITAL LETTER A WITH BREVE + {0x0103, 0x0103, prLower}, // L& LATIN SMALL LETTER A WITH BREVE + {0x0104, 0x0104, prUpper}, // L& LATIN CAPITAL LETTER A WITH OGONEK + {0x0105, 0x0105, prLower}, // L& LATIN SMALL LETTER A WITH OGONEK + {0x0106, 0x0106, prUpper}, // L& LATIN CAPITAL LETTER C WITH ACUTE + {0x0107, 0x0107, prLower}, // L& LATIN SMALL LETTER C WITH ACUTE + {0x0108, 0x0108, prUpper}, // L& LATIN CAPITAL LETTER C WITH CIRCUMFLEX + {0x0109, 0x0109, prLower}, // L& LATIN SMALL LETTER C WITH CIRCUMFLEX + {0x010A, 0x010A, prUpper}, // L& LATIN CAPITAL LETTER C WITH DOT ABOVE + {0x010B, 0x010B, prLower}, // L& LATIN SMALL LETTER C WITH DOT ABOVE + {0x010C, 0x010C, prUpper}, // L& LATIN CAPITAL LETTER C WITH CARON + {0x010D, 0x010D, prLower}, // L& LATIN SMALL LETTER C WITH CARON + {0x010E, 0x010E, prUpper}, // L& LATIN CAPITAL LETTER D WITH CARON + {0x010F, 0x010F, prLower}, // L& LATIN SMALL LETTER D WITH CARON + {0x0110, 0x0110, prUpper}, // L& LATIN CAPITAL LETTER D WITH STROKE + {0x0111, 0x0111, prLower}, // L& LATIN SMALL LETTER D WITH STROKE + {0x0112, 0x0112, prUpper}, // L& LATIN CAPITAL LETTER E WITH MACRON + {0x0113, 0x0113, prLower}, // L& LATIN SMALL LETTER E WITH MACRON + {0x0114, 0x0114, prUpper}, // L& LATIN CAPITAL LETTER E WITH BREVE + {0x0115, 0x0115, prLower}, // L& LATIN SMALL LETTER E WITH BREVE + {0x0116, 0x0116, prUpper}, // L& LATIN CAPITAL LETTER E WITH DOT ABOVE + {0x0117, 0x0117, prLower}, // L& LATIN SMALL LETTER E WITH DOT ABOVE + {0x0118, 0x0118, prUpper}, // L& LATIN CAPITAL LETTER E WITH OGONEK + {0x0119, 0x0119, prLower}, // L& LATIN SMALL LETTER E WITH OGONEK + {0x011A, 0x011A, prUpper}, // L& LATIN CAPITAL LETTER E WITH CARON + {0x011B, 0x011B, prLower}, // L& LATIN SMALL LETTER E WITH CARON + {0x011C, 0x011C, prUpper}, // L& LATIN CAPITAL LETTER G WITH CIRCUMFLEX + {0x011D, 0x011D, prLower}, // L& LATIN SMALL LETTER G WITH CIRCUMFLEX + {0x011E, 0x011E, prUpper}, // L& LATIN CAPITAL LETTER G WITH BREVE + {0x011F, 0x011F, prLower}, // L& LATIN SMALL LETTER G WITH BREVE + {0x0120, 0x0120, prUpper}, // L& LATIN CAPITAL LETTER G WITH DOT ABOVE + {0x0121, 0x0121, prLower}, // L& LATIN SMALL LETTER G WITH DOT ABOVE + {0x0122, 0x0122, prUpper}, // L& LATIN CAPITAL LETTER G WITH CEDILLA + {0x0123, 0x0123, prLower}, // L& LATIN SMALL LETTER G WITH CEDILLA + {0x0124, 0x0124, prUpper}, // L& LATIN CAPITAL LETTER H WITH CIRCUMFLEX + {0x0125, 0x0125, prLower}, // L& LATIN SMALL LETTER H WITH CIRCUMFLEX + {0x0126, 0x0126, prUpper}, // L& LATIN CAPITAL LETTER H WITH STROKE + {0x0127, 0x0127, prLower}, // L& LATIN SMALL LETTER H WITH STROKE + {0x0128, 0x0128, prUpper}, // L& LATIN CAPITAL LETTER I WITH TILDE + {0x0129, 0x0129, prLower}, // L& LATIN SMALL LETTER I WITH TILDE + {0x012A, 0x012A, prUpper}, // L& LATIN CAPITAL LETTER I WITH MACRON + {0x012B, 0x012B, prLower}, // L& LATIN SMALL LETTER I WITH MACRON + {0x012C, 0x012C, prUpper}, // L& LATIN CAPITAL LETTER I WITH BREVE + {0x012D, 0x012D, prLower}, // L& LATIN SMALL LETTER I WITH BREVE + {0x012E, 0x012E, prUpper}, // L& LATIN CAPITAL LETTER I WITH OGONEK + {0x012F, 0x012F, prLower}, // L& LATIN SMALL LETTER I WITH OGONEK + {0x0130, 0x0130, prUpper}, // L& LATIN CAPITAL LETTER I WITH DOT ABOVE + {0x0131, 0x0131, prLower}, // L& LATIN SMALL LETTER DOTLESS I + {0x0132, 0x0132, prUpper}, // L& LATIN CAPITAL LIGATURE IJ + {0x0133, 0x0133, prLower}, // L& LATIN SMALL LIGATURE IJ + {0x0134, 0x0134, prUpper}, // L& LATIN CAPITAL LETTER J WITH CIRCUMFLEX + {0x0135, 0x0135, prLower}, // L& LATIN SMALL LETTER J WITH CIRCUMFLEX + {0x0136, 0x0136, prUpper}, // L& LATIN CAPITAL LETTER K WITH CEDILLA + {0x0137, 0x0138, prLower}, // L& [2] LATIN SMALL LETTER K WITH CEDILLA..LATIN SMALL LETTER KRA + {0x0139, 0x0139, prUpper}, // L& LATIN CAPITAL LETTER L WITH ACUTE + {0x013A, 0x013A, prLower}, // L& LATIN SMALL LETTER L WITH ACUTE + {0x013B, 0x013B, prUpper}, // L& LATIN CAPITAL LETTER L WITH CEDILLA + {0x013C, 0x013C, prLower}, // L& LATIN SMALL LETTER L WITH CEDILLA + {0x013D, 0x013D, prUpper}, // L& LATIN CAPITAL LETTER L WITH CARON + {0x013E, 0x013E, prLower}, // L& LATIN SMALL LETTER L WITH CARON + {0x013F, 0x013F, prUpper}, // L& LATIN CAPITAL LETTER L WITH MIDDLE DOT + {0x0140, 0x0140, prLower}, // L& LATIN SMALL LETTER L WITH MIDDLE DOT + {0x0141, 0x0141, prUpper}, // L& LATIN CAPITAL LETTER L WITH STROKE + {0x0142, 0x0142, prLower}, // L& LATIN SMALL LETTER L WITH STROKE + {0x0143, 0x0143, prUpper}, // L& LATIN CAPITAL LETTER N WITH ACUTE + {0x0144, 0x0144, prLower}, // L& LATIN SMALL LETTER N WITH ACUTE + {0x0145, 0x0145, prUpper}, // L& LATIN CAPITAL LETTER N WITH CEDILLA + {0x0146, 0x0146, prLower}, // L& LATIN SMALL LETTER N WITH CEDILLA + {0x0147, 0x0147, prUpper}, // L& LATIN CAPITAL LETTER N WITH CARON + {0x0148, 0x0149, prLower}, // L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + {0x014A, 0x014A, prUpper}, // L& LATIN CAPITAL LETTER ENG + {0x014B, 0x014B, prLower}, // L& LATIN SMALL LETTER ENG + {0x014C, 0x014C, prUpper}, // L& LATIN CAPITAL LETTER O WITH MACRON + {0x014D, 0x014D, prLower}, // L& LATIN SMALL LETTER O WITH MACRON + {0x014E, 0x014E, prUpper}, // L& LATIN CAPITAL LETTER O WITH BREVE + {0x014F, 0x014F, prLower}, // L& LATIN SMALL LETTER O WITH BREVE + {0x0150, 0x0150, prUpper}, // L& LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + {0x0151, 0x0151, prLower}, // L& LATIN SMALL LETTER O WITH DOUBLE ACUTE + {0x0152, 0x0152, prUpper}, // L& LATIN CAPITAL LIGATURE OE + {0x0153, 0x0153, prLower}, // L& LATIN SMALL LIGATURE OE + {0x0154, 0x0154, prUpper}, // L& LATIN CAPITAL LETTER R WITH ACUTE + {0x0155, 0x0155, prLower}, // L& LATIN SMALL LETTER R WITH ACUTE + {0x0156, 0x0156, prUpper}, // L& LATIN CAPITAL LETTER R WITH CEDILLA + {0x0157, 0x0157, prLower}, // L& LATIN SMALL LETTER R WITH CEDILLA + {0x0158, 0x0158, prUpper}, // L& LATIN CAPITAL LETTER R WITH CARON + {0x0159, 0x0159, prLower}, // L& LATIN SMALL LETTER R WITH CARON + {0x015A, 0x015A, prUpper}, // L& LATIN CAPITAL LETTER S WITH ACUTE + {0x015B, 0x015B, prLower}, // L& LATIN SMALL LETTER S WITH ACUTE + {0x015C, 0x015C, prUpper}, // L& LATIN CAPITAL LETTER S WITH CIRCUMFLEX + {0x015D, 0x015D, prLower}, // L& LATIN SMALL LETTER S WITH CIRCUMFLEX + {0x015E, 0x015E, prUpper}, // L& LATIN CAPITAL LETTER S WITH CEDILLA + {0x015F, 0x015F, prLower}, // L& LATIN SMALL LETTER S WITH CEDILLA + {0x0160, 0x0160, prUpper}, // L& LATIN CAPITAL LETTER S WITH CARON + {0x0161, 0x0161, prLower}, // L& LATIN SMALL LETTER S WITH CARON + {0x0162, 0x0162, prUpper}, // L& LATIN CAPITAL LETTER T WITH CEDILLA + {0x0163, 0x0163, prLower}, // L& LATIN SMALL LETTER T WITH CEDILLA + {0x0164, 0x0164, prUpper}, // L& LATIN CAPITAL LETTER T WITH CARON + {0x0165, 0x0165, prLower}, // L& LATIN SMALL LETTER T WITH CARON + {0x0166, 0x0166, prUpper}, // L& LATIN CAPITAL LETTER T WITH STROKE + {0x0167, 0x0167, prLower}, // L& LATIN SMALL LETTER T WITH STROKE + {0x0168, 0x0168, prUpper}, // L& LATIN CAPITAL LETTER U WITH TILDE + {0x0169, 0x0169, prLower}, // L& LATIN SMALL LETTER U WITH TILDE + {0x016A, 0x016A, prUpper}, // L& LATIN CAPITAL LETTER U WITH MACRON + {0x016B, 0x016B, prLower}, // L& LATIN SMALL LETTER U WITH MACRON + {0x016C, 0x016C, prUpper}, // L& LATIN CAPITAL LETTER U WITH BREVE + {0x016D, 0x016D, prLower}, // L& LATIN SMALL LETTER U WITH BREVE + {0x016E, 0x016E, prUpper}, // L& LATIN CAPITAL LETTER U WITH RING ABOVE + {0x016F, 0x016F, prLower}, // L& LATIN SMALL LETTER U WITH RING ABOVE + {0x0170, 0x0170, prUpper}, // L& LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + {0x0171, 0x0171, prLower}, // L& LATIN SMALL LETTER U WITH DOUBLE ACUTE + {0x0172, 0x0172, prUpper}, // L& LATIN CAPITAL LETTER U WITH OGONEK + {0x0173, 0x0173, prLower}, // L& LATIN SMALL LETTER U WITH OGONEK + {0x0174, 0x0174, prUpper}, // L& LATIN CAPITAL LETTER W WITH CIRCUMFLEX + {0x0175, 0x0175, prLower}, // L& LATIN SMALL LETTER W WITH CIRCUMFLEX + {0x0176, 0x0176, prUpper}, // L& LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + {0x0177, 0x0177, prLower}, // L& LATIN SMALL LETTER Y WITH CIRCUMFLEX + {0x0178, 0x0179, prUpper}, // L& [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE + {0x017A, 0x017A, prLower}, // L& LATIN SMALL LETTER Z WITH ACUTE + {0x017B, 0x017B, prUpper}, // L& LATIN CAPITAL LETTER Z WITH DOT ABOVE + {0x017C, 0x017C, prLower}, // L& LATIN SMALL LETTER Z WITH DOT ABOVE + {0x017D, 0x017D, prUpper}, // L& LATIN CAPITAL LETTER Z WITH CARON + {0x017E, 0x0180, prLower}, // L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE + {0x0181, 0x0182, prUpper}, // L& [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR + {0x0183, 0x0183, prLower}, // L& LATIN SMALL LETTER B WITH TOPBAR + {0x0184, 0x0184, prUpper}, // L& LATIN CAPITAL LETTER TONE SIX + {0x0185, 0x0185, prLower}, // L& LATIN SMALL LETTER TONE SIX + {0x0186, 0x0187, prUpper}, // L& [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK + {0x0188, 0x0188, prLower}, // L& LATIN SMALL LETTER C WITH HOOK + {0x0189, 0x018B, prUpper}, // L& [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR + {0x018C, 0x018D, prLower}, // L& [2] LATIN SMALL LETTER D WITH TOPBAR..LATIN SMALL LETTER TURNED DELTA + {0x018E, 0x0191, prUpper}, // L& [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK + {0x0192, 0x0192, prLower}, // L& LATIN SMALL LETTER F WITH HOOK + {0x0193, 0x0194, prUpper}, // L& [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA + {0x0195, 0x0195, prLower}, // L& LATIN SMALL LETTER HV + {0x0196, 0x0198, prUpper}, // L& [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK + {0x0199, 0x019B, prLower}, // L& [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE + {0x019C, 0x019D, prUpper}, // L& [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK + {0x019E, 0x019E, prLower}, // L& LATIN SMALL LETTER N WITH LONG RIGHT LEG + {0x019F, 0x01A0, prUpper}, // L& [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN + {0x01A1, 0x01A1, prLower}, // L& LATIN SMALL LETTER O WITH HORN + {0x01A2, 0x01A2, prUpper}, // L& LATIN CAPITAL LETTER OI + {0x01A3, 0x01A3, prLower}, // L& LATIN SMALL LETTER OI + {0x01A4, 0x01A4, prUpper}, // L& LATIN CAPITAL LETTER P WITH HOOK + {0x01A5, 0x01A5, prLower}, // L& LATIN SMALL LETTER P WITH HOOK + {0x01A6, 0x01A7, prUpper}, // L& [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO + {0x01A8, 0x01A8, prLower}, // L& LATIN SMALL LETTER TONE TWO + {0x01A9, 0x01A9, prUpper}, // L& LATIN CAPITAL LETTER ESH + {0x01AA, 0x01AB, prLower}, // L& [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK + {0x01AC, 0x01AC, prUpper}, // L& LATIN CAPITAL LETTER T WITH HOOK + {0x01AD, 0x01AD, prLower}, // L& LATIN SMALL LETTER T WITH HOOK + {0x01AE, 0x01AF, prUpper}, // L& [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN + {0x01B0, 0x01B0, prLower}, // L& LATIN SMALL LETTER U WITH HORN + {0x01B1, 0x01B3, prUpper}, // L& [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK + {0x01B4, 0x01B4, prLower}, // L& LATIN SMALL LETTER Y WITH HOOK + {0x01B5, 0x01B5, prUpper}, // L& LATIN CAPITAL LETTER Z WITH STROKE + {0x01B6, 0x01B6, prLower}, // L& LATIN SMALL LETTER Z WITH STROKE + {0x01B7, 0x01B8, prUpper}, // L& [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED + {0x01B9, 0x01BA, prLower}, // L& [2] LATIN SMALL LETTER EZH REVERSED..LATIN SMALL LETTER EZH WITH TAIL + {0x01BB, 0x01BB, prOLetter}, // Lo LATIN LETTER TWO WITH STROKE + {0x01BC, 0x01BC, prUpper}, // L& LATIN CAPITAL LETTER TONE FIVE + {0x01BD, 0x01BF, prLower}, // L& [3] LATIN SMALL LETTER TONE FIVE..LATIN LETTER WYNN + {0x01C0, 0x01C3, prOLetter}, // Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK + {0x01C4, 0x01C5, prUpper}, // L& [2] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + {0x01C6, 0x01C6, prLower}, // L& LATIN SMALL LETTER DZ WITH CARON + {0x01C7, 0x01C8, prUpper}, // L& [2] LATIN CAPITAL LETTER LJ..LATIN CAPITAL LETTER L WITH SMALL LETTER J + {0x01C9, 0x01C9, prLower}, // L& LATIN SMALL LETTER LJ + {0x01CA, 0x01CB, prUpper}, // L& [2] LATIN CAPITAL LETTER NJ..LATIN CAPITAL LETTER N WITH SMALL LETTER J + {0x01CC, 0x01CC, prLower}, // L& LATIN SMALL LETTER NJ + {0x01CD, 0x01CD, prUpper}, // L& LATIN CAPITAL LETTER A WITH CARON + {0x01CE, 0x01CE, prLower}, // L& LATIN SMALL LETTER A WITH CARON + {0x01CF, 0x01CF, prUpper}, // L& LATIN CAPITAL LETTER I WITH CARON + {0x01D0, 0x01D0, prLower}, // L& LATIN SMALL LETTER I WITH CARON + {0x01D1, 0x01D1, prUpper}, // L& LATIN CAPITAL LETTER O WITH CARON + {0x01D2, 0x01D2, prLower}, // L& LATIN SMALL LETTER O WITH CARON + {0x01D3, 0x01D3, prUpper}, // L& LATIN CAPITAL LETTER U WITH CARON + {0x01D4, 0x01D4, prLower}, // L& LATIN SMALL LETTER U WITH CARON + {0x01D5, 0x01D5, prUpper}, // L& LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + {0x01D6, 0x01D6, prLower}, // L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON + {0x01D7, 0x01D7, prUpper}, // L& LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + {0x01D8, 0x01D8, prLower}, // L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE + {0x01D9, 0x01D9, prUpper}, // L& LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + {0x01DA, 0x01DA, prLower}, // L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON + {0x01DB, 0x01DB, prUpper}, // L& LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + {0x01DC, 0x01DD, prLower}, // L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E + {0x01DE, 0x01DE, prUpper}, // L& LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + {0x01DF, 0x01DF, prLower}, // L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON + {0x01E0, 0x01E0, prUpper}, // L& LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + {0x01E1, 0x01E1, prLower}, // L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON + {0x01E2, 0x01E2, prUpper}, // L& LATIN CAPITAL LETTER AE WITH MACRON + {0x01E3, 0x01E3, prLower}, // L& LATIN SMALL LETTER AE WITH MACRON + {0x01E4, 0x01E4, prUpper}, // L& LATIN CAPITAL LETTER G WITH STROKE + {0x01E5, 0x01E5, prLower}, // L& LATIN SMALL LETTER G WITH STROKE + {0x01E6, 0x01E6, prUpper}, // L& LATIN CAPITAL LETTER G WITH CARON + {0x01E7, 0x01E7, prLower}, // L& LATIN SMALL LETTER G WITH CARON + {0x01E8, 0x01E8, prUpper}, // L& LATIN CAPITAL LETTER K WITH CARON + {0x01E9, 0x01E9, prLower}, // L& LATIN SMALL LETTER K WITH CARON + {0x01EA, 0x01EA, prUpper}, // L& LATIN CAPITAL LETTER O WITH OGONEK + {0x01EB, 0x01EB, prLower}, // L& LATIN SMALL LETTER O WITH OGONEK + {0x01EC, 0x01EC, prUpper}, // L& LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + {0x01ED, 0x01ED, prLower}, // L& LATIN SMALL LETTER O WITH OGONEK AND MACRON + {0x01EE, 0x01EE, prUpper}, // L& LATIN CAPITAL LETTER EZH WITH CARON + {0x01EF, 0x01F0, prLower}, // L& [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON + {0x01F1, 0x01F2, prUpper}, // L& [2] LATIN CAPITAL LETTER DZ..LATIN CAPITAL LETTER D WITH SMALL LETTER Z + {0x01F3, 0x01F3, prLower}, // L& LATIN SMALL LETTER DZ + {0x01F4, 0x01F4, prUpper}, // L& LATIN CAPITAL LETTER G WITH ACUTE + {0x01F5, 0x01F5, prLower}, // L& LATIN SMALL LETTER G WITH ACUTE + {0x01F6, 0x01F8, prUpper}, // L& [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE + {0x01F9, 0x01F9, prLower}, // L& LATIN SMALL LETTER N WITH GRAVE + {0x01FA, 0x01FA, prUpper}, // L& LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + {0x01FB, 0x01FB, prLower}, // L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE + {0x01FC, 0x01FC, prUpper}, // L& LATIN CAPITAL LETTER AE WITH ACUTE + {0x01FD, 0x01FD, prLower}, // L& LATIN SMALL LETTER AE WITH ACUTE + {0x01FE, 0x01FE, prUpper}, // L& LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + {0x01FF, 0x01FF, prLower}, // L& LATIN SMALL LETTER O WITH STROKE AND ACUTE + {0x0200, 0x0200, prUpper}, // L& LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + {0x0201, 0x0201, prLower}, // L& LATIN SMALL LETTER A WITH DOUBLE GRAVE + {0x0202, 0x0202, prUpper}, // L& LATIN CAPITAL LETTER A WITH INVERTED BREVE + {0x0203, 0x0203, prLower}, // L& LATIN SMALL LETTER A WITH INVERTED BREVE + {0x0204, 0x0204, prUpper}, // L& LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + {0x0205, 0x0205, prLower}, // L& LATIN SMALL LETTER E WITH DOUBLE GRAVE + {0x0206, 0x0206, prUpper}, // L& LATIN CAPITAL LETTER E WITH INVERTED BREVE + {0x0207, 0x0207, prLower}, // L& LATIN SMALL LETTER E WITH INVERTED BREVE + {0x0208, 0x0208, prUpper}, // L& LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + {0x0209, 0x0209, prLower}, // L& LATIN SMALL LETTER I WITH DOUBLE GRAVE + {0x020A, 0x020A, prUpper}, // L& LATIN CAPITAL LETTER I WITH INVERTED BREVE + {0x020B, 0x020B, prLower}, // L& LATIN SMALL LETTER I WITH INVERTED BREVE + {0x020C, 0x020C, prUpper}, // L& LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + {0x020D, 0x020D, prLower}, // L& LATIN SMALL LETTER O WITH DOUBLE GRAVE + {0x020E, 0x020E, prUpper}, // L& LATIN CAPITAL LETTER O WITH INVERTED BREVE + {0x020F, 0x020F, prLower}, // L& LATIN SMALL LETTER O WITH INVERTED BREVE + {0x0210, 0x0210, prUpper}, // L& LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + {0x0211, 0x0211, prLower}, // L& LATIN SMALL LETTER R WITH DOUBLE GRAVE + {0x0212, 0x0212, prUpper}, // L& LATIN CAPITAL LETTER R WITH INVERTED BREVE + {0x0213, 0x0213, prLower}, // L& LATIN SMALL LETTER R WITH INVERTED BREVE + {0x0214, 0x0214, prUpper}, // L& LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + {0x0215, 0x0215, prLower}, // L& LATIN SMALL LETTER U WITH DOUBLE GRAVE + {0x0216, 0x0216, prUpper}, // L& LATIN CAPITAL LETTER U WITH INVERTED BREVE + {0x0217, 0x0217, prLower}, // L& LATIN SMALL LETTER U WITH INVERTED BREVE + {0x0218, 0x0218, prUpper}, // L& LATIN CAPITAL LETTER S WITH COMMA BELOW + {0x0219, 0x0219, prLower}, // L& LATIN SMALL LETTER S WITH COMMA BELOW + {0x021A, 0x021A, prUpper}, // L& LATIN CAPITAL LETTER T WITH COMMA BELOW + {0x021B, 0x021B, prLower}, // L& LATIN SMALL LETTER T WITH COMMA BELOW + {0x021C, 0x021C, prUpper}, // L& LATIN CAPITAL LETTER YOGH + {0x021D, 0x021D, prLower}, // L& LATIN SMALL LETTER YOGH + {0x021E, 0x021E, prUpper}, // L& LATIN CAPITAL LETTER H WITH CARON + {0x021F, 0x021F, prLower}, // L& LATIN SMALL LETTER H WITH CARON + {0x0220, 0x0220, prUpper}, // L& LATIN CAPITAL LETTER N WITH LONG RIGHT LEG + {0x0221, 0x0221, prLower}, // L& LATIN SMALL LETTER D WITH CURL + {0x0222, 0x0222, prUpper}, // L& LATIN CAPITAL LETTER OU + {0x0223, 0x0223, prLower}, // L& LATIN SMALL LETTER OU + {0x0224, 0x0224, prUpper}, // L& LATIN CAPITAL LETTER Z WITH HOOK + {0x0225, 0x0225, prLower}, // L& LATIN SMALL LETTER Z WITH HOOK + {0x0226, 0x0226, prUpper}, // L& LATIN CAPITAL LETTER A WITH DOT ABOVE + {0x0227, 0x0227, prLower}, // L& LATIN SMALL LETTER A WITH DOT ABOVE + {0x0228, 0x0228, prUpper}, // L& LATIN CAPITAL LETTER E WITH CEDILLA + {0x0229, 0x0229, prLower}, // L& LATIN SMALL LETTER E WITH CEDILLA + {0x022A, 0x022A, prUpper}, // L& LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + {0x022B, 0x022B, prLower}, // L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON + {0x022C, 0x022C, prUpper}, // L& LATIN CAPITAL LETTER O WITH TILDE AND MACRON + {0x022D, 0x022D, prLower}, // L& LATIN SMALL LETTER O WITH TILDE AND MACRON + {0x022E, 0x022E, prUpper}, // L& LATIN CAPITAL LETTER O WITH DOT ABOVE + {0x022F, 0x022F, prLower}, // L& LATIN SMALL LETTER O WITH DOT ABOVE + {0x0230, 0x0230, prUpper}, // L& LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + {0x0231, 0x0231, prLower}, // L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON + {0x0232, 0x0232, prUpper}, // L& LATIN CAPITAL LETTER Y WITH MACRON + {0x0233, 0x0239, prLower}, // L& [7] LATIN SMALL LETTER Y WITH MACRON..LATIN SMALL LETTER QP DIGRAPH + {0x023A, 0x023B, prUpper}, // L& [2] LATIN CAPITAL LETTER A WITH STROKE..LATIN CAPITAL LETTER C WITH STROKE + {0x023C, 0x023C, prLower}, // L& LATIN SMALL LETTER C WITH STROKE + {0x023D, 0x023E, prUpper}, // L& [2] LATIN CAPITAL LETTER L WITH BAR..LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + {0x023F, 0x0240, prLower}, // L& [2] LATIN SMALL LETTER S WITH SWASH TAIL..LATIN SMALL LETTER Z WITH SWASH TAIL + {0x0241, 0x0241, prUpper}, // L& LATIN CAPITAL LETTER GLOTTAL STOP + {0x0242, 0x0242, prLower}, // L& LATIN SMALL LETTER GLOTTAL STOP + {0x0243, 0x0246, prUpper}, // L& [4] LATIN CAPITAL LETTER B WITH STROKE..LATIN CAPITAL LETTER E WITH STROKE + {0x0247, 0x0247, prLower}, // L& LATIN SMALL LETTER E WITH STROKE + {0x0248, 0x0248, prUpper}, // L& LATIN CAPITAL LETTER J WITH STROKE + {0x0249, 0x0249, prLower}, // L& LATIN SMALL LETTER J WITH STROKE + {0x024A, 0x024A, prUpper}, // L& LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL + {0x024B, 0x024B, prLower}, // L& LATIN SMALL LETTER Q WITH HOOK TAIL + {0x024C, 0x024C, prUpper}, // L& LATIN CAPITAL LETTER R WITH STROKE + {0x024D, 0x024D, prLower}, // L& LATIN SMALL LETTER R WITH STROKE + {0x024E, 0x024E, prUpper}, // L& LATIN CAPITAL LETTER Y WITH STROKE + {0x024F, 0x0293, prLower}, // L& [69] LATIN SMALL LETTER Y WITH STROKE..LATIN SMALL LETTER EZH WITH CURL + {0x0294, 0x0294, prOLetter}, // Lo LATIN LETTER GLOTTAL STOP + {0x0295, 0x02AF, prLower}, // L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL + {0x02B0, 0x02B8, prLower}, // Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y + {0x02B9, 0x02BF, prOLetter}, // Lm [7] MODIFIER LETTER PRIME..MODIFIER LETTER LEFT HALF RING + {0x02C0, 0x02C1, prLower}, // Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP + {0x02C6, 0x02D1, prOLetter}, // Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON + {0x02E0, 0x02E4, prLower}, // Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP + {0x02EC, 0x02EC, prOLetter}, // Lm MODIFIER LETTER VOICING + {0x02EE, 0x02EE, prOLetter}, // Lm MODIFIER LETTER DOUBLE APOSTROPHE + {0x0300, 0x036F, prExtend}, // Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X + {0x0370, 0x0370, prUpper}, // L& GREEK CAPITAL LETTER HETA + {0x0371, 0x0371, prLower}, // L& GREEK SMALL LETTER HETA + {0x0372, 0x0372, prUpper}, // L& GREEK CAPITAL LETTER ARCHAIC SAMPI + {0x0373, 0x0373, prLower}, // L& GREEK SMALL LETTER ARCHAIC SAMPI + {0x0374, 0x0374, prOLetter}, // Lm GREEK NUMERAL SIGN + {0x0376, 0x0376, prUpper}, // L& GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA + {0x0377, 0x0377, prLower}, // L& GREEK SMALL LETTER PAMPHYLIAN DIGAMMA + {0x037A, 0x037A, prLower}, // Lm GREEK YPOGEGRAMMENI + {0x037B, 0x037D, prLower}, // L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL + {0x037F, 0x037F, prUpper}, // L& GREEK CAPITAL LETTER YOT + {0x0386, 0x0386, prUpper}, // L& GREEK CAPITAL LETTER ALPHA WITH TONOS + {0x0388, 0x038A, prUpper}, // L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS + {0x038C, 0x038C, prUpper}, // L& GREEK CAPITAL LETTER OMICRON WITH TONOS + {0x038E, 0x038F, prUpper}, // L& [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS + {0x0390, 0x0390, prLower}, // L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + {0x0391, 0x03A1, prUpper}, // L& [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO + {0x03A3, 0x03AB, prUpper}, // L& [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + {0x03AC, 0x03CE, prLower}, // L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS + {0x03CF, 0x03CF, prUpper}, // L& GREEK CAPITAL KAI SYMBOL + {0x03D0, 0x03D1, prLower}, // L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL + {0x03D2, 0x03D4, prUpper}, // L& [3] GREEK UPSILON WITH HOOK SYMBOL..GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL + {0x03D5, 0x03D7, prLower}, // L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL + {0x03D8, 0x03D8, prUpper}, // L& GREEK LETTER ARCHAIC KOPPA + {0x03D9, 0x03D9, prLower}, // L& GREEK SMALL LETTER ARCHAIC KOPPA + {0x03DA, 0x03DA, prUpper}, // L& GREEK LETTER STIGMA + {0x03DB, 0x03DB, prLower}, // L& GREEK SMALL LETTER STIGMA + {0x03DC, 0x03DC, prUpper}, // L& GREEK LETTER DIGAMMA + {0x03DD, 0x03DD, prLower}, // L& GREEK SMALL LETTER DIGAMMA + {0x03DE, 0x03DE, prUpper}, // L& GREEK LETTER KOPPA + {0x03DF, 0x03DF, prLower}, // L& GREEK SMALL LETTER KOPPA + {0x03E0, 0x03E0, prUpper}, // L& GREEK LETTER SAMPI + {0x03E1, 0x03E1, prLower}, // L& GREEK SMALL LETTER SAMPI + {0x03E2, 0x03E2, prUpper}, // L& COPTIC CAPITAL LETTER SHEI + {0x03E3, 0x03E3, prLower}, // L& COPTIC SMALL LETTER SHEI + {0x03E4, 0x03E4, prUpper}, // L& COPTIC CAPITAL LETTER FEI + {0x03E5, 0x03E5, prLower}, // L& COPTIC SMALL LETTER FEI + {0x03E6, 0x03E6, prUpper}, // L& COPTIC CAPITAL LETTER KHEI + {0x03E7, 0x03E7, prLower}, // L& COPTIC SMALL LETTER KHEI + {0x03E8, 0x03E8, prUpper}, // L& COPTIC CAPITAL LETTER HORI + {0x03E9, 0x03E9, prLower}, // L& COPTIC SMALL LETTER HORI + {0x03EA, 0x03EA, prUpper}, // L& COPTIC CAPITAL LETTER GANGIA + {0x03EB, 0x03EB, prLower}, // L& COPTIC SMALL LETTER GANGIA + {0x03EC, 0x03EC, prUpper}, // L& COPTIC CAPITAL LETTER SHIMA + {0x03ED, 0x03ED, prLower}, // L& COPTIC SMALL LETTER SHIMA + {0x03EE, 0x03EE, prUpper}, // L& COPTIC CAPITAL LETTER DEI + {0x03EF, 0x03F3, prLower}, // L& [5] COPTIC SMALL LETTER DEI..GREEK LETTER YOT + {0x03F4, 0x03F4, prUpper}, // L& GREEK CAPITAL THETA SYMBOL + {0x03F5, 0x03F5, prLower}, // L& GREEK LUNATE EPSILON SYMBOL + {0x03F7, 0x03F7, prUpper}, // L& GREEK CAPITAL LETTER SHO + {0x03F8, 0x03F8, prLower}, // L& GREEK SMALL LETTER SHO + {0x03F9, 0x03FA, prUpper}, // L& [2] GREEK CAPITAL LUNATE SIGMA SYMBOL..GREEK CAPITAL LETTER SAN + {0x03FB, 0x03FC, prLower}, // L& [2] GREEK SMALL LETTER SAN..GREEK RHO WITH STROKE SYMBOL + {0x03FD, 0x042F, prUpper}, // L& [51] GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL..CYRILLIC CAPITAL LETTER YA + {0x0430, 0x045F, prLower}, // L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE + {0x0460, 0x0460, prUpper}, // L& CYRILLIC CAPITAL LETTER OMEGA + {0x0461, 0x0461, prLower}, // L& CYRILLIC SMALL LETTER OMEGA + {0x0462, 0x0462, prUpper}, // L& CYRILLIC CAPITAL LETTER YAT + {0x0463, 0x0463, prLower}, // L& CYRILLIC SMALL LETTER YAT + {0x0464, 0x0464, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTIFIED E + {0x0465, 0x0465, prLower}, // L& CYRILLIC SMALL LETTER IOTIFIED E + {0x0466, 0x0466, prUpper}, // L& CYRILLIC CAPITAL LETTER LITTLE YUS + {0x0467, 0x0467, prLower}, // L& CYRILLIC SMALL LETTER LITTLE YUS + {0x0468, 0x0468, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS + {0x0469, 0x0469, prLower}, // L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS + {0x046A, 0x046A, prUpper}, // L& CYRILLIC CAPITAL LETTER BIG YUS + {0x046B, 0x046B, prLower}, // L& CYRILLIC SMALL LETTER BIG YUS + {0x046C, 0x046C, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS + {0x046D, 0x046D, prLower}, // L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS + {0x046E, 0x046E, prUpper}, // L& CYRILLIC CAPITAL LETTER KSI + {0x046F, 0x046F, prLower}, // L& CYRILLIC SMALL LETTER KSI + {0x0470, 0x0470, prUpper}, // L& CYRILLIC CAPITAL LETTER PSI + {0x0471, 0x0471, prLower}, // L& CYRILLIC SMALL LETTER PSI + {0x0472, 0x0472, prUpper}, // L& CYRILLIC CAPITAL LETTER FITA + {0x0473, 0x0473, prLower}, // L& CYRILLIC SMALL LETTER FITA + {0x0474, 0x0474, prUpper}, // L& CYRILLIC CAPITAL LETTER IZHITSA + {0x0475, 0x0475, prLower}, // L& CYRILLIC SMALL LETTER IZHITSA + {0x0476, 0x0476, prUpper}, // L& CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + {0x0477, 0x0477, prLower}, // L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + {0x0478, 0x0478, prUpper}, // L& CYRILLIC CAPITAL LETTER UK + {0x0479, 0x0479, prLower}, // L& CYRILLIC SMALL LETTER UK + {0x047A, 0x047A, prUpper}, // L& CYRILLIC CAPITAL LETTER ROUND OMEGA + {0x047B, 0x047B, prLower}, // L& CYRILLIC SMALL LETTER ROUND OMEGA + {0x047C, 0x047C, prUpper}, // L& CYRILLIC CAPITAL LETTER OMEGA WITH TITLO + {0x047D, 0x047D, prLower}, // L& CYRILLIC SMALL LETTER OMEGA WITH TITLO + {0x047E, 0x047E, prUpper}, // L& CYRILLIC CAPITAL LETTER OT + {0x047F, 0x047F, prLower}, // L& CYRILLIC SMALL LETTER OT + {0x0480, 0x0480, prUpper}, // L& CYRILLIC CAPITAL LETTER KOPPA + {0x0481, 0x0481, prLower}, // L& CYRILLIC SMALL LETTER KOPPA + {0x0483, 0x0487, prExtend}, // Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE + {0x0488, 0x0489, prExtend}, // Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN + {0x048A, 0x048A, prUpper}, // L& CYRILLIC CAPITAL LETTER SHORT I WITH TAIL + {0x048B, 0x048B, prLower}, // L& CYRILLIC SMALL LETTER SHORT I WITH TAIL + {0x048C, 0x048C, prUpper}, // L& CYRILLIC CAPITAL LETTER SEMISOFT SIGN + {0x048D, 0x048D, prLower}, // L& CYRILLIC SMALL LETTER SEMISOFT SIGN + {0x048E, 0x048E, prUpper}, // L& CYRILLIC CAPITAL LETTER ER WITH TICK + {0x048F, 0x048F, prLower}, // L& CYRILLIC SMALL LETTER ER WITH TICK + {0x0490, 0x0490, prUpper}, // L& CYRILLIC CAPITAL LETTER GHE WITH UPTURN + {0x0491, 0x0491, prLower}, // L& CYRILLIC SMALL LETTER GHE WITH UPTURN + {0x0492, 0x0492, prUpper}, // L& CYRILLIC CAPITAL LETTER GHE WITH STROKE + {0x0493, 0x0493, prLower}, // L& CYRILLIC SMALL LETTER GHE WITH STROKE + {0x0494, 0x0494, prUpper}, // L& CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK + {0x0495, 0x0495, prLower}, // L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK + {0x0496, 0x0496, prUpper}, // L& CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER + {0x0497, 0x0497, prLower}, // L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER + {0x0498, 0x0498, prUpper}, // L& CYRILLIC CAPITAL LETTER ZE WITH DESCENDER + {0x0499, 0x0499, prLower}, // L& CYRILLIC SMALL LETTER ZE WITH DESCENDER + {0x049A, 0x049A, prUpper}, // L& CYRILLIC CAPITAL LETTER KA WITH DESCENDER + {0x049B, 0x049B, prLower}, // L& CYRILLIC SMALL LETTER KA WITH DESCENDER + {0x049C, 0x049C, prUpper}, // L& CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE + {0x049D, 0x049D, prLower}, // L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE + {0x049E, 0x049E, prUpper}, // L& CYRILLIC CAPITAL LETTER KA WITH STROKE + {0x049F, 0x049F, prLower}, // L& CYRILLIC SMALL LETTER KA WITH STROKE + {0x04A0, 0x04A0, prUpper}, // L& CYRILLIC CAPITAL LETTER BASHKIR KA + {0x04A1, 0x04A1, prLower}, // L& CYRILLIC SMALL LETTER BASHKIR KA + {0x04A2, 0x04A2, prUpper}, // L& CYRILLIC CAPITAL LETTER EN WITH DESCENDER + {0x04A3, 0x04A3, prLower}, // L& CYRILLIC SMALL LETTER EN WITH DESCENDER + {0x04A4, 0x04A4, prUpper}, // L& CYRILLIC CAPITAL LIGATURE EN GHE + {0x04A5, 0x04A5, prLower}, // L& CYRILLIC SMALL LIGATURE EN GHE + {0x04A6, 0x04A6, prUpper}, // L& CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK + {0x04A7, 0x04A7, prLower}, // L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK + {0x04A8, 0x04A8, prUpper}, // L& CYRILLIC CAPITAL LETTER ABKHASIAN HA + {0x04A9, 0x04A9, prLower}, // L& CYRILLIC SMALL LETTER ABKHASIAN HA + {0x04AA, 0x04AA, prUpper}, // L& CYRILLIC CAPITAL LETTER ES WITH DESCENDER + {0x04AB, 0x04AB, prLower}, // L& CYRILLIC SMALL LETTER ES WITH DESCENDER + {0x04AC, 0x04AC, prUpper}, // L& CYRILLIC CAPITAL LETTER TE WITH DESCENDER + {0x04AD, 0x04AD, prLower}, // L& CYRILLIC SMALL LETTER TE WITH DESCENDER + {0x04AE, 0x04AE, prUpper}, // L& CYRILLIC CAPITAL LETTER STRAIGHT U + {0x04AF, 0x04AF, prLower}, // L& CYRILLIC SMALL LETTER STRAIGHT U + {0x04B0, 0x04B0, prUpper}, // L& CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE + {0x04B1, 0x04B1, prLower}, // L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE + {0x04B2, 0x04B2, prUpper}, // L& CYRILLIC CAPITAL LETTER HA WITH DESCENDER + {0x04B3, 0x04B3, prLower}, // L& CYRILLIC SMALL LETTER HA WITH DESCENDER + {0x04B4, 0x04B4, prUpper}, // L& CYRILLIC CAPITAL LIGATURE TE TSE + {0x04B5, 0x04B5, prLower}, // L& CYRILLIC SMALL LIGATURE TE TSE + {0x04B6, 0x04B6, prUpper}, // L& CYRILLIC CAPITAL LETTER CHE WITH DESCENDER + {0x04B7, 0x04B7, prLower}, // L& CYRILLIC SMALL LETTER CHE WITH DESCENDER + {0x04B8, 0x04B8, prUpper}, // L& CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE + {0x04B9, 0x04B9, prLower}, // L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE + {0x04BA, 0x04BA, prUpper}, // L& CYRILLIC CAPITAL LETTER SHHA + {0x04BB, 0x04BB, prLower}, // L& CYRILLIC SMALL LETTER SHHA + {0x04BC, 0x04BC, prUpper}, // L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE + {0x04BD, 0x04BD, prLower}, // L& CYRILLIC SMALL LETTER ABKHASIAN CHE + {0x04BE, 0x04BE, prUpper}, // L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER + {0x04BF, 0x04BF, prLower}, // L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER + {0x04C0, 0x04C1, prUpper}, // L& [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE + {0x04C2, 0x04C2, prLower}, // L& CYRILLIC SMALL LETTER ZHE WITH BREVE + {0x04C3, 0x04C3, prUpper}, // L& CYRILLIC CAPITAL LETTER KA WITH HOOK + {0x04C4, 0x04C4, prLower}, // L& CYRILLIC SMALL LETTER KA WITH HOOK + {0x04C5, 0x04C5, prUpper}, // L& CYRILLIC CAPITAL LETTER EL WITH TAIL + {0x04C6, 0x04C6, prLower}, // L& CYRILLIC SMALL LETTER EL WITH TAIL + {0x04C7, 0x04C7, prUpper}, // L& CYRILLIC CAPITAL LETTER EN WITH HOOK + {0x04C8, 0x04C8, prLower}, // L& CYRILLIC SMALL LETTER EN WITH HOOK + {0x04C9, 0x04C9, prUpper}, // L& CYRILLIC CAPITAL LETTER EN WITH TAIL + {0x04CA, 0x04CA, prLower}, // L& CYRILLIC SMALL LETTER EN WITH TAIL + {0x04CB, 0x04CB, prUpper}, // L& CYRILLIC CAPITAL LETTER KHAKASSIAN CHE + {0x04CC, 0x04CC, prLower}, // L& CYRILLIC SMALL LETTER KHAKASSIAN CHE + {0x04CD, 0x04CD, prUpper}, // L& CYRILLIC CAPITAL LETTER EM WITH TAIL + {0x04CE, 0x04CF, prLower}, // L& [2] CYRILLIC SMALL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER PALOCHKA + {0x04D0, 0x04D0, prUpper}, // L& CYRILLIC CAPITAL LETTER A WITH BREVE + {0x04D1, 0x04D1, prLower}, // L& CYRILLIC SMALL LETTER A WITH BREVE + {0x04D2, 0x04D2, prUpper}, // L& CYRILLIC CAPITAL LETTER A WITH DIAERESIS + {0x04D3, 0x04D3, prLower}, // L& CYRILLIC SMALL LETTER A WITH DIAERESIS + {0x04D4, 0x04D4, prUpper}, // L& CYRILLIC CAPITAL LIGATURE A IE + {0x04D5, 0x04D5, prLower}, // L& CYRILLIC SMALL LIGATURE A IE + {0x04D6, 0x04D6, prUpper}, // L& CYRILLIC CAPITAL LETTER IE WITH BREVE + {0x04D7, 0x04D7, prLower}, // L& CYRILLIC SMALL LETTER IE WITH BREVE + {0x04D8, 0x04D8, prUpper}, // L& CYRILLIC CAPITAL LETTER SCHWA + {0x04D9, 0x04D9, prLower}, // L& CYRILLIC SMALL LETTER SCHWA + {0x04DA, 0x04DA, prUpper}, // L& CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS + {0x04DB, 0x04DB, prLower}, // L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS + {0x04DC, 0x04DC, prUpper}, // L& CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS + {0x04DD, 0x04DD, prLower}, // L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS + {0x04DE, 0x04DE, prUpper}, // L& CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS + {0x04DF, 0x04DF, prLower}, // L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS + {0x04E0, 0x04E0, prUpper}, // L& CYRILLIC CAPITAL LETTER ABKHASIAN DZE + {0x04E1, 0x04E1, prLower}, // L& CYRILLIC SMALL LETTER ABKHASIAN DZE + {0x04E2, 0x04E2, prUpper}, // L& CYRILLIC CAPITAL LETTER I WITH MACRON + {0x04E3, 0x04E3, prLower}, // L& CYRILLIC SMALL LETTER I WITH MACRON + {0x04E4, 0x04E4, prUpper}, // L& CYRILLIC CAPITAL LETTER I WITH DIAERESIS + {0x04E5, 0x04E5, prLower}, // L& CYRILLIC SMALL LETTER I WITH DIAERESIS + {0x04E6, 0x04E6, prUpper}, // L& CYRILLIC CAPITAL LETTER O WITH DIAERESIS + {0x04E7, 0x04E7, prLower}, // L& CYRILLIC SMALL LETTER O WITH DIAERESIS + {0x04E8, 0x04E8, prUpper}, // L& CYRILLIC CAPITAL LETTER BARRED O + {0x04E9, 0x04E9, prLower}, // L& CYRILLIC SMALL LETTER BARRED O + {0x04EA, 0x04EA, prUpper}, // L& CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS + {0x04EB, 0x04EB, prLower}, // L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS + {0x04EC, 0x04EC, prUpper}, // L& CYRILLIC CAPITAL LETTER E WITH DIAERESIS + {0x04ED, 0x04ED, prLower}, // L& CYRILLIC SMALL LETTER E WITH DIAERESIS + {0x04EE, 0x04EE, prUpper}, // L& CYRILLIC CAPITAL LETTER U WITH MACRON + {0x04EF, 0x04EF, prLower}, // L& CYRILLIC SMALL LETTER U WITH MACRON + {0x04F0, 0x04F0, prUpper}, // L& CYRILLIC CAPITAL LETTER U WITH DIAERESIS + {0x04F1, 0x04F1, prLower}, // L& CYRILLIC SMALL LETTER U WITH DIAERESIS + {0x04F2, 0x04F2, prUpper}, // L& CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE + {0x04F3, 0x04F3, prLower}, // L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE + {0x04F4, 0x04F4, prUpper}, // L& CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS + {0x04F5, 0x04F5, prLower}, // L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS + {0x04F6, 0x04F6, prUpper}, // L& CYRILLIC CAPITAL LETTER GHE WITH DESCENDER + {0x04F7, 0x04F7, prLower}, // L& CYRILLIC SMALL LETTER GHE WITH DESCENDER + {0x04F8, 0x04F8, prUpper}, // L& CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS + {0x04F9, 0x04F9, prLower}, // L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS + {0x04FA, 0x04FA, prUpper}, // L& CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK + {0x04FB, 0x04FB, prLower}, // L& CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK + {0x04FC, 0x04FC, prUpper}, // L& CYRILLIC CAPITAL LETTER HA WITH HOOK + {0x04FD, 0x04FD, prLower}, // L& CYRILLIC SMALL LETTER HA WITH HOOK + {0x04FE, 0x04FE, prUpper}, // L& CYRILLIC CAPITAL LETTER HA WITH STROKE + {0x04FF, 0x04FF, prLower}, // L& CYRILLIC SMALL LETTER HA WITH STROKE + {0x0500, 0x0500, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI DE + {0x0501, 0x0501, prLower}, // L& CYRILLIC SMALL LETTER KOMI DE + {0x0502, 0x0502, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI DJE + {0x0503, 0x0503, prLower}, // L& CYRILLIC SMALL LETTER KOMI DJE + {0x0504, 0x0504, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI ZJE + {0x0505, 0x0505, prLower}, // L& CYRILLIC SMALL LETTER KOMI ZJE + {0x0506, 0x0506, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI DZJE + {0x0507, 0x0507, prLower}, // L& CYRILLIC SMALL LETTER KOMI DZJE + {0x0508, 0x0508, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI LJE + {0x0509, 0x0509, prLower}, // L& CYRILLIC SMALL LETTER KOMI LJE + {0x050A, 0x050A, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI NJE + {0x050B, 0x050B, prLower}, // L& CYRILLIC SMALL LETTER KOMI NJE + {0x050C, 0x050C, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI SJE + {0x050D, 0x050D, prLower}, // L& CYRILLIC SMALL LETTER KOMI SJE + {0x050E, 0x050E, prUpper}, // L& CYRILLIC CAPITAL LETTER KOMI TJE + {0x050F, 0x050F, prLower}, // L& CYRILLIC SMALL LETTER KOMI TJE + {0x0510, 0x0510, prUpper}, // L& CYRILLIC CAPITAL LETTER REVERSED ZE + {0x0511, 0x0511, prLower}, // L& CYRILLIC SMALL LETTER REVERSED ZE + {0x0512, 0x0512, prUpper}, // L& CYRILLIC CAPITAL LETTER EL WITH HOOK + {0x0513, 0x0513, prLower}, // L& CYRILLIC SMALL LETTER EL WITH HOOK + {0x0514, 0x0514, prUpper}, // L& CYRILLIC CAPITAL LETTER LHA + {0x0515, 0x0515, prLower}, // L& CYRILLIC SMALL LETTER LHA + {0x0516, 0x0516, prUpper}, // L& CYRILLIC CAPITAL LETTER RHA + {0x0517, 0x0517, prLower}, // L& CYRILLIC SMALL LETTER RHA + {0x0518, 0x0518, prUpper}, // L& CYRILLIC CAPITAL LETTER YAE + {0x0519, 0x0519, prLower}, // L& CYRILLIC SMALL LETTER YAE + {0x051A, 0x051A, prUpper}, // L& CYRILLIC CAPITAL LETTER QA + {0x051B, 0x051B, prLower}, // L& CYRILLIC SMALL LETTER QA + {0x051C, 0x051C, prUpper}, // L& CYRILLIC CAPITAL LETTER WE + {0x051D, 0x051D, prLower}, // L& CYRILLIC SMALL LETTER WE + {0x051E, 0x051E, prUpper}, // L& CYRILLIC CAPITAL LETTER ALEUT KA + {0x051F, 0x051F, prLower}, // L& CYRILLIC SMALL LETTER ALEUT KA + {0x0520, 0x0520, prUpper}, // L& CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK + {0x0521, 0x0521, prLower}, // L& CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK + {0x0522, 0x0522, prUpper}, // L& CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK + {0x0523, 0x0523, prLower}, // L& CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK + {0x0524, 0x0524, prUpper}, // L& CYRILLIC CAPITAL LETTER PE WITH DESCENDER + {0x0525, 0x0525, prLower}, // L& CYRILLIC SMALL LETTER PE WITH DESCENDER + {0x0526, 0x0526, prUpper}, // L& CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER + {0x0527, 0x0527, prLower}, // L& CYRILLIC SMALL LETTER SHHA WITH DESCENDER + {0x0528, 0x0528, prUpper}, // L& CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK + {0x0529, 0x0529, prLower}, // L& CYRILLIC SMALL LETTER EN WITH LEFT HOOK + {0x052A, 0x052A, prUpper}, // L& CYRILLIC CAPITAL LETTER DZZHE + {0x052B, 0x052B, prLower}, // L& CYRILLIC SMALL LETTER DZZHE + {0x052C, 0x052C, prUpper}, // L& CYRILLIC CAPITAL LETTER DCHE + {0x052D, 0x052D, prLower}, // L& CYRILLIC SMALL LETTER DCHE + {0x052E, 0x052E, prUpper}, // L& CYRILLIC CAPITAL LETTER EL WITH DESCENDER + {0x052F, 0x052F, prLower}, // L& CYRILLIC SMALL LETTER EL WITH DESCENDER + {0x0531, 0x0556, prUpper}, // L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH + {0x0559, 0x0559, prOLetter}, // Lm ARMENIAN MODIFIER LETTER LEFT HALF RING + {0x055D, 0x055D, prSContinue}, // Po ARMENIAN COMMA + {0x0560, 0x0588, prLower}, // L& [41] ARMENIAN SMALL LETTER TURNED AYB..ARMENIAN SMALL LETTER YI WITH STROKE + {0x0589, 0x0589, prSTerm}, // Po ARMENIAN FULL STOP + {0x0591, 0x05BD, prExtend}, // Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG + {0x05BF, 0x05BF, prExtend}, // Mn HEBREW POINT RAFE + {0x05C1, 0x05C2, prExtend}, // Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT + {0x05C4, 0x05C5, prExtend}, // Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT + {0x05C7, 0x05C7, prExtend}, // Mn HEBREW POINT QAMATS QATAN + {0x05D0, 0x05EA, prOLetter}, // Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV + {0x05EF, 0x05F2, prOLetter}, // Lo [4] HEBREW YOD TRIANGLE..HEBREW LIGATURE YIDDISH DOUBLE YOD + {0x05F3, 0x05F3, prOLetter}, // Po HEBREW PUNCTUATION GERESH + {0x0600, 0x0605, prFormat}, // Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE + {0x060C, 0x060D, prSContinue}, // Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR + {0x0610, 0x061A, prExtend}, // Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA + {0x061C, 0x061C, prFormat}, // Cf ARABIC LETTER MARK + {0x061D, 0x061F, prSTerm}, // Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK + {0x0620, 0x063F, prOLetter}, // Lo [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE + {0x0640, 0x0640, prOLetter}, // Lm ARABIC TATWEEL + {0x0641, 0x064A, prOLetter}, // Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH + {0x064B, 0x065F, prExtend}, // Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW + {0x0660, 0x0669, prNumeric}, // Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE + {0x066B, 0x066C, prNumeric}, // Po [2] ARABIC DECIMAL SEPARATOR..ARABIC THOUSANDS SEPARATOR + {0x066E, 0x066F, prOLetter}, // Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF + {0x0670, 0x0670, prExtend}, // Mn ARABIC LETTER SUPERSCRIPT ALEF + {0x0671, 0x06D3, prOLetter}, // Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE + {0x06D4, 0x06D4, prSTerm}, // Po ARABIC FULL STOP + {0x06D5, 0x06D5, prOLetter}, // Lo ARABIC LETTER AE + {0x06D6, 0x06DC, prExtend}, // Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN + {0x06DD, 0x06DD, prFormat}, // Cf ARABIC END OF AYAH + {0x06DF, 0x06E4, prExtend}, // Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA + {0x06E5, 0x06E6, prOLetter}, // Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH + {0x06E7, 0x06E8, prExtend}, // Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON + {0x06EA, 0x06ED, prExtend}, // Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM + {0x06EE, 0x06EF, prOLetter}, // Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V + {0x06F0, 0x06F9, prNumeric}, // Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE + {0x06FA, 0x06FC, prOLetter}, // Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW + {0x06FF, 0x06FF, prOLetter}, // Lo ARABIC LETTER HEH WITH INVERTED V + {0x0700, 0x0702, prSTerm}, // Po [3] SYRIAC END OF PARAGRAPH..SYRIAC SUBLINEAR FULL STOP + {0x070F, 0x070F, prFormat}, // Cf SYRIAC ABBREVIATION MARK + {0x0710, 0x0710, prOLetter}, // Lo SYRIAC LETTER ALAPH + {0x0711, 0x0711, prExtend}, // Mn SYRIAC LETTER SUPERSCRIPT ALAPH + {0x0712, 0x072F, prOLetter}, // Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH + {0x0730, 0x074A, prExtend}, // Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH + {0x074D, 0x07A5, prOLetter}, // Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU + {0x07A6, 0x07B0, prExtend}, // Mn [11] THAANA ABAFILI..THAANA SUKUN + {0x07B1, 0x07B1, prOLetter}, // Lo THAANA LETTER NAA + {0x07C0, 0x07C9, prNumeric}, // Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE + {0x07CA, 0x07EA, prOLetter}, // Lo [33] NKO LETTER A..NKO LETTER JONA RA + {0x07EB, 0x07F3, prExtend}, // Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE + {0x07F4, 0x07F5, prOLetter}, // Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE + {0x07F8, 0x07F8, prSContinue}, // Po NKO COMMA + {0x07F9, 0x07F9, prSTerm}, // Po NKO EXCLAMATION MARK + {0x07FA, 0x07FA, prOLetter}, // Lm NKO LAJANYALAN + {0x07FD, 0x07FD, prExtend}, // Mn NKO DANTAYALAN + {0x0800, 0x0815, prOLetter}, // Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF + {0x0816, 0x0819, prExtend}, // Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH + {0x081A, 0x081A, prOLetter}, // Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT + {0x081B, 0x0823, prExtend}, // Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A + {0x0824, 0x0824, prOLetter}, // Lm SAMARITAN MODIFIER LETTER SHORT A + {0x0825, 0x0827, prExtend}, // Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U + {0x0828, 0x0828, prOLetter}, // Lm SAMARITAN MODIFIER LETTER I + {0x0829, 0x082D, prExtend}, // Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA + {0x0837, 0x0837, prSTerm}, // Po SAMARITAN PUNCTUATION MELODIC QITSA + {0x0839, 0x0839, prSTerm}, // Po SAMARITAN PUNCTUATION QITSA + {0x083D, 0x083E, prSTerm}, // Po [2] SAMARITAN PUNCTUATION SOF MASHFAAT..SAMARITAN PUNCTUATION ANNAAU + {0x0840, 0x0858, prOLetter}, // Lo [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN + {0x0859, 0x085B, prExtend}, // Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK + {0x0860, 0x086A, prOLetter}, // Lo [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA + {0x0870, 0x0887, prOLetter}, // Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT + {0x0889, 0x088E, prOLetter}, // Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL + {0x0890, 0x0891, prFormat}, // Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE + {0x0898, 0x089F, prExtend}, // Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA + {0x08A0, 0x08C8, prOLetter}, // Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF + {0x08C9, 0x08C9, prOLetter}, // Lm ARABIC SMALL FARSI YEH + {0x08CA, 0x08E1, prExtend}, // Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA + {0x08E2, 0x08E2, prFormat}, // Cf ARABIC DISPUTED END OF AYAH + {0x08E3, 0x0902, prExtend}, // Mn [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA + {0x0903, 0x0903, prExtend}, // Mc DEVANAGARI SIGN VISARGA + {0x0904, 0x0939, prOLetter}, // Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA + {0x093A, 0x093A, prExtend}, // Mn DEVANAGARI VOWEL SIGN OE + {0x093B, 0x093B, prExtend}, // Mc DEVANAGARI VOWEL SIGN OOE + {0x093C, 0x093C, prExtend}, // Mn DEVANAGARI SIGN NUKTA + {0x093D, 0x093D, prOLetter}, // Lo DEVANAGARI SIGN AVAGRAHA + {0x093E, 0x0940, prExtend}, // Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II + {0x0941, 0x0948, prExtend}, // Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI + {0x0949, 0x094C, prExtend}, // Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU + {0x094D, 0x094D, prExtend}, // Mn DEVANAGARI SIGN VIRAMA + {0x094E, 0x094F, prExtend}, // Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW + {0x0950, 0x0950, prOLetter}, // Lo DEVANAGARI OM + {0x0951, 0x0957, prExtend}, // Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE + {0x0958, 0x0961, prOLetter}, // Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL + {0x0962, 0x0963, prExtend}, // Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL + {0x0964, 0x0965, prSTerm}, // Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA + {0x0966, 0x096F, prNumeric}, // Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE + {0x0971, 0x0971, prOLetter}, // Lm DEVANAGARI SIGN HIGH SPACING DOT + {0x0972, 0x0980, prOLetter}, // Lo [15] DEVANAGARI LETTER CANDRA A..BENGALI ANJI + {0x0981, 0x0981, prExtend}, // Mn BENGALI SIGN CANDRABINDU + {0x0982, 0x0983, prExtend}, // Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA + {0x0985, 0x098C, prOLetter}, // Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L + {0x098F, 0x0990, prOLetter}, // Lo [2] BENGALI LETTER E..BENGALI LETTER AI + {0x0993, 0x09A8, prOLetter}, // Lo [22] BENGALI LETTER O..BENGALI LETTER NA + {0x09AA, 0x09B0, prOLetter}, // Lo [7] BENGALI LETTER PA..BENGALI LETTER RA + {0x09B2, 0x09B2, prOLetter}, // Lo BENGALI LETTER LA + {0x09B6, 0x09B9, prOLetter}, // Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA + {0x09BC, 0x09BC, prExtend}, // Mn BENGALI SIGN NUKTA + {0x09BD, 0x09BD, prOLetter}, // Lo BENGALI SIGN AVAGRAHA + {0x09BE, 0x09C0, prExtend}, // Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II + {0x09C1, 0x09C4, prExtend}, // Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR + {0x09C7, 0x09C8, prExtend}, // Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI + {0x09CB, 0x09CC, prExtend}, // Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU + {0x09CD, 0x09CD, prExtend}, // Mn BENGALI SIGN VIRAMA + {0x09CE, 0x09CE, prOLetter}, // Lo BENGALI LETTER KHANDA TA + {0x09D7, 0x09D7, prExtend}, // Mc BENGALI AU LENGTH MARK + {0x09DC, 0x09DD, prOLetter}, // Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA + {0x09DF, 0x09E1, prOLetter}, // Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL + {0x09E2, 0x09E3, prExtend}, // Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL + {0x09E6, 0x09EF, prNumeric}, // Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE + {0x09F0, 0x09F1, prOLetter}, // Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL + {0x09FC, 0x09FC, prOLetter}, // Lo BENGALI LETTER VEDIC ANUSVARA + {0x09FE, 0x09FE, prExtend}, // Mn BENGALI SANDHI MARK + {0x0A01, 0x0A02, prExtend}, // Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI + {0x0A03, 0x0A03, prExtend}, // Mc GURMUKHI SIGN VISARGA + {0x0A05, 0x0A0A, prOLetter}, // Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU + {0x0A0F, 0x0A10, prOLetter}, // Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI + {0x0A13, 0x0A28, prOLetter}, // Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA + {0x0A2A, 0x0A30, prOLetter}, // Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA + {0x0A32, 0x0A33, prOLetter}, // Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA + {0x0A35, 0x0A36, prOLetter}, // Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA + {0x0A38, 0x0A39, prOLetter}, // Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA + {0x0A3C, 0x0A3C, prExtend}, // Mn GURMUKHI SIGN NUKTA + {0x0A3E, 0x0A40, prExtend}, // Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II + {0x0A41, 0x0A42, prExtend}, // Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU + {0x0A47, 0x0A48, prExtend}, // Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI + {0x0A4B, 0x0A4D, prExtend}, // Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA + {0x0A51, 0x0A51, prExtend}, // Mn GURMUKHI SIGN UDAAT + {0x0A59, 0x0A5C, prOLetter}, // Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA + {0x0A5E, 0x0A5E, prOLetter}, // Lo GURMUKHI LETTER FA + {0x0A66, 0x0A6F, prNumeric}, // Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE + {0x0A70, 0x0A71, prExtend}, // Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK + {0x0A72, 0x0A74, prOLetter}, // Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR + {0x0A75, 0x0A75, prExtend}, // Mn GURMUKHI SIGN YAKASH + {0x0A81, 0x0A82, prExtend}, // Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA + {0x0A83, 0x0A83, prExtend}, // Mc GUJARATI SIGN VISARGA + {0x0A85, 0x0A8D, prOLetter}, // Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E + {0x0A8F, 0x0A91, prOLetter}, // Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O + {0x0A93, 0x0AA8, prOLetter}, // Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA + {0x0AAA, 0x0AB0, prOLetter}, // Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA + {0x0AB2, 0x0AB3, prOLetter}, // Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA + {0x0AB5, 0x0AB9, prOLetter}, // Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA + {0x0ABC, 0x0ABC, prExtend}, // Mn GUJARATI SIGN NUKTA + {0x0ABD, 0x0ABD, prOLetter}, // Lo GUJARATI SIGN AVAGRAHA + {0x0ABE, 0x0AC0, prExtend}, // Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II + {0x0AC1, 0x0AC5, prExtend}, // Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E + {0x0AC7, 0x0AC8, prExtend}, // Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI + {0x0AC9, 0x0AC9, prExtend}, // Mc GUJARATI VOWEL SIGN CANDRA O + {0x0ACB, 0x0ACC, prExtend}, // Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU + {0x0ACD, 0x0ACD, prExtend}, // Mn GUJARATI SIGN VIRAMA + {0x0AD0, 0x0AD0, prOLetter}, // Lo GUJARATI OM + {0x0AE0, 0x0AE1, prOLetter}, // Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL + {0x0AE2, 0x0AE3, prExtend}, // Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL + {0x0AE6, 0x0AEF, prNumeric}, // Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE + {0x0AF9, 0x0AF9, prOLetter}, // Lo GUJARATI LETTER ZHA + {0x0AFA, 0x0AFF, prExtend}, // Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE + {0x0B01, 0x0B01, prExtend}, // Mn ORIYA SIGN CANDRABINDU + {0x0B02, 0x0B03, prExtend}, // Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA + {0x0B05, 0x0B0C, prOLetter}, // Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L + {0x0B0F, 0x0B10, prOLetter}, // Lo [2] ORIYA LETTER E..ORIYA LETTER AI + {0x0B13, 0x0B28, prOLetter}, // Lo [22] ORIYA LETTER O..ORIYA LETTER NA + {0x0B2A, 0x0B30, prOLetter}, // Lo [7] ORIYA LETTER PA..ORIYA LETTER RA + {0x0B32, 0x0B33, prOLetter}, // Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA + {0x0B35, 0x0B39, prOLetter}, // Lo [5] ORIYA LETTER VA..ORIYA LETTER HA + {0x0B3C, 0x0B3C, prExtend}, // Mn ORIYA SIGN NUKTA + {0x0B3D, 0x0B3D, prOLetter}, // Lo ORIYA SIGN AVAGRAHA + {0x0B3E, 0x0B3E, prExtend}, // Mc ORIYA VOWEL SIGN AA + {0x0B3F, 0x0B3F, prExtend}, // Mn ORIYA VOWEL SIGN I + {0x0B40, 0x0B40, prExtend}, // Mc ORIYA VOWEL SIGN II + {0x0B41, 0x0B44, prExtend}, // Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR + {0x0B47, 0x0B48, prExtend}, // Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI + {0x0B4B, 0x0B4C, prExtend}, // Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU + {0x0B4D, 0x0B4D, prExtend}, // Mn ORIYA SIGN VIRAMA + {0x0B55, 0x0B56, prExtend}, // Mn [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK + {0x0B57, 0x0B57, prExtend}, // Mc ORIYA AU LENGTH MARK + {0x0B5C, 0x0B5D, prOLetter}, // Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA + {0x0B5F, 0x0B61, prOLetter}, // Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL + {0x0B62, 0x0B63, prExtend}, // Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL + {0x0B66, 0x0B6F, prNumeric}, // Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE + {0x0B71, 0x0B71, prOLetter}, // Lo ORIYA LETTER WA + {0x0B82, 0x0B82, prExtend}, // Mn TAMIL SIGN ANUSVARA + {0x0B83, 0x0B83, prOLetter}, // Lo TAMIL SIGN VISARGA + {0x0B85, 0x0B8A, prOLetter}, // Lo [6] TAMIL LETTER A..TAMIL LETTER UU + {0x0B8E, 0x0B90, prOLetter}, // Lo [3] TAMIL LETTER E..TAMIL LETTER AI + {0x0B92, 0x0B95, prOLetter}, // Lo [4] TAMIL LETTER O..TAMIL LETTER KA + {0x0B99, 0x0B9A, prOLetter}, // Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA + {0x0B9C, 0x0B9C, prOLetter}, // Lo TAMIL LETTER JA + {0x0B9E, 0x0B9F, prOLetter}, // Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA + {0x0BA3, 0x0BA4, prOLetter}, // Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA + {0x0BA8, 0x0BAA, prOLetter}, // Lo [3] TAMIL LETTER NA..TAMIL LETTER PA + {0x0BAE, 0x0BB9, prOLetter}, // Lo [12] TAMIL LETTER MA..TAMIL LETTER HA + {0x0BBE, 0x0BBF, prExtend}, // Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I + {0x0BC0, 0x0BC0, prExtend}, // Mn TAMIL VOWEL SIGN II + {0x0BC1, 0x0BC2, prExtend}, // Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU + {0x0BC6, 0x0BC8, prExtend}, // Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI + {0x0BCA, 0x0BCC, prExtend}, // Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU + {0x0BCD, 0x0BCD, prExtend}, // Mn TAMIL SIGN VIRAMA + {0x0BD0, 0x0BD0, prOLetter}, // Lo TAMIL OM + {0x0BD7, 0x0BD7, prExtend}, // Mc TAMIL AU LENGTH MARK + {0x0BE6, 0x0BEF, prNumeric}, // Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE + {0x0C00, 0x0C00, prExtend}, // Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE + {0x0C01, 0x0C03, prExtend}, // Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA + {0x0C04, 0x0C04, prExtend}, // Mn TELUGU SIGN COMBINING ANUSVARA ABOVE + {0x0C05, 0x0C0C, prOLetter}, // Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L + {0x0C0E, 0x0C10, prOLetter}, // Lo [3] TELUGU LETTER E..TELUGU LETTER AI + {0x0C12, 0x0C28, prOLetter}, // Lo [23] TELUGU LETTER O..TELUGU LETTER NA + {0x0C2A, 0x0C39, prOLetter}, // Lo [16] TELUGU LETTER PA..TELUGU LETTER HA + {0x0C3C, 0x0C3C, prExtend}, // Mn TELUGU SIGN NUKTA + {0x0C3D, 0x0C3D, prOLetter}, // Lo TELUGU SIGN AVAGRAHA + {0x0C3E, 0x0C40, prExtend}, // Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II + {0x0C41, 0x0C44, prExtend}, // Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR + {0x0C46, 0x0C48, prExtend}, // Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI + {0x0C4A, 0x0C4D, prExtend}, // Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA + {0x0C55, 0x0C56, prExtend}, // Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK + {0x0C58, 0x0C5A, prOLetter}, // Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA + {0x0C5D, 0x0C5D, prOLetter}, // Lo TELUGU LETTER NAKAARA POLLU + {0x0C60, 0x0C61, prOLetter}, // Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL + {0x0C62, 0x0C63, prExtend}, // Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL + {0x0C66, 0x0C6F, prNumeric}, // Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE + {0x0C80, 0x0C80, prOLetter}, // Lo KANNADA SIGN SPACING CANDRABINDU + {0x0C81, 0x0C81, prExtend}, // Mn KANNADA SIGN CANDRABINDU + {0x0C82, 0x0C83, prExtend}, // Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA + {0x0C85, 0x0C8C, prOLetter}, // Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L + {0x0C8E, 0x0C90, prOLetter}, // Lo [3] KANNADA LETTER E..KANNADA LETTER AI + {0x0C92, 0x0CA8, prOLetter}, // Lo [23] KANNADA LETTER O..KANNADA LETTER NA + {0x0CAA, 0x0CB3, prOLetter}, // Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA + {0x0CB5, 0x0CB9, prOLetter}, // Lo [5] KANNADA LETTER VA..KANNADA LETTER HA + {0x0CBC, 0x0CBC, prExtend}, // Mn KANNADA SIGN NUKTA + {0x0CBD, 0x0CBD, prOLetter}, // Lo KANNADA SIGN AVAGRAHA + {0x0CBE, 0x0CBE, prExtend}, // Mc KANNADA VOWEL SIGN AA + {0x0CBF, 0x0CBF, prExtend}, // Mn KANNADA VOWEL SIGN I + {0x0CC0, 0x0CC4, prExtend}, // Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR + {0x0CC6, 0x0CC6, prExtend}, // Mn KANNADA VOWEL SIGN E + {0x0CC7, 0x0CC8, prExtend}, // Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI + {0x0CCA, 0x0CCB, prExtend}, // Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO + {0x0CCC, 0x0CCD, prExtend}, // Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA + {0x0CD5, 0x0CD6, prExtend}, // Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK + {0x0CDD, 0x0CDE, prOLetter}, // Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA + {0x0CE0, 0x0CE1, prOLetter}, // Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL + {0x0CE2, 0x0CE3, prExtend}, // Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL + {0x0CE6, 0x0CEF, prNumeric}, // Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE + {0x0CF1, 0x0CF2, prOLetter}, // Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA + {0x0CF3, 0x0CF3, prExtend}, // Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT + {0x0D00, 0x0D01, prExtend}, // Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU + {0x0D02, 0x0D03, prExtend}, // Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA + {0x0D04, 0x0D0C, prOLetter}, // Lo [9] MALAYALAM LETTER VEDIC ANUSVARA..MALAYALAM LETTER VOCALIC L + {0x0D0E, 0x0D10, prOLetter}, // Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI + {0x0D12, 0x0D3A, prOLetter}, // Lo [41] MALAYALAM LETTER O..MALAYALAM LETTER TTTA + {0x0D3B, 0x0D3C, prExtend}, // Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA + {0x0D3D, 0x0D3D, prOLetter}, // Lo MALAYALAM SIGN AVAGRAHA + {0x0D3E, 0x0D40, prExtend}, // Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II + {0x0D41, 0x0D44, prExtend}, // Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR + {0x0D46, 0x0D48, prExtend}, // Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI + {0x0D4A, 0x0D4C, prExtend}, // Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU + {0x0D4D, 0x0D4D, prExtend}, // Mn MALAYALAM SIGN VIRAMA + {0x0D4E, 0x0D4E, prOLetter}, // Lo MALAYALAM LETTER DOT REPH + {0x0D54, 0x0D56, prOLetter}, // Lo [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL + {0x0D57, 0x0D57, prExtend}, // Mc MALAYALAM AU LENGTH MARK + {0x0D5F, 0x0D61, prOLetter}, // Lo [3] MALAYALAM LETTER ARCHAIC II..MALAYALAM LETTER VOCALIC LL + {0x0D62, 0x0D63, prExtend}, // Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL + {0x0D66, 0x0D6F, prNumeric}, // Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE + {0x0D7A, 0x0D7F, prOLetter}, // Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K + {0x0D81, 0x0D81, prExtend}, // Mn SINHALA SIGN CANDRABINDU + {0x0D82, 0x0D83, prExtend}, // Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA + {0x0D85, 0x0D96, prOLetter}, // Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA + {0x0D9A, 0x0DB1, prOLetter}, // Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA + {0x0DB3, 0x0DBB, prOLetter}, // Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA + {0x0DBD, 0x0DBD, prOLetter}, // Lo SINHALA LETTER DANTAJA LAYANNA + {0x0DC0, 0x0DC6, prOLetter}, // Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA + {0x0DCA, 0x0DCA, prExtend}, // Mn SINHALA SIGN AL-LAKUNA + {0x0DCF, 0x0DD1, prExtend}, // Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA + {0x0DD2, 0x0DD4, prExtend}, // Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA + {0x0DD6, 0x0DD6, prExtend}, // Mn SINHALA VOWEL SIGN DIGA PAA-PILLA + {0x0DD8, 0x0DDF, prExtend}, // Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA + {0x0DE6, 0x0DEF, prNumeric}, // Nd [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE + {0x0DF2, 0x0DF3, prExtend}, // Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA + {0x0E01, 0x0E30, prOLetter}, // Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A + {0x0E31, 0x0E31, prExtend}, // Mn THAI CHARACTER MAI HAN-AKAT + {0x0E32, 0x0E33, prOLetter}, // Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM + {0x0E34, 0x0E3A, prExtend}, // Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU + {0x0E40, 0x0E45, prOLetter}, // Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO + {0x0E46, 0x0E46, prOLetter}, // Lm THAI CHARACTER MAIYAMOK + {0x0E47, 0x0E4E, prExtend}, // Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN + {0x0E50, 0x0E59, prNumeric}, // Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE + {0x0E81, 0x0E82, prOLetter}, // Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG + {0x0E84, 0x0E84, prOLetter}, // Lo LAO LETTER KHO TAM + {0x0E86, 0x0E8A, prOLetter}, // Lo [5] LAO LETTER PALI GHA..LAO LETTER SO TAM + {0x0E8C, 0x0EA3, prOLetter}, // Lo [24] LAO LETTER PALI JHA..LAO LETTER LO LING + {0x0EA5, 0x0EA5, prOLetter}, // Lo LAO LETTER LO LOOT + {0x0EA7, 0x0EB0, prOLetter}, // Lo [10] LAO LETTER WO..LAO VOWEL SIGN A + {0x0EB1, 0x0EB1, prExtend}, // Mn LAO VOWEL SIGN MAI KAN + {0x0EB2, 0x0EB3, prOLetter}, // Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM + {0x0EB4, 0x0EBC, prExtend}, // Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO + {0x0EBD, 0x0EBD, prOLetter}, // Lo LAO SEMIVOWEL SIGN NYO + {0x0EC0, 0x0EC4, prOLetter}, // Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI + {0x0EC6, 0x0EC6, prOLetter}, // Lm LAO KO LA + {0x0EC8, 0x0ECE, prExtend}, // Mn [7] LAO TONE MAI EK..LAO YAMAKKAN + {0x0ED0, 0x0ED9, prNumeric}, // Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE + {0x0EDC, 0x0EDF, prOLetter}, // Lo [4] LAO HO NO..LAO LETTER KHMU NYO + {0x0F00, 0x0F00, prOLetter}, // Lo TIBETAN SYLLABLE OM + {0x0F18, 0x0F19, prExtend}, // Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS + {0x0F20, 0x0F29, prNumeric}, // Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE + {0x0F35, 0x0F35, prExtend}, // Mn TIBETAN MARK NGAS BZUNG NYI ZLA + {0x0F37, 0x0F37, prExtend}, // Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS + {0x0F39, 0x0F39, prExtend}, // Mn TIBETAN MARK TSA -PHRU + {0x0F3A, 0x0F3A, prClose}, // Ps TIBETAN MARK GUG RTAGS GYON + {0x0F3B, 0x0F3B, prClose}, // Pe TIBETAN MARK GUG RTAGS GYAS + {0x0F3C, 0x0F3C, prClose}, // Ps TIBETAN MARK ANG KHANG GYON + {0x0F3D, 0x0F3D, prClose}, // Pe TIBETAN MARK ANG KHANG GYAS + {0x0F3E, 0x0F3F, prExtend}, // Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES + {0x0F40, 0x0F47, prOLetter}, // Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA + {0x0F49, 0x0F6C, prOLetter}, // Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA + {0x0F71, 0x0F7E, prExtend}, // Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO + {0x0F7F, 0x0F7F, prExtend}, // Mc TIBETAN SIGN RNAM BCAD + {0x0F80, 0x0F84, prExtend}, // Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA + {0x0F86, 0x0F87, prExtend}, // Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS + {0x0F88, 0x0F8C, prOLetter}, // Lo [5] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN INVERTED MCHU CAN + {0x0F8D, 0x0F97, prExtend}, // Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA + {0x0F99, 0x0FBC, prExtend}, // Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA + {0x0FC6, 0x0FC6, prExtend}, // Mn TIBETAN SYMBOL PADMA GDAN + {0x1000, 0x102A, prOLetter}, // Lo [43] MYANMAR LETTER KA..MYANMAR LETTER AU + {0x102B, 0x102C, prExtend}, // Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA + {0x102D, 0x1030, prExtend}, // Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU + {0x1031, 0x1031, prExtend}, // Mc MYANMAR VOWEL SIGN E + {0x1032, 0x1037, prExtend}, // Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW + {0x1038, 0x1038, prExtend}, // Mc MYANMAR SIGN VISARGA + {0x1039, 0x103A, prExtend}, // Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT + {0x103B, 0x103C, prExtend}, // Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA + {0x103D, 0x103E, prExtend}, // Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA + {0x103F, 0x103F, prOLetter}, // Lo MYANMAR LETTER GREAT SA + {0x1040, 0x1049, prNumeric}, // Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE + {0x104A, 0x104B, prSTerm}, // Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION + {0x1050, 0x1055, prOLetter}, // Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL + {0x1056, 0x1057, prExtend}, // Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR + {0x1058, 0x1059, prExtend}, // Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL + {0x105A, 0x105D, prOLetter}, // Lo [4] MYANMAR LETTER MON NGA..MYANMAR LETTER MON BBE + {0x105E, 0x1060, prExtend}, // Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA + {0x1061, 0x1061, prOLetter}, // Lo MYANMAR LETTER SGAW KAREN SHA + {0x1062, 0x1064, prExtend}, // Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO + {0x1065, 0x1066, prOLetter}, // Lo [2] MYANMAR LETTER WESTERN PWO KAREN THA..MYANMAR LETTER WESTERN PWO KAREN PWA + {0x1067, 0x106D, prExtend}, // Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 + {0x106E, 0x1070, prOLetter}, // Lo [3] MYANMAR LETTER EASTERN PWO KAREN NNA..MYANMAR LETTER EASTERN PWO KAREN GHWA + {0x1071, 0x1074, prExtend}, // Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE + {0x1075, 0x1081, prOLetter}, // Lo [13] MYANMAR LETTER SHAN KA..MYANMAR LETTER SHAN HA + {0x1082, 0x1082, prExtend}, // Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA + {0x1083, 0x1084, prExtend}, // Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E + {0x1085, 0x1086, prExtend}, // Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y + {0x1087, 0x108C, prExtend}, // Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 + {0x108D, 0x108D, prExtend}, // Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE + {0x108E, 0x108E, prOLetter}, // Lo MYANMAR LETTER RUMAI PALAUNG FA + {0x108F, 0x108F, prExtend}, // Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 + {0x1090, 0x1099, prNumeric}, // Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE + {0x109A, 0x109C, prExtend}, // Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A + {0x109D, 0x109D, prExtend}, // Mn MYANMAR VOWEL SIGN AITON AI + {0x10A0, 0x10C5, prUpper}, // L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE + {0x10C7, 0x10C7, prUpper}, // L& GEORGIAN CAPITAL LETTER YN + {0x10CD, 0x10CD, prUpper}, // L& GEORGIAN CAPITAL LETTER AEN + {0x10D0, 0x10FA, prOLetter}, // L& [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN + {0x10FC, 0x10FC, prLower}, // Lm MODIFIER LETTER GEORGIAN NAR + {0x10FD, 0x10FF, prOLetter}, // L& [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN + {0x1100, 0x1248, prOLetter}, // Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA + {0x124A, 0x124D, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE + {0x1250, 0x1256, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO + {0x1258, 0x1258, prOLetter}, // Lo ETHIOPIC SYLLABLE QHWA + {0x125A, 0x125D, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE + {0x1260, 0x1288, prOLetter}, // Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA + {0x128A, 0x128D, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE + {0x1290, 0x12B0, prOLetter}, // Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA + {0x12B2, 0x12B5, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE + {0x12B8, 0x12BE, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO + {0x12C0, 0x12C0, prOLetter}, // Lo ETHIOPIC SYLLABLE KXWA + {0x12C2, 0x12C5, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE + {0x12C8, 0x12D6, prOLetter}, // Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O + {0x12D8, 0x1310, prOLetter}, // Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA + {0x1312, 0x1315, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE + {0x1318, 0x135A, prOLetter}, // Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA + {0x135D, 0x135F, prExtend}, // Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK + {0x1362, 0x1362, prSTerm}, // Po ETHIOPIC FULL STOP + {0x1367, 0x1368, prSTerm}, // Po [2] ETHIOPIC QUESTION MARK..ETHIOPIC PARAGRAPH SEPARATOR + {0x1380, 0x138F, prOLetter}, // Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE + {0x13A0, 0x13F5, prUpper}, // L& [86] CHEROKEE LETTER A..CHEROKEE LETTER MV + {0x13F8, 0x13FD, prLower}, // L& [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV + {0x1401, 0x166C, prOLetter}, // Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA + {0x166E, 0x166E, prSTerm}, // Po CANADIAN SYLLABICS FULL STOP + {0x166F, 0x167F, prOLetter}, // Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W + {0x1680, 0x1680, prSp}, // Zs OGHAM SPACE MARK + {0x1681, 0x169A, prOLetter}, // Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH + {0x169B, 0x169B, prClose}, // Ps OGHAM FEATHER MARK + {0x169C, 0x169C, prClose}, // Pe OGHAM REVERSED FEATHER MARK + {0x16A0, 0x16EA, prOLetter}, // Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X + {0x16EE, 0x16F0, prOLetter}, // Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL + {0x16F1, 0x16F8, prOLetter}, // Lo [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC + {0x1700, 0x1711, prOLetter}, // Lo [18] TAGALOG LETTER A..TAGALOG LETTER HA + {0x1712, 0x1714, prExtend}, // Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA + {0x1715, 0x1715, prExtend}, // Mc TAGALOG SIGN PAMUDPOD + {0x171F, 0x1731, prOLetter}, // Lo [19] TAGALOG LETTER ARCHAIC RA..HANUNOO LETTER HA + {0x1732, 0x1733, prExtend}, // Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U + {0x1734, 0x1734, prExtend}, // Mc HANUNOO SIGN PAMUDPOD + {0x1735, 0x1736, prSTerm}, // Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION + {0x1740, 0x1751, prOLetter}, // Lo [18] BUHID LETTER A..BUHID LETTER HA + {0x1752, 0x1753, prExtend}, // Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U + {0x1760, 0x176C, prOLetter}, // Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA + {0x176E, 0x1770, prOLetter}, // Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA + {0x1772, 0x1773, prExtend}, // Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U + {0x1780, 0x17B3, prOLetter}, // Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU + {0x17B4, 0x17B5, prExtend}, // Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA + {0x17B6, 0x17B6, prExtend}, // Mc KHMER VOWEL SIGN AA + {0x17B7, 0x17BD, prExtend}, // Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA + {0x17BE, 0x17C5, prExtend}, // Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU + {0x17C6, 0x17C6, prExtend}, // Mn KHMER SIGN NIKAHIT + {0x17C7, 0x17C8, prExtend}, // Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU + {0x17C9, 0x17D3, prExtend}, // Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT + {0x17D7, 0x17D7, prOLetter}, // Lm KHMER SIGN LEK TOO + {0x17DC, 0x17DC, prOLetter}, // Lo KHMER SIGN AVAKRAHASANYA + {0x17DD, 0x17DD, prExtend}, // Mn KHMER SIGN ATTHACAN + {0x17E0, 0x17E9, prNumeric}, // Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE + {0x1802, 0x1802, prSContinue}, // Po MONGOLIAN COMMA + {0x1803, 0x1803, prSTerm}, // Po MONGOLIAN FULL STOP + {0x1808, 0x1808, prSContinue}, // Po MONGOLIAN MANCHU COMMA + {0x1809, 0x1809, prSTerm}, // Po MONGOLIAN MANCHU FULL STOP + {0x180B, 0x180D, prExtend}, // Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE + {0x180E, 0x180E, prFormat}, // Cf MONGOLIAN VOWEL SEPARATOR + {0x180F, 0x180F, prExtend}, // Mn MONGOLIAN FREE VARIATION SELECTOR FOUR + {0x1810, 0x1819, prNumeric}, // Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE + {0x1820, 0x1842, prOLetter}, // Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI + {0x1843, 0x1843, prOLetter}, // Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN + {0x1844, 0x1878, prOLetter}, // Lo [53] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER CHA WITH TWO DOTS + {0x1880, 0x1884, prOLetter}, // Lo [5] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER ALI GALI INVERTED UBADAMA + {0x1885, 0x1886, prExtend}, // Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA + {0x1887, 0x18A8, prOLetter}, // Lo [34] MONGOLIAN LETTER ALI GALI A..MONGOLIAN LETTER MANCHU ALI GALI BHA + {0x18A9, 0x18A9, prExtend}, // Mn MONGOLIAN LETTER ALI GALI DAGALGA + {0x18AA, 0x18AA, prOLetter}, // Lo MONGOLIAN LETTER MANCHU ALI GALI LHA + {0x18B0, 0x18F5, prOLetter}, // Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S + {0x1900, 0x191E, prOLetter}, // Lo [31] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER TRA + {0x1920, 0x1922, prExtend}, // Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U + {0x1923, 0x1926, prExtend}, // Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU + {0x1927, 0x1928, prExtend}, // Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O + {0x1929, 0x192B, prExtend}, // Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA + {0x1930, 0x1931, prExtend}, // Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA + {0x1932, 0x1932, prExtend}, // Mn LIMBU SMALL LETTER ANUSVARA + {0x1933, 0x1938, prExtend}, // Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA + {0x1939, 0x193B, prExtend}, // Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I + {0x1944, 0x1945, prSTerm}, // Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK + {0x1946, 0x194F, prNumeric}, // Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE + {0x1950, 0x196D, prOLetter}, // Lo [30] TAI LE LETTER KA..TAI LE LETTER AI + {0x1970, 0x1974, prOLetter}, // Lo [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 + {0x1980, 0x19AB, prOLetter}, // Lo [44] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW SUA + {0x19B0, 0x19C9, prOLetter}, // Lo [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 + {0x19D0, 0x19D9, prNumeric}, // Nd [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE + {0x1A00, 0x1A16, prOLetter}, // Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA + {0x1A17, 0x1A18, prExtend}, // Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U + {0x1A19, 0x1A1A, prExtend}, // Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O + {0x1A1B, 0x1A1B, prExtend}, // Mn BUGINESE VOWEL SIGN AE + {0x1A20, 0x1A54, prOLetter}, // Lo [53] TAI THAM LETTER HIGH KA..TAI THAM LETTER GREAT SA + {0x1A55, 0x1A55, prExtend}, // Mc TAI THAM CONSONANT SIGN MEDIAL RA + {0x1A56, 0x1A56, prExtend}, // Mn TAI THAM CONSONANT SIGN MEDIAL LA + {0x1A57, 0x1A57, prExtend}, // Mc TAI THAM CONSONANT SIGN LA TANG LAI + {0x1A58, 0x1A5E, prExtend}, // Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA + {0x1A60, 0x1A60, prExtend}, // Mn TAI THAM SIGN SAKOT + {0x1A61, 0x1A61, prExtend}, // Mc TAI THAM VOWEL SIGN A + {0x1A62, 0x1A62, prExtend}, // Mn TAI THAM VOWEL SIGN MAI SAT + {0x1A63, 0x1A64, prExtend}, // Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA + {0x1A65, 0x1A6C, prExtend}, // Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW + {0x1A6D, 0x1A72, prExtend}, // Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI + {0x1A73, 0x1A7C, prExtend}, // Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN + {0x1A7F, 0x1A7F, prExtend}, // Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT + {0x1A80, 0x1A89, prNumeric}, // Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE + {0x1A90, 0x1A99, prNumeric}, // Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE + {0x1AA7, 0x1AA7, prOLetter}, // Lm TAI THAM SIGN MAI YAMOK + {0x1AA8, 0x1AAB, prSTerm}, // Po [4] TAI THAM SIGN KAAN..TAI THAM SIGN SATKAANKUU + {0x1AB0, 0x1ABD, prExtend}, // Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW + {0x1ABE, 0x1ABE, prExtend}, // Me COMBINING PARENTHESES OVERLAY + {0x1ABF, 0x1ACE, prExtend}, // Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T + {0x1B00, 0x1B03, prExtend}, // Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG + {0x1B04, 0x1B04, prExtend}, // Mc BALINESE SIGN BISAH + {0x1B05, 0x1B33, prOLetter}, // Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA + {0x1B34, 0x1B34, prExtend}, // Mn BALINESE SIGN REREKAN + {0x1B35, 0x1B35, prExtend}, // Mc BALINESE VOWEL SIGN TEDUNG + {0x1B36, 0x1B3A, prExtend}, // Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA + {0x1B3B, 0x1B3B, prExtend}, // Mc BALINESE VOWEL SIGN RA REPA TEDUNG + {0x1B3C, 0x1B3C, prExtend}, // Mn BALINESE VOWEL SIGN LA LENGA + {0x1B3D, 0x1B41, prExtend}, // Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG + {0x1B42, 0x1B42, prExtend}, // Mn BALINESE VOWEL SIGN PEPET + {0x1B43, 0x1B44, prExtend}, // Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG + {0x1B45, 0x1B4C, prOLetter}, // Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA + {0x1B50, 0x1B59, prNumeric}, // Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE + {0x1B5A, 0x1B5B, prSTerm}, // Po [2] BALINESE PANTI..BALINESE PAMADA + {0x1B5E, 0x1B5F, prSTerm}, // Po [2] BALINESE CARIK SIKI..BALINESE CARIK PAREREN + {0x1B6B, 0x1B73, prExtend}, // Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG + {0x1B7D, 0x1B7E, prSTerm}, // Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG + {0x1B80, 0x1B81, prExtend}, // Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR + {0x1B82, 0x1B82, prExtend}, // Mc SUNDANESE SIGN PANGWISAD + {0x1B83, 0x1BA0, prOLetter}, // Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA + {0x1BA1, 0x1BA1, prExtend}, // Mc SUNDANESE CONSONANT SIGN PAMINGKAL + {0x1BA2, 0x1BA5, prExtend}, // Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU + {0x1BA6, 0x1BA7, prExtend}, // Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG + {0x1BA8, 0x1BA9, prExtend}, // Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG + {0x1BAA, 0x1BAA, prExtend}, // Mc SUNDANESE SIGN PAMAAEH + {0x1BAB, 0x1BAD, prExtend}, // Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA + {0x1BAE, 0x1BAF, prOLetter}, // Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA + {0x1BB0, 0x1BB9, prNumeric}, // Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE + {0x1BBA, 0x1BE5, prOLetter}, // Lo [44] SUNDANESE AVAGRAHA..BATAK LETTER U + {0x1BE6, 0x1BE6, prExtend}, // Mn BATAK SIGN TOMPI + {0x1BE7, 0x1BE7, prExtend}, // Mc BATAK VOWEL SIGN E + {0x1BE8, 0x1BE9, prExtend}, // Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE + {0x1BEA, 0x1BEC, prExtend}, // Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O + {0x1BED, 0x1BED, prExtend}, // Mn BATAK VOWEL SIGN KARO O + {0x1BEE, 0x1BEE, prExtend}, // Mc BATAK VOWEL SIGN U + {0x1BEF, 0x1BF1, prExtend}, // Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H + {0x1BF2, 0x1BF3, prExtend}, // Mc [2] BATAK PANGOLAT..BATAK PANONGONAN + {0x1C00, 0x1C23, prOLetter}, // Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A + {0x1C24, 0x1C2B, prExtend}, // Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU + {0x1C2C, 0x1C33, prExtend}, // Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T + {0x1C34, 0x1C35, prExtend}, // Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG + {0x1C36, 0x1C37, prExtend}, // Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA + {0x1C3B, 0x1C3C, prSTerm}, // Po [2] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION NYET THYOOM TA-ROL + {0x1C40, 0x1C49, prNumeric}, // Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE + {0x1C4D, 0x1C4F, prOLetter}, // Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA + {0x1C50, 0x1C59, prNumeric}, // Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE + {0x1C5A, 0x1C77, prOLetter}, // Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH + {0x1C78, 0x1C7D, prOLetter}, // Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD + {0x1C7E, 0x1C7F, prSTerm}, // Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD + {0x1C80, 0x1C88, prLower}, // L& [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK + {0x1C90, 0x1CBA, prOLetter}, // L& [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN + {0x1CBD, 0x1CBF, prOLetter}, // L& [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + {0x1CD0, 0x1CD2, prExtend}, // Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA + {0x1CD4, 0x1CE0, prExtend}, // Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA + {0x1CE1, 0x1CE1, prExtend}, // Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA + {0x1CE2, 0x1CE8, prExtend}, // Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL + {0x1CE9, 0x1CEC, prOLetter}, // Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL + {0x1CED, 0x1CED, prExtend}, // Mn VEDIC SIGN TIRYAK + {0x1CEE, 0x1CF3, prOLetter}, // Lo [6] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ROTATED ARDHAVISARGA + {0x1CF4, 0x1CF4, prExtend}, // Mn VEDIC TONE CANDRA ABOVE + {0x1CF5, 0x1CF6, prOLetter}, // Lo [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN UPADHMANIYA + {0x1CF7, 0x1CF7, prExtend}, // Mc VEDIC SIGN ATIKRAMA + {0x1CF8, 0x1CF9, prExtend}, // Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE + {0x1CFA, 0x1CFA, prOLetter}, // Lo VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA + {0x1D00, 0x1D2B, prLower}, // L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL + {0x1D2C, 0x1D6A, prLower}, // Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI + {0x1D6B, 0x1D77, prLower}, // L& [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G + {0x1D78, 0x1D78, prLower}, // Lm MODIFIER LETTER CYRILLIC EN + {0x1D79, 0x1D9A, prLower}, // L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK + {0x1D9B, 0x1DBF, prLower}, // Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA + {0x1DC0, 0x1DFF, prExtend}, // Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW + {0x1E00, 0x1E00, prUpper}, // L& LATIN CAPITAL LETTER A WITH RING BELOW + {0x1E01, 0x1E01, prLower}, // L& LATIN SMALL LETTER A WITH RING BELOW + {0x1E02, 0x1E02, prUpper}, // L& LATIN CAPITAL LETTER B WITH DOT ABOVE + {0x1E03, 0x1E03, prLower}, // L& LATIN SMALL LETTER B WITH DOT ABOVE + {0x1E04, 0x1E04, prUpper}, // L& LATIN CAPITAL LETTER B WITH DOT BELOW + {0x1E05, 0x1E05, prLower}, // L& LATIN SMALL LETTER B WITH DOT BELOW + {0x1E06, 0x1E06, prUpper}, // L& LATIN CAPITAL LETTER B WITH LINE BELOW + {0x1E07, 0x1E07, prLower}, // L& LATIN SMALL LETTER B WITH LINE BELOW + {0x1E08, 0x1E08, prUpper}, // L& LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE + {0x1E09, 0x1E09, prLower}, // L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE + {0x1E0A, 0x1E0A, prUpper}, // L& LATIN CAPITAL LETTER D WITH DOT ABOVE + {0x1E0B, 0x1E0B, prLower}, // L& LATIN SMALL LETTER D WITH DOT ABOVE + {0x1E0C, 0x1E0C, prUpper}, // L& LATIN CAPITAL LETTER D WITH DOT BELOW + {0x1E0D, 0x1E0D, prLower}, // L& LATIN SMALL LETTER D WITH DOT BELOW + {0x1E0E, 0x1E0E, prUpper}, // L& LATIN CAPITAL LETTER D WITH LINE BELOW + {0x1E0F, 0x1E0F, prLower}, // L& LATIN SMALL LETTER D WITH LINE BELOW + {0x1E10, 0x1E10, prUpper}, // L& LATIN CAPITAL LETTER D WITH CEDILLA + {0x1E11, 0x1E11, prLower}, // L& LATIN SMALL LETTER D WITH CEDILLA + {0x1E12, 0x1E12, prUpper}, // L& LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW + {0x1E13, 0x1E13, prLower}, // L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW + {0x1E14, 0x1E14, prUpper}, // L& LATIN CAPITAL LETTER E WITH MACRON AND GRAVE + {0x1E15, 0x1E15, prLower}, // L& LATIN SMALL LETTER E WITH MACRON AND GRAVE + {0x1E16, 0x1E16, prUpper}, // L& LATIN CAPITAL LETTER E WITH MACRON AND ACUTE + {0x1E17, 0x1E17, prLower}, // L& LATIN SMALL LETTER E WITH MACRON AND ACUTE + {0x1E18, 0x1E18, prUpper}, // L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW + {0x1E19, 0x1E19, prLower}, // L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW + {0x1E1A, 0x1E1A, prUpper}, // L& LATIN CAPITAL LETTER E WITH TILDE BELOW + {0x1E1B, 0x1E1B, prLower}, // L& LATIN SMALL LETTER E WITH TILDE BELOW + {0x1E1C, 0x1E1C, prUpper}, // L& LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE + {0x1E1D, 0x1E1D, prLower}, // L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE + {0x1E1E, 0x1E1E, prUpper}, // L& LATIN CAPITAL LETTER F WITH DOT ABOVE + {0x1E1F, 0x1E1F, prLower}, // L& LATIN SMALL LETTER F WITH DOT ABOVE + {0x1E20, 0x1E20, prUpper}, // L& LATIN CAPITAL LETTER G WITH MACRON + {0x1E21, 0x1E21, prLower}, // L& LATIN SMALL LETTER G WITH MACRON + {0x1E22, 0x1E22, prUpper}, // L& LATIN CAPITAL LETTER H WITH DOT ABOVE + {0x1E23, 0x1E23, prLower}, // L& LATIN SMALL LETTER H WITH DOT ABOVE + {0x1E24, 0x1E24, prUpper}, // L& LATIN CAPITAL LETTER H WITH DOT BELOW + {0x1E25, 0x1E25, prLower}, // L& LATIN SMALL LETTER H WITH DOT BELOW + {0x1E26, 0x1E26, prUpper}, // L& LATIN CAPITAL LETTER H WITH DIAERESIS + {0x1E27, 0x1E27, prLower}, // L& LATIN SMALL LETTER H WITH DIAERESIS + {0x1E28, 0x1E28, prUpper}, // L& LATIN CAPITAL LETTER H WITH CEDILLA + {0x1E29, 0x1E29, prLower}, // L& LATIN SMALL LETTER H WITH CEDILLA + {0x1E2A, 0x1E2A, prUpper}, // L& LATIN CAPITAL LETTER H WITH BREVE BELOW + {0x1E2B, 0x1E2B, prLower}, // L& LATIN SMALL LETTER H WITH BREVE BELOW + {0x1E2C, 0x1E2C, prUpper}, // L& LATIN CAPITAL LETTER I WITH TILDE BELOW + {0x1E2D, 0x1E2D, prLower}, // L& LATIN SMALL LETTER I WITH TILDE BELOW + {0x1E2E, 0x1E2E, prUpper}, // L& LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE + {0x1E2F, 0x1E2F, prLower}, // L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE + {0x1E30, 0x1E30, prUpper}, // L& LATIN CAPITAL LETTER K WITH ACUTE + {0x1E31, 0x1E31, prLower}, // L& LATIN SMALL LETTER K WITH ACUTE + {0x1E32, 0x1E32, prUpper}, // L& LATIN CAPITAL LETTER K WITH DOT BELOW + {0x1E33, 0x1E33, prLower}, // L& LATIN SMALL LETTER K WITH DOT BELOW + {0x1E34, 0x1E34, prUpper}, // L& LATIN CAPITAL LETTER K WITH LINE BELOW + {0x1E35, 0x1E35, prLower}, // L& LATIN SMALL LETTER K WITH LINE BELOW + {0x1E36, 0x1E36, prUpper}, // L& LATIN CAPITAL LETTER L WITH DOT BELOW + {0x1E37, 0x1E37, prLower}, // L& LATIN SMALL LETTER L WITH DOT BELOW + {0x1E38, 0x1E38, prUpper}, // L& LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON + {0x1E39, 0x1E39, prLower}, // L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON + {0x1E3A, 0x1E3A, prUpper}, // L& LATIN CAPITAL LETTER L WITH LINE BELOW + {0x1E3B, 0x1E3B, prLower}, // L& LATIN SMALL LETTER L WITH LINE BELOW + {0x1E3C, 0x1E3C, prUpper}, // L& LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW + {0x1E3D, 0x1E3D, prLower}, // L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW + {0x1E3E, 0x1E3E, prUpper}, // L& LATIN CAPITAL LETTER M WITH ACUTE + {0x1E3F, 0x1E3F, prLower}, // L& LATIN SMALL LETTER M WITH ACUTE + {0x1E40, 0x1E40, prUpper}, // L& LATIN CAPITAL LETTER M WITH DOT ABOVE + {0x1E41, 0x1E41, prLower}, // L& LATIN SMALL LETTER M WITH DOT ABOVE + {0x1E42, 0x1E42, prUpper}, // L& LATIN CAPITAL LETTER M WITH DOT BELOW + {0x1E43, 0x1E43, prLower}, // L& LATIN SMALL LETTER M WITH DOT BELOW + {0x1E44, 0x1E44, prUpper}, // L& LATIN CAPITAL LETTER N WITH DOT ABOVE + {0x1E45, 0x1E45, prLower}, // L& LATIN SMALL LETTER N WITH DOT ABOVE + {0x1E46, 0x1E46, prUpper}, // L& LATIN CAPITAL LETTER N WITH DOT BELOW + {0x1E47, 0x1E47, prLower}, // L& LATIN SMALL LETTER N WITH DOT BELOW + {0x1E48, 0x1E48, prUpper}, // L& LATIN CAPITAL LETTER N WITH LINE BELOW + {0x1E49, 0x1E49, prLower}, // L& LATIN SMALL LETTER N WITH LINE BELOW + {0x1E4A, 0x1E4A, prUpper}, // L& LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW + {0x1E4B, 0x1E4B, prLower}, // L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW + {0x1E4C, 0x1E4C, prUpper}, // L& LATIN CAPITAL LETTER O WITH TILDE AND ACUTE + {0x1E4D, 0x1E4D, prLower}, // L& LATIN SMALL LETTER O WITH TILDE AND ACUTE + {0x1E4E, 0x1E4E, prUpper}, // L& LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS + {0x1E4F, 0x1E4F, prLower}, // L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS + {0x1E50, 0x1E50, prUpper}, // L& LATIN CAPITAL LETTER O WITH MACRON AND GRAVE + {0x1E51, 0x1E51, prLower}, // L& LATIN SMALL LETTER O WITH MACRON AND GRAVE + {0x1E52, 0x1E52, prUpper}, // L& LATIN CAPITAL LETTER O WITH MACRON AND ACUTE + {0x1E53, 0x1E53, prLower}, // L& LATIN SMALL LETTER O WITH MACRON AND ACUTE + {0x1E54, 0x1E54, prUpper}, // L& LATIN CAPITAL LETTER P WITH ACUTE + {0x1E55, 0x1E55, prLower}, // L& LATIN SMALL LETTER P WITH ACUTE + {0x1E56, 0x1E56, prUpper}, // L& LATIN CAPITAL LETTER P WITH DOT ABOVE + {0x1E57, 0x1E57, prLower}, // L& LATIN SMALL LETTER P WITH DOT ABOVE + {0x1E58, 0x1E58, prUpper}, // L& LATIN CAPITAL LETTER R WITH DOT ABOVE + {0x1E59, 0x1E59, prLower}, // L& LATIN SMALL LETTER R WITH DOT ABOVE + {0x1E5A, 0x1E5A, prUpper}, // L& LATIN CAPITAL LETTER R WITH DOT BELOW + {0x1E5B, 0x1E5B, prLower}, // L& LATIN SMALL LETTER R WITH DOT BELOW + {0x1E5C, 0x1E5C, prUpper}, // L& LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON + {0x1E5D, 0x1E5D, prLower}, // L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON + {0x1E5E, 0x1E5E, prUpper}, // L& LATIN CAPITAL LETTER R WITH LINE BELOW + {0x1E5F, 0x1E5F, prLower}, // L& LATIN SMALL LETTER R WITH LINE BELOW + {0x1E60, 0x1E60, prUpper}, // L& LATIN CAPITAL LETTER S WITH DOT ABOVE + {0x1E61, 0x1E61, prLower}, // L& LATIN SMALL LETTER S WITH DOT ABOVE + {0x1E62, 0x1E62, prUpper}, // L& LATIN CAPITAL LETTER S WITH DOT BELOW + {0x1E63, 0x1E63, prLower}, // L& LATIN SMALL LETTER S WITH DOT BELOW + {0x1E64, 0x1E64, prUpper}, // L& LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE + {0x1E65, 0x1E65, prLower}, // L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE + {0x1E66, 0x1E66, prUpper}, // L& LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE + {0x1E67, 0x1E67, prLower}, // L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE + {0x1E68, 0x1E68, prUpper}, // L& LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE + {0x1E69, 0x1E69, prLower}, // L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE + {0x1E6A, 0x1E6A, prUpper}, // L& LATIN CAPITAL LETTER T WITH DOT ABOVE + {0x1E6B, 0x1E6B, prLower}, // L& LATIN SMALL LETTER T WITH DOT ABOVE + {0x1E6C, 0x1E6C, prUpper}, // L& LATIN CAPITAL LETTER T WITH DOT BELOW + {0x1E6D, 0x1E6D, prLower}, // L& LATIN SMALL LETTER T WITH DOT BELOW + {0x1E6E, 0x1E6E, prUpper}, // L& LATIN CAPITAL LETTER T WITH LINE BELOW + {0x1E6F, 0x1E6F, prLower}, // L& LATIN SMALL LETTER T WITH LINE BELOW + {0x1E70, 0x1E70, prUpper}, // L& LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW + {0x1E71, 0x1E71, prLower}, // L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW + {0x1E72, 0x1E72, prUpper}, // L& LATIN CAPITAL LETTER U WITH DIAERESIS BELOW + {0x1E73, 0x1E73, prLower}, // L& LATIN SMALL LETTER U WITH DIAERESIS BELOW + {0x1E74, 0x1E74, prUpper}, // L& LATIN CAPITAL LETTER U WITH TILDE BELOW + {0x1E75, 0x1E75, prLower}, // L& LATIN SMALL LETTER U WITH TILDE BELOW + {0x1E76, 0x1E76, prUpper}, // L& LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW + {0x1E77, 0x1E77, prLower}, // L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW + {0x1E78, 0x1E78, prUpper}, // L& LATIN CAPITAL LETTER U WITH TILDE AND ACUTE + {0x1E79, 0x1E79, prLower}, // L& LATIN SMALL LETTER U WITH TILDE AND ACUTE + {0x1E7A, 0x1E7A, prUpper}, // L& LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS + {0x1E7B, 0x1E7B, prLower}, // L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS + {0x1E7C, 0x1E7C, prUpper}, // L& LATIN CAPITAL LETTER V WITH TILDE + {0x1E7D, 0x1E7D, prLower}, // L& LATIN SMALL LETTER V WITH TILDE + {0x1E7E, 0x1E7E, prUpper}, // L& LATIN CAPITAL LETTER V WITH DOT BELOW + {0x1E7F, 0x1E7F, prLower}, // L& LATIN SMALL LETTER V WITH DOT BELOW + {0x1E80, 0x1E80, prUpper}, // L& LATIN CAPITAL LETTER W WITH GRAVE + {0x1E81, 0x1E81, prLower}, // L& LATIN SMALL LETTER W WITH GRAVE + {0x1E82, 0x1E82, prUpper}, // L& LATIN CAPITAL LETTER W WITH ACUTE + {0x1E83, 0x1E83, prLower}, // L& LATIN SMALL LETTER W WITH ACUTE + {0x1E84, 0x1E84, prUpper}, // L& LATIN CAPITAL LETTER W WITH DIAERESIS + {0x1E85, 0x1E85, prLower}, // L& LATIN SMALL LETTER W WITH DIAERESIS + {0x1E86, 0x1E86, prUpper}, // L& LATIN CAPITAL LETTER W WITH DOT ABOVE + {0x1E87, 0x1E87, prLower}, // L& LATIN SMALL LETTER W WITH DOT ABOVE + {0x1E88, 0x1E88, prUpper}, // L& LATIN CAPITAL LETTER W WITH DOT BELOW + {0x1E89, 0x1E89, prLower}, // L& LATIN SMALL LETTER W WITH DOT BELOW + {0x1E8A, 0x1E8A, prUpper}, // L& LATIN CAPITAL LETTER X WITH DOT ABOVE + {0x1E8B, 0x1E8B, prLower}, // L& LATIN SMALL LETTER X WITH DOT ABOVE + {0x1E8C, 0x1E8C, prUpper}, // L& LATIN CAPITAL LETTER X WITH DIAERESIS + {0x1E8D, 0x1E8D, prLower}, // L& LATIN SMALL LETTER X WITH DIAERESIS + {0x1E8E, 0x1E8E, prUpper}, // L& LATIN CAPITAL LETTER Y WITH DOT ABOVE + {0x1E8F, 0x1E8F, prLower}, // L& LATIN SMALL LETTER Y WITH DOT ABOVE + {0x1E90, 0x1E90, prUpper}, // L& LATIN CAPITAL LETTER Z WITH CIRCUMFLEX + {0x1E91, 0x1E91, prLower}, // L& LATIN SMALL LETTER Z WITH CIRCUMFLEX + {0x1E92, 0x1E92, prUpper}, // L& LATIN CAPITAL LETTER Z WITH DOT BELOW + {0x1E93, 0x1E93, prLower}, // L& LATIN SMALL LETTER Z WITH DOT BELOW + {0x1E94, 0x1E94, prUpper}, // L& LATIN CAPITAL LETTER Z WITH LINE BELOW + {0x1E95, 0x1E9D, prLower}, // L& [9] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH HIGH STROKE + {0x1E9E, 0x1E9E, prUpper}, // L& LATIN CAPITAL LETTER SHARP S + {0x1E9F, 0x1E9F, prLower}, // L& LATIN SMALL LETTER DELTA + {0x1EA0, 0x1EA0, prUpper}, // L& LATIN CAPITAL LETTER A WITH DOT BELOW + {0x1EA1, 0x1EA1, prLower}, // L& LATIN SMALL LETTER A WITH DOT BELOW + {0x1EA2, 0x1EA2, prUpper}, // L& LATIN CAPITAL LETTER A WITH HOOK ABOVE + {0x1EA3, 0x1EA3, prLower}, // L& LATIN SMALL LETTER A WITH HOOK ABOVE + {0x1EA4, 0x1EA4, prUpper}, // L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE + {0x1EA5, 0x1EA5, prLower}, // L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE + {0x1EA6, 0x1EA6, prUpper}, // L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE + {0x1EA7, 0x1EA7, prLower}, // L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE + {0x1EA8, 0x1EA8, prUpper}, // L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + {0x1EA9, 0x1EA9, prLower}, // L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + {0x1EAA, 0x1EAA, prUpper}, // L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE + {0x1EAB, 0x1EAB, prLower}, // L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE + {0x1EAC, 0x1EAC, prUpper}, // L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW + {0x1EAD, 0x1EAD, prLower}, // L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW + {0x1EAE, 0x1EAE, prUpper}, // L& LATIN CAPITAL LETTER A WITH BREVE AND ACUTE + {0x1EAF, 0x1EAF, prLower}, // L& LATIN SMALL LETTER A WITH BREVE AND ACUTE + {0x1EB0, 0x1EB0, prUpper}, // L& LATIN CAPITAL LETTER A WITH BREVE AND GRAVE + {0x1EB1, 0x1EB1, prLower}, // L& LATIN SMALL LETTER A WITH BREVE AND GRAVE + {0x1EB2, 0x1EB2, prUpper}, // L& LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE + {0x1EB3, 0x1EB3, prLower}, // L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE + {0x1EB4, 0x1EB4, prUpper}, // L& LATIN CAPITAL LETTER A WITH BREVE AND TILDE + {0x1EB5, 0x1EB5, prLower}, // L& LATIN SMALL LETTER A WITH BREVE AND TILDE + {0x1EB6, 0x1EB6, prUpper}, // L& LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW + {0x1EB7, 0x1EB7, prLower}, // L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW + {0x1EB8, 0x1EB8, prUpper}, // L& LATIN CAPITAL LETTER E WITH DOT BELOW + {0x1EB9, 0x1EB9, prLower}, // L& LATIN SMALL LETTER E WITH DOT BELOW + {0x1EBA, 0x1EBA, prUpper}, // L& LATIN CAPITAL LETTER E WITH HOOK ABOVE + {0x1EBB, 0x1EBB, prLower}, // L& LATIN SMALL LETTER E WITH HOOK ABOVE + {0x1EBC, 0x1EBC, prUpper}, // L& LATIN CAPITAL LETTER E WITH TILDE + {0x1EBD, 0x1EBD, prLower}, // L& LATIN SMALL LETTER E WITH TILDE + {0x1EBE, 0x1EBE, prUpper}, // L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE + {0x1EBF, 0x1EBF, prLower}, // L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE + {0x1EC0, 0x1EC0, prUpper}, // L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE + {0x1EC1, 0x1EC1, prLower}, // L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE + {0x1EC2, 0x1EC2, prUpper}, // L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + {0x1EC3, 0x1EC3, prLower}, // L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + {0x1EC4, 0x1EC4, prUpper}, // L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE + {0x1EC5, 0x1EC5, prLower}, // L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE + {0x1EC6, 0x1EC6, prUpper}, // L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW + {0x1EC7, 0x1EC7, prLower}, // L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW + {0x1EC8, 0x1EC8, prUpper}, // L& LATIN CAPITAL LETTER I WITH HOOK ABOVE + {0x1EC9, 0x1EC9, prLower}, // L& LATIN SMALL LETTER I WITH HOOK ABOVE + {0x1ECA, 0x1ECA, prUpper}, // L& LATIN CAPITAL LETTER I WITH DOT BELOW + {0x1ECB, 0x1ECB, prLower}, // L& LATIN SMALL LETTER I WITH DOT BELOW + {0x1ECC, 0x1ECC, prUpper}, // L& LATIN CAPITAL LETTER O WITH DOT BELOW + {0x1ECD, 0x1ECD, prLower}, // L& LATIN SMALL LETTER O WITH DOT BELOW + {0x1ECE, 0x1ECE, prUpper}, // L& LATIN CAPITAL LETTER O WITH HOOK ABOVE + {0x1ECF, 0x1ECF, prLower}, // L& LATIN SMALL LETTER O WITH HOOK ABOVE + {0x1ED0, 0x1ED0, prUpper}, // L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE + {0x1ED1, 0x1ED1, prLower}, // L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE + {0x1ED2, 0x1ED2, prUpper}, // L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE + {0x1ED3, 0x1ED3, prLower}, // L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE + {0x1ED4, 0x1ED4, prUpper}, // L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + {0x1ED5, 0x1ED5, prLower}, // L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + {0x1ED6, 0x1ED6, prUpper}, // L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE + {0x1ED7, 0x1ED7, prLower}, // L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE + {0x1ED8, 0x1ED8, prUpper}, // L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW + {0x1ED9, 0x1ED9, prLower}, // L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW + {0x1EDA, 0x1EDA, prUpper}, // L& LATIN CAPITAL LETTER O WITH HORN AND ACUTE + {0x1EDB, 0x1EDB, prLower}, // L& LATIN SMALL LETTER O WITH HORN AND ACUTE + {0x1EDC, 0x1EDC, prUpper}, // L& LATIN CAPITAL LETTER O WITH HORN AND GRAVE + {0x1EDD, 0x1EDD, prLower}, // L& LATIN SMALL LETTER O WITH HORN AND GRAVE + {0x1EDE, 0x1EDE, prUpper}, // L& LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE + {0x1EDF, 0x1EDF, prLower}, // L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE + {0x1EE0, 0x1EE0, prUpper}, // L& LATIN CAPITAL LETTER O WITH HORN AND TILDE + {0x1EE1, 0x1EE1, prLower}, // L& LATIN SMALL LETTER O WITH HORN AND TILDE + {0x1EE2, 0x1EE2, prUpper}, // L& LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW + {0x1EE3, 0x1EE3, prLower}, // L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW + {0x1EE4, 0x1EE4, prUpper}, // L& LATIN CAPITAL LETTER U WITH DOT BELOW + {0x1EE5, 0x1EE5, prLower}, // L& LATIN SMALL LETTER U WITH DOT BELOW + {0x1EE6, 0x1EE6, prUpper}, // L& LATIN CAPITAL LETTER U WITH HOOK ABOVE + {0x1EE7, 0x1EE7, prLower}, // L& LATIN SMALL LETTER U WITH HOOK ABOVE + {0x1EE8, 0x1EE8, prUpper}, // L& LATIN CAPITAL LETTER U WITH HORN AND ACUTE + {0x1EE9, 0x1EE9, prLower}, // L& LATIN SMALL LETTER U WITH HORN AND ACUTE + {0x1EEA, 0x1EEA, prUpper}, // L& LATIN CAPITAL LETTER U WITH HORN AND GRAVE + {0x1EEB, 0x1EEB, prLower}, // L& LATIN SMALL LETTER U WITH HORN AND GRAVE + {0x1EEC, 0x1EEC, prUpper}, // L& LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE + {0x1EED, 0x1EED, prLower}, // L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE + {0x1EEE, 0x1EEE, prUpper}, // L& LATIN CAPITAL LETTER U WITH HORN AND TILDE + {0x1EEF, 0x1EEF, prLower}, // L& LATIN SMALL LETTER U WITH HORN AND TILDE + {0x1EF0, 0x1EF0, prUpper}, // L& LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW + {0x1EF1, 0x1EF1, prLower}, // L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW + {0x1EF2, 0x1EF2, prUpper}, // L& LATIN CAPITAL LETTER Y WITH GRAVE + {0x1EF3, 0x1EF3, prLower}, // L& LATIN SMALL LETTER Y WITH GRAVE + {0x1EF4, 0x1EF4, prUpper}, // L& LATIN CAPITAL LETTER Y WITH DOT BELOW + {0x1EF5, 0x1EF5, prLower}, // L& LATIN SMALL LETTER Y WITH DOT BELOW + {0x1EF6, 0x1EF6, prUpper}, // L& LATIN CAPITAL LETTER Y WITH HOOK ABOVE + {0x1EF7, 0x1EF7, prLower}, // L& LATIN SMALL LETTER Y WITH HOOK ABOVE + {0x1EF8, 0x1EF8, prUpper}, // L& LATIN CAPITAL LETTER Y WITH TILDE + {0x1EF9, 0x1EF9, prLower}, // L& LATIN SMALL LETTER Y WITH TILDE + {0x1EFA, 0x1EFA, prUpper}, // L& LATIN CAPITAL LETTER MIDDLE-WELSH LL + {0x1EFB, 0x1EFB, prLower}, // L& LATIN SMALL LETTER MIDDLE-WELSH LL + {0x1EFC, 0x1EFC, prUpper}, // L& LATIN CAPITAL LETTER MIDDLE-WELSH V + {0x1EFD, 0x1EFD, prLower}, // L& LATIN SMALL LETTER MIDDLE-WELSH V + {0x1EFE, 0x1EFE, prUpper}, // L& LATIN CAPITAL LETTER Y WITH LOOP + {0x1EFF, 0x1F07, prLower}, // L& [9] LATIN SMALL LETTER Y WITH LOOP..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI + {0x1F08, 0x1F0F, prUpper}, // L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI + {0x1F10, 0x1F15, prLower}, // L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA + {0x1F18, 0x1F1D, prUpper}, // L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + {0x1F20, 0x1F27, prLower}, // L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI + {0x1F28, 0x1F2F, prUpper}, // L& [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI + {0x1F30, 0x1F37, prLower}, // L& [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI + {0x1F38, 0x1F3F, prUpper}, // L& [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI + {0x1F40, 0x1F45, prLower}, // L& [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA + {0x1F48, 0x1F4D, prUpper}, // L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + {0x1F50, 0x1F57, prLower}, // L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI + {0x1F59, 0x1F59, prUpper}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA + {0x1F5B, 0x1F5B, prUpper}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + {0x1F5D, 0x1F5D, prUpper}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + {0x1F5F, 0x1F5F, prUpper}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI + {0x1F60, 0x1F67, prLower}, // L& [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI + {0x1F68, 0x1F6F, prUpper}, // L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI + {0x1F70, 0x1F7D, prLower}, // L& [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA + {0x1F80, 0x1F87, prLower}, // L& [8] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + {0x1F88, 0x1F8F, prUpper}, // L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + {0x1F90, 0x1F97, prLower}, // L& [8] GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + {0x1F98, 0x1F9F, prUpper}, // L& [8] GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + {0x1FA0, 0x1FA7, prLower}, // L& [8] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + {0x1FA8, 0x1FAF, prUpper}, // L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + {0x1FB0, 0x1FB4, prLower}, // L& [5] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + {0x1FB6, 0x1FB7, prLower}, // L& [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + {0x1FB8, 0x1FBC, prUpper}, // L& [5] GREEK CAPITAL LETTER ALPHA WITH VRACHY..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + {0x1FBE, 0x1FBE, prLower}, // L& GREEK PROSGEGRAMMENI + {0x1FC2, 0x1FC4, prLower}, // L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + {0x1FC6, 0x1FC7, prLower}, // L& [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + {0x1FC8, 0x1FCC, prUpper}, // L& [5] GREEK CAPITAL LETTER EPSILON WITH VARIA..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + {0x1FD0, 0x1FD3, prLower}, // L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + {0x1FD6, 0x1FD7, prLower}, // L& [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + {0x1FD8, 0x1FDB, prUpper}, // L& [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA + {0x1FE0, 0x1FE7, prLower}, // L& [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + {0x1FE8, 0x1FEC, prUpper}, // L& [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA + {0x1FF2, 0x1FF4, prLower}, // L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + {0x1FF6, 0x1FF7, prLower}, // L& [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + {0x1FF8, 0x1FFC, prUpper}, // L& [5] GREEK CAPITAL LETTER OMICRON WITH VARIA..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + {0x2000, 0x200A, prSp}, // Zs [11] EN QUAD..HAIR SPACE + {0x200B, 0x200B, prFormat}, // Cf ZERO WIDTH SPACE + {0x200C, 0x200D, prExtend}, // Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER + {0x200E, 0x200F, prFormat}, // Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK + {0x2013, 0x2014, prSContinue}, // Pd [2] EN DASH..EM DASH + {0x2018, 0x2018, prClose}, // Pi LEFT SINGLE QUOTATION MARK + {0x2019, 0x2019, prClose}, // Pf RIGHT SINGLE QUOTATION MARK + {0x201A, 0x201A, prClose}, // Ps SINGLE LOW-9 QUOTATION MARK + {0x201B, 0x201C, prClose}, // Pi [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK + {0x201D, 0x201D, prClose}, // Pf RIGHT DOUBLE QUOTATION MARK + {0x201E, 0x201E, prClose}, // Ps DOUBLE LOW-9 QUOTATION MARK + {0x201F, 0x201F, prClose}, // Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK + {0x2024, 0x2024, prATerm}, // Po ONE DOT LEADER + {0x2028, 0x2028, prSep}, // Zl LINE SEPARATOR + {0x2029, 0x2029, prSep}, // Zp PARAGRAPH SEPARATOR + {0x202A, 0x202E, prFormat}, // Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE + {0x202F, 0x202F, prSp}, // Zs NARROW NO-BREAK SPACE + {0x2039, 0x2039, prClose}, // Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK + {0x203A, 0x203A, prClose}, // Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + {0x203C, 0x203D, prSTerm}, // Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG + {0x2045, 0x2045, prClose}, // Ps LEFT SQUARE BRACKET WITH QUILL + {0x2046, 0x2046, prClose}, // Pe RIGHT SQUARE BRACKET WITH QUILL + {0x2047, 0x2049, prSTerm}, // Po [3] DOUBLE QUESTION MARK..EXCLAMATION QUESTION MARK + {0x205F, 0x205F, prSp}, // Zs MEDIUM MATHEMATICAL SPACE + {0x2060, 0x2064, prFormat}, // Cf [5] WORD JOINER..INVISIBLE PLUS + {0x2066, 0x206F, prFormat}, // Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES + {0x2071, 0x2071, prLower}, // Lm SUPERSCRIPT LATIN SMALL LETTER I + {0x207D, 0x207D, prClose}, // Ps SUPERSCRIPT LEFT PARENTHESIS + {0x207E, 0x207E, prClose}, // Pe SUPERSCRIPT RIGHT PARENTHESIS + {0x207F, 0x207F, prLower}, // Lm SUPERSCRIPT LATIN SMALL LETTER N + {0x208D, 0x208D, prClose}, // Ps SUBSCRIPT LEFT PARENTHESIS + {0x208E, 0x208E, prClose}, // Pe SUBSCRIPT RIGHT PARENTHESIS + {0x2090, 0x209C, prLower}, // Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T + {0x20D0, 0x20DC, prExtend}, // Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE + {0x20DD, 0x20E0, prExtend}, // Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH + {0x20E1, 0x20E1, prExtend}, // Mn COMBINING LEFT RIGHT ARROW ABOVE + {0x20E2, 0x20E4, prExtend}, // Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE + {0x20E5, 0x20F0, prExtend}, // Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE + {0x2102, 0x2102, prUpper}, // L& DOUBLE-STRUCK CAPITAL C + {0x2107, 0x2107, prUpper}, // L& EULER CONSTANT + {0x210A, 0x210A, prLower}, // L& SCRIPT SMALL G + {0x210B, 0x210D, prUpper}, // L& [3] SCRIPT CAPITAL H..DOUBLE-STRUCK CAPITAL H + {0x210E, 0x210F, prLower}, // L& [2] PLANCK CONSTANT..PLANCK CONSTANT OVER TWO PI + {0x2110, 0x2112, prUpper}, // L& [3] SCRIPT CAPITAL I..SCRIPT CAPITAL L + {0x2113, 0x2113, prLower}, // L& SCRIPT SMALL L + {0x2115, 0x2115, prUpper}, // L& DOUBLE-STRUCK CAPITAL N + {0x2119, 0x211D, prUpper}, // L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R + {0x2124, 0x2124, prUpper}, // L& DOUBLE-STRUCK CAPITAL Z + {0x2126, 0x2126, prUpper}, // L& OHM SIGN + {0x2128, 0x2128, prUpper}, // L& BLACK-LETTER CAPITAL Z + {0x212A, 0x212D, prUpper}, // L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C + {0x212F, 0x212F, prLower}, // L& SCRIPT SMALL E + {0x2130, 0x2133, prUpper}, // L& [4] SCRIPT CAPITAL E..SCRIPT CAPITAL M + {0x2134, 0x2134, prLower}, // L& SCRIPT SMALL O + {0x2135, 0x2138, prOLetter}, // Lo [4] ALEF SYMBOL..DALET SYMBOL + {0x2139, 0x2139, prLower}, // L& INFORMATION SOURCE + {0x213C, 0x213D, prLower}, // L& [2] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK SMALL GAMMA + {0x213E, 0x213F, prUpper}, // L& [2] DOUBLE-STRUCK CAPITAL GAMMA..DOUBLE-STRUCK CAPITAL PI + {0x2145, 0x2145, prUpper}, // L& DOUBLE-STRUCK ITALIC CAPITAL D + {0x2146, 0x2149, prLower}, // L& [4] DOUBLE-STRUCK ITALIC SMALL D..DOUBLE-STRUCK ITALIC SMALL J + {0x214E, 0x214E, prLower}, // L& TURNED SMALL F + {0x2160, 0x216F, prUpper}, // Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND + {0x2170, 0x217F, prLower}, // Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND + {0x2180, 0x2182, prOLetter}, // Nl [3] ROMAN NUMERAL ONE THOUSAND C D..ROMAN NUMERAL TEN THOUSAND + {0x2183, 0x2183, prUpper}, // L& ROMAN NUMERAL REVERSED ONE HUNDRED + {0x2184, 0x2184, prLower}, // L& LATIN SMALL LETTER REVERSED C + {0x2185, 0x2188, prOLetter}, // Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND + {0x2308, 0x2308, prClose}, // Ps LEFT CEILING + {0x2309, 0x2309, prClose}, // Pe RIGHT CEILING + {0x230A, 0x230A, prClose}, // Ps LEFT FLOOR + {0x230B, 0x230B, prClose}, // Pe RIGHT FLOOR + {0x2329, 0x2329, prClose}, // Ps LEFT-POINTING ANGLE BRACKET + {0x232A, 0x232A, prClose}, // Pe RIGHT-POINTING ANGLE BRACKET + {0x24B6, 0x24CF, prUpper}, // So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z + {0x24D0, 0x24E9, prLower}, // So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z + {0x275B, 0x2760, prClose}, // So [6] HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT..HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT + {0x2768, 0x2768, prClose}, // Ps MEDIUM LEFT PARENTHESIS ORNAMENT + {0x2769, 0x2769, prClose}, // Pe MEDIUM RIGHT PARENTHESIS ORNAMENT + {0x276A, 0x276A, prClose}, // Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT + {0x276B, 0x276B, prClose}, // Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT + {0x276C, 0x276C, prClose}, // Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT + {0x276D, 0x276D, prClose}, // Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT + {0x276E, 0x276E, prClose}, // Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT + {0x276F, 0x276F, prClose}, // Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT + {0x2770, 0x2770, prClose}, // Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT + {0x2771, 0x2771, prClose}, // Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT + {0x2772, 0x2772, prClose}, // Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT + {0x2773, 0x2773, prClose}, // Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT + {0x2774, 0x2774, prClose}, // Ps MEDIUM LEFT CURLY BRACKET ORNAMENT + {0x2775, 0x2775, prClose}, // Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT + {0x27C5, 0x27C5, prClose}, // Ps LEFT S-SHAPED BAG DELIMITER + {0x27C6, 0x27C6, prClose}, // Pe RIGHT S-SHAPED BAG DELIMITER + {0x27E6, 0x27E6, prClose}, // Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET + {0x27E7, 0x27E7, prClose}, // Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET + {0x27E8, 0x27E8, prClose}, // Ps MATHEMATICAL LEFT ANGLE BRACKET + {0x27E9, 0x27E9, prClose}, // Pe MATHEMATICAL RIGHT ANGLE BRACKET + {0x27EA, 0x27EA, prClose}, // Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET + {0x27EB, 0x27EB, prClose}, // Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET + {0x27EC, 0x27EC, prClose}, // Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET + {0x27ED, 0x27ED, prClose}, // Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET + {0x27EE, 0x27EE, prClose}, // Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS + {0x27EF, 0x27EF, prClose}, // Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS + {0x2983, 0x2983, prClose}, // Ps LEFT WHITE CURLY BRACKET + {0x2984, 0x2984, prClose}, // Pe RIGHT WHITE CURLY BRACKET + {0x2985, 0x2985, prClose}, // Ps LEFT WHITE PARENTHESIS + {0x2986, 0x2986, prClose}, // Pe RIGHT WHITE PARENTHESIS + {0x2987, 0x2987, prClose}, // Ps Z NOTATION LEFT IMAGE BRACKET + {0x2988, 0x2988, prClose}, // Pe Z NOTATION RIGHT IMAGE BRACKET + {0x2989, 0x2989, prClose}, // Ps Z NOTATION LEFT BINDING BRACKET + {0x298A, 0x298A, prClose}, // Pe Z NOTATION RIGHT BINDING BRACKET + {0x298B, 0x298B, prClose}, // Ps LEFT SQUARE BRACKET WITH UNDERBAR + {0x298C, 0x298C, prClose}, // Pe RIGHT SQUARE BRACKET WITH UNDERBAR + {0x298D, 0x298D, prClose}, // Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER + {0x298E, 0x298E, prClose}, // Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + {0x298F, 0x298F, prClose}, // Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + {0x2990, 0x2990, prClose}, // Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER + {0x2991, 0x2991, prClose}, // Ps LEFT ANGLE BRACKET WITH DOT + {0x2992, 0x2992, prClose}, // Pe RIGHT ANGLE BRACKET WITH DOT + {0x2993, 0x2993, prClose}, // Ps LEFT ARC LESS-THAN BRACKET + {0x2994, 0x2994, prClose}, // Pe RIGHT ARC GREATER-THAN BRACKET + {0x2995, 0x2995, prClose}, // Ps DOUBLE LEFT ARC GREATER-THAN BRACKET + {0x2996, 0x2996, prClose}, // Pe DOUBLE RIGHT ARC LESS-THAN BRACKET + {0x2997, 0x2997, prClose}, // Ps LEFT BLACK TORTOISE SHELL BRACKET + {0x2998, 0x2998, prClose}, // Pe RIGHT BLACK TORTOISE SHELL BRACKET + {0x29D8, 0x29D8, prClose}, // Ps LEFT WIGGLY FENCE + {0x29D9, 0x29D9, prClose}, // Pe RIGHT WIGGLY FENCE + {0x29DA, 0x29DA, prClose}, // Ps LEFT DOUBLE WIGGLY FENCE + {0x29DB, 0x29DB, prClose}, // Pe RIGHT DOUBLE WIGGLY FENCE + {0x29FC, 0x29FC, prClose}, // Ps LEFT-POINTING CURVED ANGLE BRACKET + {0x29FD, 0x29FD, prClose}, // Pe RIGHT-POINTING CURVED ANGLE BRACKET + {0x2C00, 0x2C2F, prUpper}, // L& [48] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI + {0x2C30, 0x2C5F, prLower}, // L& [48] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER CAUDATE CHRIVI + {0x2C60, 0x2C60, prUpper}, // L& LATIN CAPITAL LETTER L WITH DOUBLE BAR + {0x2C61, 0x2C61, prLower}, // L& LATIN SMALL LETTER L WITH DOUBLE BAR + {0x2C62, 0x2C64, prUpper}, // L& [3] LATIN CAPITAL LETTER L WITH MIDDLE TILDE..LATIN CAPITAL LETTER R WITH TAIL + {0x2C65, 0x2C66, prLower}, // L& [2] LATIN SMALL LETTER A WITH STROKE..LATIN SMALL LETTER T WITH DIAGONAL STROKE + {0x2C67, 0x2C67, prUpper}, // L& LATIN CAPITAL LETTER H WITH DESCENDER + {0x2C68, 0x2C68, prLower}, // L& LATIN SMALL LETTER H WITH DESCENDER + {0x2C69, 0x2C69, prUpper}, // L& LATIN CAPITAL LETTER K WITH DESCENDER + {0x2C6A, 0x2C6A, prLower}, // L& LATIN SMALL LETTER K WITH DESCENDER + {0x2C6B, 0x2C6B, prUpper}, // L& LATIN CAPITAL LETTER Z WITH DESCENDER + {0x2C6C, 0x2C6C, prLower}, // L& LATIN SMALL LETTER Z WITH DESCENDER + {0x2C6D, 0x2C70, prUpper}, // L& [4] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED ALPHA + {0x2C71, 0x2C71, prLower}, // L& LATIN SMALL LETTER V WITH RIGHT HOOK + {0x2C72, 0x2C72, prUpper}, // L& LATIN CAPITAL LETTER W WITH HOOK + {0x2C73, 0x2C74, prLower}, // L& [2] LATIN SMALL LETTER W WITH HOOK..LATIN SMALL LETTER V WITH CURL + {0x2C75, 0x2C75, prUpper}, // L& LATIN CAPITAL LETTER HALF H + {0x2C76, 0x2C7B, prLower}, // L& [6] LATIN SMALL LETTER HALF H..LATIN LETTER SMALL CAPITAL TURNED E + {0x2C7C, 0x2C7D, prLower}, // Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V + {0x2C7E, 0x2C80, prUpper}, // L& [3] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC CAPITAL LETTER ALFA + {0x2C81, 0x2C81, prLower}, // L& COPTIC SMALL LETTER ALFA + {0x2C82, 0x2C82, prUpper}, // L& COPTIC CAPITAL LETTER VIDA + {0x2C83, 0x2C83, prLower}, // L& COPTIC SMALL LETTER VIDA + {0x2C84, 0x2C84, prUpper}, // L& COPTIC CAPITAL LETTER GAMMA + {0x2C85, 0x2C85, prLower}, // L& COPTIC SMALL LETTER GAMMA + {0x2C86, 0x2C86, prUpper}, // L& COPTIC CAPITAL LETTER DALDA + {0x2C87, 0x2C87, prLower}, // L& COPTIC SMALL LETTER DALDA + {0x2C88, 0x2C88, prUpper}, // L& COPTIC CAPITAL LETTER EIE + {0x2C89, 0x2C89, prLower}, // L& COPTIC SMALL LETTER EIE + {0x2C8A, 0x2C8A, prUpper}, // L& COPTIC CAPITAL LETTER SOU + {0x2C8B, 0x2C8B, prLower}, // L& COPTIC SMALL LETTER SOU + {0x2C8C, 0x2C8C, prUpper}, // L& COPTIC CAPITAL LETTER ZATA + {0x2C8D, 0x2C8D, prLower}, // L& COPTIC SMALL LETTER ZATA + {0x2C8E, 0x2C8E, prUpper}, // L& COPTIC CAPITAL LETTER HATE + {0x2C8F, 0x2C8F, prLower}, // L& COPTIC SMALL LETTER HATE + {0x2C90, 0x2C90, prUpper}, // L& COPTIC CAPITAL LETTER THETHE + {0x2C91, 0x2C91, prLower}, // L& COPTIC SMALL LETTER THETHE + {0x2C92, 0x2C92, prUpper}, // L& COPTIC CAPITAL LETTER IAUDA + {0x2C93, 0x2C93, prLower}, // L& COPTIC SMALL LETTER IAUDA + {0x2C94, 0x2C94, prUpper}, // L& COPTIC CAPITAL LETTER KAPA + {0x2C95, 0x2C95, prLower}, // L& COPTIC SMALL LETTER KAPA + {0x2C96, 0x2C96, prUpper}, // L& COPTIC CAPITAL LETTER LAULA + {0x2C97, 0x2C97, prLower}, // L& COPTIC SMALL LETTER LAULA + {0x2C98, 0x2C98, prUpper}, // L& COPTIC CAPITAL LETTER MI + {0x2C99, 0x2C99, prLower}, // L& COPTIC SMALL LETTER MI + {0x2C9A, 0x2C9A, prUpper}, // L& COPTIC CAPITAL LETTER NI + {0x2C9B, 0x2C9B, prLower}, // L& COPTIC SMALL LETTER NI + {0x2C9C, 0x2C9C, prUpper}, // L& COPTIC CAPITAL LETTER KSI + {0x2C9D, 0x2C9D, prLower}, // L& COPTIC SMALL LETTER KSI + {0x2C9E, 0x2C9E, prUpper}, // L& COPTIC CAPITAL LETTER O + {0x2C9F, 0x2C9F, prLower}, // L& COPTIC SMALL LETTER O + {0x2CA0, 0x2CA0, prUpper}, // L& COPTIC CAPITAL LETTER PI + {0x2CA1, 0x2CA1, prLower}, // L& COPTIC SMALL LETTER PI + {0x2CA2, 0x2CA2, prUpper}, // L& COPTIC CAPITAL LETTER RO + {0x2CA3, 0x2CA3, prLower}, // L& COPTIC SMALL LETTER RO + {0x2CA4, 0x2CA4, prUpper}, // L& COPTIC CAPITAL LETTER SIMA + {0x2CA5, 0x2CA5, prLower}, // L& COPTIC SMALL LETTER SIMA + {0x2CA6, 0x2CA6, prUpper}, // L& COPTIC CAPITAL LETTER TAU + {0x2CA7, 0x2CA7, prLower}, // L& COPTIC SMALL LETTER TAU + {0x2CA8, 0x2CA8, prUpper}, // L& COPTIC CAPITAL LETTER UA + {0x2CA9, 0x2CA9, prLower}, // L& COPTIC SMALL LETTER UA + {0x2CAA, 0x2CAA, prUpper}, // L& COPTIC CAPITAL LETTER FI + {0x2CAB, 0x2CAB, prLower}, // L& COPTIC SMALL LETTER FI + {0x2CAC, 0x2CAC, prUpper}, // L& COPTIC CAPITAL LETTER KHI + {0x2CAD, 0x2CAD, prLower}, // L& COPTIC SMALL LETTER KHI + {0x2CAE, 0x2CAE, prUpper}, // L& COPTIC CAPITAL LETTER PSI + {0x2CAF, 0x2CAF, prLower}, // L& COPTIC SMALL LETTER PSI + {0x2CB0, 0x2CB0, prUpper}, // L& COPTIC CAPITAL LETTER OOU + {0x2CB1, 0x2CB1, prLower}, // L& COPTIC SMALL LETTER OOU + {0x2CB2, 0x2CB2, prUpper}, // L& COPTIC CAPITAL LETTER DIALECT-P ALEF + {0x2CB3, 0x2CB3, prLower}, // L& COPTIC SMALL LETTER DIALECT-P ALEF + {0x2CB4, 0x2CB4, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC AIN + {0x2CB5, 0x2CB5, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC AIN + {0x2CB6, 0x2CB6, prUpper}, // L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE + {0x2CB7, 0x2CB7, prLower}, // L& COPTIC SMALL LETTER CRYPTOGRAMMIC EIE + {0x2CB8, 0x2CB8, prUpper}, // L& COPTIC CAPITAL LETTER DIALECT-P KAPA + {0x2CB9, 0x2CB9, prLower}, // L& COPTIC SMALL LETTER DIALECT-P KAPA + {0x2CBA, 0x2CBA, prUpper}, // L& COPTIC CAPITAL LETTER DIALECT-P NI + {0x2CBB, 0x2CBB, prLower}, // L& COPTIC SMALL LETTER DIALECT-P NI + {0x2CBC, 0x2CBC, prUpper}, // L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI + {0x2CBD, 0x2CBD, prLower}, // L& COPTIC SMALL LETTER CRYPTOGRAMMIC NI + {0x2CBE, 0x2CBE, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC OOU + {0x2CBF, 0x2CBF, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC OOU + {0x2CC0, 0x2CC0, prUpper}, // L& COPTIC CAPITAL LETTER SAMPI + {0x2CC1, 0x2CC1, prLower}, // L& COPTIC SMALL LETTER SAMPI + {0x2CC2, 0x2CC2, prUpper}, // L& COPTIC CAPITAL LETTER CROSSED SHEI + {0x2CC3, 0x2CC3, prLower}, // L& COPTIC SMALL LETTER CROSSED SHEI + {0x2CC4, 0x2CC4, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC SHEI + {0x2CC5, 0x2CC5, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC SHEI + {0x2CC6, 0x2CC6, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC ESH + {0x2CC7, 0x2CC7, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC ESH + {0x2CC8, 0x2CC8, prUpper}, // L& COPTIC CAPITAL LETTER AKHMIMIC KHEI + {0x2CC9, 0x2CC9, prLower}, // L& COPTIC SMALL LETTER AKHMIMIC KHEI + {0x2CCA, 0x2CCA, prUpper}, // L& COPTIC CAPITAL LETTER DIALECT-P HORI + {0x2CCB, 0x2CCB, prLower}, // L& COPTIC SMALL LETTER DIALECT-P HORI + {0x2CCC, 0x2CCC, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC HORI + {0x2CCD, 0x2CCD, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC HORI + {0x2CCE, 0x2CCE, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC HA + {0x2CCF, 0x2CCF, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC HA + {0x2CD0, 0x2CD0, prUpper}, // L& COPTIC CAPITAL LETTER L-SHAPED HA + {0x2CD1, 0x2CD1, prLower}, // L& COPTIC SMALL LETTER L-SHAPED HA + {0x2CD2, 0x2CD2, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC HEI + {0x2CD3, 0x2CD3, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC HEI + {0x2CD4, 0x2CD4, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC HAT + {0x2CD5, 0x2CD5, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC HAT + {0x2CD6, 0x2CD6, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC GANGIA + {0x2CD7, 0x2CD7, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC GANGIA + {0x2CD8, 0x2CD8, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC DJA + {0x2CD9, 0x2CD9, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC DJA + {0x2CDA, 0x2CDA, prUpper}, // L& COPTIC CAPITAL LETTER OLD COPTIC SHIMA + {0x2CDB, 0x2CDB, prLower}, // L& COPTIC SMALL LETTER OLD COPTIC SHIMA + {0x2CDC, 0x2CDC, prUpper}, // L& COPTIC CAPITAL LETTER OLD NUBIAN SHIMA + {0x2CDD, 0x2CDD, prLower}, // L& COPTIC SMALL LETTER OLD NUBIAN SHIMA + {0x2CDE, 0x2CDE, prUpper}, // L& COPTIC CAPITAL LETTER OLD NUBIAN NGI + {0x2CDF, 0x2CDF, prLower}, // L& COPTIC SMALL LETTER OLD NUBIAN NGI + {0x2CE0, 0x2CE0, prUpper}, // L& COPTIC CAPITAL LETTER OLD NUBIAN NYI + {0x2CE1, 0x2CE1, prLower}, // L& COPTIC SMALL LETTER OLD NUBIAN NYI + {0x2CE2, 0x2CE2, prUpper}, // L& COPTIC CAPITAL LETTER OLD NUBIAN WAU + {0x2CE3, 0x2CE4, prLower}, // L& [2] COPTIC SMALL LETTER OLD NUBIAN WAU..COPTIC SYMBOL KAI + {0x2CEB, 0x2CEB, prUpper}, // L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI + {0x2CEC, 0x2CEC, prLower}, // L& COPTIC SMALL LETTER CRYPTOGRAMMIC SHEI + {0x2CED, 0x2CED, prUpper}, // L& COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA + {0x2CEE, 0x2CEE, prLower}, // L& COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA + {0x2CEF, 0x2CF1, prExtend}, // Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS + {0x2CF2, 0x2CF2, prUpper}, // L& COPTIC CAPITAL LETTER BOHAIRIC KHEI + {0x2CF3, 0x2CF3, prLower}, // L& COPTIC SMALL LETTER BOHAIRIC KHEI + {0x2D00, 0x2D25, prLower}, // L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE + {0x2D27, 0x2D27, prLower}, // L& GEORGIAN SMALL LETTER YN + {0x2D2D, 0x2D2D, prLower}, // L& GEORGIAN SMALL LETTER AEN + {0x2D30, 0x2D67, prOLetter}, // Lo [56] TIFINAGH LETTER YA..TIFINAGH LETTER YO + {0x2D6F, 0x2D6F, prOLetter}, // Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK + {0x2D7F, 0x2D7F, prExtend}, // Mn TIFINAGH CONSONANT JOINER + {0x2D80, 0x2D96, prOLetter}, // Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE + {0x2DA0, 0x2DA6, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO + {0x2DA8, 0x2DAE, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO + {0x2DB0, 0x2DB6, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO + {0x2DB8, 0x2DBE, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO + {0x2DC0, 0x2DC6, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO + {0x2DC8, 0x2DCE, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO + {0x2DD0, 0x2DD6, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO + {0x2DD8, 0x2DDE, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO + {0x2DE0, 0x2DFF, prExtend}, // Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS + {0x2E00, 0x2E01, prClose}, // Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER + {0x2E02, 0x2E02, prClose}, // Pi LEFT SUBSTITUTION BRACKET + {0x2E03, 0x2E03, prClose}, // Pf RIGHT SUBSTITUTION BRACKET + {0x2E04, 0x2E04, prClose}, // Pi LEFT DOTTED SUBSTITUTION BRACKET + {0x2E05, 0x2E05, prClose}, // Pf RIGHT DOTTED SUBSTITUTION BRACKET + {0x2E06, 0x2E08, prClose}, // Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER + {0x2E09, 0x2E09, prClose}, // Pi LEFT TRANSPOSITION BRACKET + {0x2E0A, 0x2E0A, prClose}, // Pf RIGHT TRANSPOSITION BRACKET + {0x2E0B, 0x2E0B, prClose}, // Po RAISED SQUARE + {0x2E0C, 0x2E0C, prClose}, // Pi LEFT RAISED OMISSION BRACKET + {0x2E0D, 0x2E0D, prClose}, // Pf RIGHT RAISED OMISSION BRACKET + {0x2E1C, 0x2E1C, prClose}, // Pi LEFT LOW PARAPHRASE BRACKET + {0x2E1D, 0x2E1D, prClose}, // Pf RIGHT LOW PARAPHRASE BRACKET + {0x2E20, 0x2E20, prClose}, // Pi LEFT VERTICAL BAR WITH QUILL + {0x2E21, 0x2E21, prClose}, // Pf RIGHT VERTICAL BAR WITH QUILL + {0x2E22, 0x2E22, prClose}, // Ps TOP LEFT HALF BRACKET + {0x2E23, 0x2E23, prClose}, // Pe TOP RIGHT HALF BRACKET + {0x2E24, 0x2E24, prClose}, // Ps BOTTOM LEFT HALF BRACKET + {0x2E25, 0x2E25, prClose}, // Pe BOTTOM RIGHT HALF BRACKET + {0x2E26, 0x2E26, prClose}, // Ps LEFT SIDEWAYS U BRACKET + {0x2E27, 0x2E27, prClose}, // Pe RIGHT SIDEWAYS U BRACKET + {0x2E28, 0x2E28, prClose}, // Ps LEFT DOUBLE PARENTHESIS + {0x2E29, 0x2E29, prClose}, // Pe RIGHT DOUBLE PARENTHESIS + {0x2E2E, 0x2E2E, prSTerm}, // Po REVERSED QUESTION MARK + {0x2E2F, 0x2E2F, prOLetter}, // Lm VERTICAL TILDE + {0x2E3C, 0x2E3C, prSTerm}, // Po STENOGRAPHIC FULL STOP + {0x2E42, 0x2E42, prClose}, // Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK + {0x2E53, 0x2E54, prSTerm}, // Po [2] MEDIEVAL EXCLAMATION MARK..MEDIEVAL QUESTION MARK + {0x2E55, 0x2E55, prClose}, // Ps LEFT SQUARE BRACKET WITH STROKE + {0x2E56, 0x2E56, prClose}, // Pe RIGHT SQUARE BRACKET WITH STROKE + {0x2E57, 0x2E57, prClose}, // Ps LEFT SQUARE BRACKET WITH DOUBLE STROKE + {0x2E58, 0x2E58, prClose}, // Pe RIGHT SQUARE BRACKET WITH DOUBLE STROKE + {0x2E59, 0x2E59, prClose}, // Ps TOP HALF LEFT PARENTHESIS + {0x2E5A, 0x2E5A, prClose}, // Pe TOP HALF RIGHT PARENTHESIS + {0x2E5B, 0x2E5B, prClose}, // Ps BOTTOM HALF LEFT PARENTHESIS + {0x2E5C, 0x2E5C, prClose}, // Pe BOTTOM HALF RIGHT PARENTHESIS + {0x3000, 0x3000, prSp}, // Zs IDEOGRAPHIC SPACE + {0x3001, 0x3001, prSContinue}, // Po IDEOGRAPHIC COMMA + {0x3002, 0x3002, prSTerm}, // Po IDEOGRAPHIC FULL STOP + {0x3005, 0x3005, prOLetter}, // Lm IDEOGRAPHIC ITERATION MARK + {0x3006, 0x3006, prOLetter}, // Lo IDEOGRAPHIC CLOSING MARK + {0x3007, 0x3007, prOLetter}, // Nl IDEOGRAPHIC NUMBER ZERO + {0x3008, 0x3008, prClose}, // Ps LEFT ANGLE BRACKET + {0x3009, 0x3009, prClose}, // Pe RIGHT ANGLE BRACKET + {0x300A, 0x300A, prClose}, // Ps LEFT DOUBLE ANGLE BRACKET + {0x300B, 0x300B, prClose}, // Pe RIGHT DOUBLE ANGLE BRACKET + {0x300C, 0x300C, prClose}, // Ps LEFT CORNER BRACKET + {0x300D, 0x300D, prClose}, // Pe RIGHT CORNER BRACKET + {0x300E, 0x300E, prClose}, // Ps LEFT WHITE CORNER BRACKET + {0x300F, 0x300F, prClose}, // Pe RIGHT WHITE CORNER BRACKET + {0x3010, 0x3010, prClose}, // Ps LEFT BLACK LENTICULAR BRACKET + {0x3011, 0x3011, prClose}, // Pe RIGHT BLACK LENTICULAR BRACKET + {0x3014, 0x3014, prClose}, // Ps LEFT TORTOISE SHELL BRACKET + {0x3015, 0x3015, prClose}, // Pe RIGHT TORTOISE SHELL BRACKET + {0x3016, 0x3016, prClose}, // Ps LEFT WHITE LENTICULAR BRACKET + {0x3017, 0x3017, prClose}, // Pe RIGHT WHITE LENTICULAR BRACKET + {0x3018, 0x3018, prClose}, // Ps LEFT WHITE TORTOISE SHELL BRACKET + {0x3019, 0x3019, prClose}, // Pe RIGHT WHITE TORTOISE SHELL BRACKET + {0x301A, 0x301A, prClose}, // Ps LEFT WHITE SQUARE BRACKET + {0x301B, 0x301B, prClose}, // Pe RIGHT WHITE SQUARE BRACKET + {0x301D, 0x301D, prClose}, // Ps REVERSED DOUBLE PRIME QUOTATION MARK + {0x301E, 0x301F, prClose}, // Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK + {0x3021, 0x3029, prOLetter}, // Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE + {0x302A, 0x302D, prExtend}, // Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK + {0x302E, 0x302F, prExtend}, // Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK + {0x3031, 0x3035, prOLetter}, // Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF + {0x3038, 0x303A, prOLetter}, // Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY + {0x303B, 0x303B, prOLetter}, // Lm VERTICAL IDEOGRAPHIC ITERATION MARK + {0x303C, 0x303C, prOLetter}, // Lo MASU MARK + {0x3041, 0x3096, prOLetter}, // Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE + {0x3099, 0x309A, prExtend}, // Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x309D, 0x309E, prOLetter}, // Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK + {0x309F, 0x309F, prOLetter}, // Lo HIRAGANA DIGRAPH YORI + {0x30A1, 0x30FA, prOLetter}, // Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO + {0x30FC, 0x30FE, prOLetter}, // Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK + {0x30FF, 0x30FF, prOLetter}, // Lo KATAKANA DIGRAPH KOTO + {0x3105, 0x312F, prOLetter}, // Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN + {0x3131, 0x318E, prOLetter}, // Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE + {0x31A0, 0x31BF, prOLetter}, // Lo [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH + {0x31F0, 0x31FF, prOLetter}, // Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO + {0x3400, 0x4DBF, prOLetter}, // Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF + {0x4E00, 0xA014, prOLetter}, // Lo [21013] CJK UNIFIED IDEOGRAPH-4E00..YI SYLLABLE E + {0xA015, 0xA015, prOLetter}, // Lm YI SYLLABLE WU + {0xA016, 0xA48C, prOLetter}, // Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR + {0xA4D0, 0xA4F7, prOLetter}, // Lo [40] LISU LETTER BA..LISU LETTER OE + {0xA4F8, 0xA4FD, prOLetter}, // Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU + {0xA4FF, 0xA4FF, prSTerm}, // Po LISU PUNCTUATION FULL STOP + {0xA500, 0xA60B, prOLetter}, // Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG + {0xA60C, 0xA60C, prOLetter}, // Lm VAI SYLLABLE LENGTHENER + {0xA60E, 0xA60F, prSTerm}, // Po [2] VAI FULL STOP..VAI QUESTION MARK + {0xA610, 0xA61F, prOLetter}, // Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG + {0xA620, 0xA629, prNumeric}, // Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE + {0xA62A, 0xA62B, prOLetter}, // Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO + {0xA640, 0xA640, prUpper}, // L& CYRILLIC CAPITAL LETTER ZEMLYA + {0xA641, 0xA641, prLower}, // L& CYRILLIC SMALL LETTER ZEMLYA + {0xA642, 0xA642, prUpper}, // L& CYRILLIC CAPITAL LETTER DZELO + {0xA643, 0xA643, prLower}, // L& CYRILLIC SMALL LETTER DZELO + {0xA644, 0xA644, prUpper}, // L& CYRILLIC CAPITAL LETTER REVERSED DZE + {0xA645, 0xA645, prLower}, // L& CYRILLIC SMALL LETTER REVERSED DZE + {0xA646, 0xA646, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTA + {0xA647, 0xA647, prLower}, // L& CYRILLIC SMALL LETTER IOTA + {0xA648, 0xA648, prUpper}, // L& CYRILLIC CAPITAL LETTER DJERV + {0xA649, 0xA649, prLower}, // L& CYRILLIC SMALL LETTER DJERV + {0xA64A, 0xA64A, prUpper}, // L& CYRILLIC CAPITAL LETTER MONOGRAPH UK + {0xA64B, 0xA64B, prLower}, // L& CYRILLIC SMALL LETTER MONOGRAPH UK + {0xA64C, 0xA64C, prUpper}, // L& CYRILLIC CAPITAL LETTER BROAD OMEGA + {0xA64D, 0xA64D, prLower}, // L& CYRILLIC SMALL LETTER BROAD OMEGA + {0xA64E, 0xA64E, prUpper}, // L& CYRILLIC CAPITAL LETTER NEUTRAL YER + {0xA64F, 0xA64F, prLower}, // L& CYRILLIC SMALL LETTER NEUTRAL YER + {0xA650, 0xA650, prUpper}, // L& CYRILLIC CAPITAL LETTER YERU WITH BACK YER + {0xA651, 0xA651, prLower}, // L& CYRILLIC SMALL LETTER YERU WITH BACK YER + {0xA652, 0xA652, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTIFIED YAT + {0xA653, 0xA653, prLower}, // L& CYRILLIC SMALL LETTER IOTIFIED YAT + {0xA654, 0xA654, prUpper}, // L& CYRILLIC CAPITAL LETTER REVERSED YU + {0xA655, 0xA655, prLower}, // L& CYRILLIC SMALL LETTER REVERSED YU + {0xA656, 0xA656, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTIFIED A + {0xA657, 0xA657, prLower}, // L& CYRILLIC SMALL LETTER IOTIFIED A + {0xA658, 0xA658, prUpper}, // L& CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS + {0xA659, 0xA659, prLower}, // L& CYRILLIC SMALL LETTER CLOSED LITTLE YUS + {0xA65A, 0xA65A, prUpper}, // L& CYRILLIC CAPITAL LETTER BLENDED YUS + {0xA65B, 0xA65B, prLower}, // L& CYRILLIC SMALL LETTER BLENDED YUS + {0xA65C, 0xA65C, prUpper}, // L& CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS + {0xA65D, 0xA65D, prLower}, // L& CYRILLIC SMALL LETTER IOTIFIED CLOSED LITTLE YUS + {0xA65E, 0xA65E, prUpper}, // L& CYRILLIC CAPITAL LETTER YN + {0xA65F, 0xA65F, prLower}, // L& CYRILLIC SMALL LETTER YN + {0xA660, 0xA660, prUpper}, // L& CYRILLIC CAPITAL LETTER REVERSED TSE + {0xA661, 0xA661, prLower}, // L& CYRILLIC SMALL LETTER REVERSED TSE + {0xA662, 0xA662, prUpper}, // L& CYRILLIC CAPITAL LETTER SOFT DE + {0xA663, 0xA663, prLower}, // L& CYRILLIC SMALL LETTER SOFT DE + {0xA664, 0xA664, prUpper}, // L& CYRILLIC CAPITAL LETTER SOFT EL + {0xA665, 0xA665, prLower}, // L& CYRILLIC SMALL LETTER SOFT EL + {0xA666, 0xA666, prUpper}, // L& CYRILLIC CAPITAL LETTER SOFT EM + {0xA667, 0xA667, prLower}, // L& CYRILLIC SMALL LETTER SOFT EM + {0xA668, 0xA668, prUpper}, // L& CYRILLIC CAPITAL LETTER MONOCULAR O + {0xA669, 0xA669, prLower}, // L& CYRILLIC SMALL LETTER MONOCULAR O + {0xA66A, 0xA66A, prUpper}, // L& CYRILLIC CAPITAL LETTER BINOCULAR O + {0xA66B, 0xA66B, prLower}, // L& CYRILLIC SMALL LETTER BINOCULAR O + {0xA66C, 0xA66C, prUpper}, // L& CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O + {0xA66D, 0xA66D, prLower}, // L& CYRILLIC SMALL LETTER DOUBLE MONOCULAR O + {0xA66E, 0xA66E, prOLetter}, // Lo CYRILLIC LETTER MULTIOCULAR O + {0xA66F, 0xA66F, prExtend}, // Mn COMBINING CYRILLIC VZMET + {0xA670, 0xA672, prExtend}, // Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN + {0xA674, 0xA67D, prExtend}, // Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK + {0xA67F, 0xA67F, prOLetter}, // Lm CYRILLIC PAYEROK + {0xA680, 0xA680, prUpper}, // L& CYRILLIC CAPITAL LETTER DWE + {0xA681, 0xA681, prLower}, // L& CYRILLIC SMALL LETTER DWE + {0xA682, 0xA682, prUpper}, // L& CYRILLIC CAPITAL LETTER DZWE + {0xA683, 0xA683, prLower}, // L& CYRILLIC SMALL LETTER DZWE + {0xA684, 0xA684, prUpper}, // L& CYRILLIC CAPITAL LETTER ZHWE + {0xA685, 0xA685, prLower}, // L& CYRILLIC SMALL LETTER ZHWE + {0xA686, 0xA686, prUpper}, // L& CYRILLIC CAPITAL LETTER CCHE + {0xA687, 0xA687, prLower}, // L& CYRILLIC SMALL LETTER CCHE + {0xA688, 0xA688, prUpper}, // L& CYRILLIC CAPITAL LETTER DZZE + {0xA689, 0xA689, prLower}, // L& CYRILLIC SMALL LETTER DZZE + {0xA68A, 0xA68A, prUpper}, // L& CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK + {0xA68B, 0xA68B, prLower}, // L& CYRILLIC SMALL LETTER TE WITH MIDDLE HOOK + {0xA68C, 0xA68C, prUpper}, // L& CYRILLIC CAPITAL LETTER TWE + {0xA68D, 0xA68D, prLower}, // L& CYRILLIC SMALL LETTER TWE + {0xA68E, 0xA68E, prUpper}, // L& CYRILLIC CAPITAL LETTER TSWE + {0xA68F, 0xA68F, prLower}, // L& CYRILLIC SMALL LETTER TSWE + {0xA690, 0xA690, prUpper}, // L& CYRILLIC CAPITAL LETTER TSSE + {0xA691, 0xA691, prLower}, // L& CYRILLIC SMALL LETTER TSSE + {0xA692, 0xA692, prUpper}, // L& CYRILLIC CAPITAL LETTER TCHE + {0xA693, 0xA693, prLower}, // L& CYRILLIC SMALL LETTER TCHE + {0xA694, 0xA694, prUpper}, // L& CYRILLIC CAPITAL LETTER HWE + {0xA695, 0xA695, prLower}, // L& CYRILLIC SMALL LETTER HWE + {0xA696, 0xA696, prUpper}, // L& CYRILLIC CAPITAL LETTER SHWE + {0xA697, 0xA697, prLower}, // L& CYRILLIC SMALL LETTER SHWE + {0xA698, 0xA698, prUpper}, // L& CYRILLIC CAPITAL LETTER DOUBLE O + {0xA699, 0xA699, prLower}, // L& CYRILLIC SMALL LETTER DOUBLE O + {0xA69A, 0xA69A, prUpper}, // L& CYRILLIC CAPITAL LETTER CROSSED O + {0xA69B, 0xA69B, prLower}, // L& CYRILLIC SMALL LETTER CROSSED O + {0xA69C, 0xA69D, prLower}, // Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN + {0xA69E, 0xA69F, prExtend}, // Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E + {0xA6A0, 0xA6E5, prOLetter}, // Lo [70] BAMUM LETTER A..BAMUM LETTER KI + {0xA6E6, 0xA6EF, prOLetter}, // Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM + {0xA6F0, 0xA6F1, prExtend}, // Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS + {0xA6F3, 0xA6F3, prSTerm}, // Po BAMUM FULL STOP + {0xA6F7, 0xA6F7, prSTerm}, // Po BAMUM QUESTION MARK + {0xA717, 0xA71F, prOLetter}, // Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK + {0xA722, 0xA722, prUpper}, // L& LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF + {0xA723, 0xA723, prLower}, // L& LATIN SMALL LETTER EGYPTOLOGICAL ALEF + {0xA724, 0xA724, prUpper}, // L& LATIN CAPITAL LETTER EGYPTOLOGICAL AIN + {0xA725, 0xA725, prLower}, // L& LATIN SMALL LETTER EGYPTOLOGICAL AIN + {0xA726, 0xA726, prUpper}, // L& LATIN CAPITAL LETTER HENG + {0xA727, 0xA727, prLower}, // L& LATIN SMALL LETTER HENG + {0xA728, 0xA728, prUpper}, // L& LATIN CAPITAL LETTER TZ + {0xA729, 0xA729, prLower}, // L& LATIN SMALL LETTER TZ + {0xA72A, 0xA72A, prUpper}, // L& LATIN CAPITAL LETTER TRESILLO + {0xA72B, 0xA72B, prLower}, // L& LATIN SMALL LETTER TRESILLO + {0xA72C, 0xA72C, prUpper}, // L& LATIN CAPITAL LETTER CUATRILLO + {0xA72D, 0xA72D, prLower}, // L& LATIN SMALL LETTER CUATRILLO + {0xA72E, 0xA72E, prUpper}, // L& LATIN CAPITAL LETTER CUATRILLO WITH COMMA + {0xA72F, 0xA731, prLower}, // L& [3] LATIN SMALL LETTER CUATRILLO WITH COMMA..LATIN LETTER SMALL CAPITAL S + {0xA732, 0xA732, prUpper}, // L& LATIN CAPITAL LETTER AA + {0xA733, 0xA733, prLower}, // L& LATIN SMALL LETTER AA + {0xA734, 0xA734, prUpper}, // L& LATIN CAPITAL LETTER AO + {0xA735, 0xA735, prLower}, // L& LATIN SMALL LETTER AO + {0xA736, 0xA736, prUpper}, // L& LATIN CAPITAL LETTER AU + {0xA737, 0xA737, prLower}, // L& LATIN SMALL LETTER AU + {0xA738, 0xA738, prUpper}, // L& LATIN CAPITAL LETTER AV + {0xA739, 0xA739, prLower}, // L& LATIN SMALL LETTER AV + {0xA73A, 0xA73A, prUpper}, // L& LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR + {0xA73B, 0xA73B, prLower}, // L& LATIN SMALL LETTER AV WITH HORIZONTAL BAR + {0xA73C, 0xA73C, prUpper}, // L& LATIN CAPITAL LETTER AY + {0xA73D, 0xA73D, prLower}, // L& LATIN SMALL LETTER AY + {0xA73E, 0xA73E, prUpper}, // L& LATIN CAPITAL LETTER REVERSED C WITH DOT + {0xA73F, 0xA73F, prLower}, // L& LATIN SMALL LETTER REVERSED C WITH DOT + {0xA740, 0xA740, prUpper}, // L& LATIN CAPITAL LETTER K WITH STROKE + {0xA741, 0xA741, prLower}, // L& LATIN SMALL LETTER K WITH STROKE + {0xA742, 0xA742, prUpper}, // L& LATIN CAPITAL LETTER K WITH DIAGONAL STROKE + {0xA743, 0xA743, prLower}, // L& LATIN SMALL LETTER K WITH DIAGONAL STROKE + {0xA744, 0xA744, prUpper}, // L& LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE + {0xA745, 0xA745, prLower}, // L& LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE + {0xA746, 0xA746, prUpper}, // L& LATIN CAPITAL LETTER BROKEN L + {0xA747, 0xA747, prLower}, // L& LATIN SMALL LETTER BROKEN L + {0xA748, 0xA748, prUpper}, // L& LATIN CAPITAL LETTER L WITH HIGH STROKE + {0xA749, 0xA749, prLower}, // L& LATIN SMALL LETTER L WITH HIGH STROKE + {0xA74A, 0xA74A, prUpper}, // L& LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY + {0xA74B, 0xA74B, prLower}, // L& LATIN SMALL LETTER O WITH LONG STROKE OVERLAY + {0xA74C, 0xA74C, prUpper}, // L& LATIN CAPITAL LETTER O WITH LOOP + {0xA74D, 0xA74D, prLower}, // L& LATIN SMALL LETTER O WITH LOOP + {0xA74E, 0xA74E, prUpper}, // L& LATIN CAPITAL LETTER OO + {0xA74F, 0xA74F, prLower}, // L& LATIN SMALL LETTER OO + {0xA750, 0xA750, prUpper}, // L& LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER + {0xA751, 0xA751, prLower}, // L& LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER + {0xA752, 0xA752, prUpper}, // L& LATIN CAPITAL LETTER P WITH FLOURISH + {0xA753, 0xA753, prLower}, // L& LATIN SMALL LETTER P WITH FLOURISH + {0xA754, 0xA754, prUpper}, // L& LATIN CAPITAL LETTER P WITH SQUIRREL TAIL + {0xA755, 0xA755, prLower}, // L& LATIN SMALL LETTER P WITH SQUIRREL TAIL + {0xA756, 0xA756, prUpper}, // L& LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER + {0xA757, 0xA757, prLower}, // L& LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER + {0xA758, 0xA758, prUpper}, // L& LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE + {0xA759, 0xA759, prLower}, // L& LATIN SMALL LETTER Q WITH DIAGONAL STROKE + {0xA75A, 0xA75A, prUpper}, // L& LATIN CAPITAL LETTER R ROTUNDA + {0xA75B, 0xA75B, prLower}, // L& LATIN SMALL LETTER R ROTUNDA + {0xA75C, 0xA75C, prUpper}, // L& LATIN CAPITAL LETTER RUM ROTUNDA + {0xA75D, 0xA75D, prLower}, // L& LATIN SMALL LETTER RUM ROTUNDA + {0xA75E, 0xA75E, prUpper}, // L& LATIN CAPITAL LETTER V WITH DIAGONAL STROKE + {0xA75F, 0xA75F, prLower}, // L& LATIN SMALL LETTER V WITH DIAGONAL STROKE + {0xA760, 0xA760, prUpper}, // L& LATIN CAPITAL LETTER VY + {0xA761, 0xA761, prLower}, // L& LATIN SMALL LETTER VY + {0xA762, 0xA762, prUpper}, // L& LATIN CAPITAL LETTER VISIGOTHIC Z + {0xA763, 0xA763, prLower}, // L& LATIN SMALL LETTER VISIGOTHIC Z + {0xA764, 0xA764, prUpper}, // L& LATIN CAPITAL LETTER THORN WITH STROKE + {0xA765, 0xA765, prLower}, // L& LATIN SMALL LETTER THORN WITH STROKE + {0xA766, 0xA766, prUpper}, // L& LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER + {0xA767, 0xA767, prLower}, // L& LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER + {0xA768, 0xA768, prUpper}, // L& LATIN CAPITAL LETTER VEND + {0xA769, 0xA769, prLower}, // L& LATIN SMALL LETTER VEND + {0xA76A, 0xA76A, prUpper}, // L& LATIN CAPITAL LETTER ET + {0xA76B, 0xA76B, prLower}, // L& LATIN SMALL LETTER ET + {0xA76C, 0xA76C, prUpper}, // L& LATIN CAPITAL LETTER IS + {0xA76D, 0xA76D, prLower}, // L& LATIN SMALL LETTER IS + {0xA76E, 0xA76E, prUpper}, // L& LATIN CAPITAL LETTER CON + {0xA76F, 0xA76F, prLower}, // L& LATIN SMALL LETTER CON + {0xA770, 0xA770, prLower}, // Lm MODIFIER LETTER US + {0xA771, 0xA778, prLower}, // L& [8] LATIN SMALL LETTER DUM..LATIN SMALL LETTER UM + {0xA779, 0xA779, prUpper}, // L& LATIN CAPITAL LETTER INSULAR D + {0xA77A, 0xA77A, prLower}, // L& LATIN SMALL LETTER INSULAR D + {0xA77B, 0xA77B, prUpper}, // L& LATIN CAPITAL LETTER INSULAR F + {0xA77C, 0xA77C, prLower}, // L& LATIN SMALL LETTER INSULAR F + {0xA77D, 0xA77E, prUpper}, // L& [2] LATIN CAPITAL LETTER INSULAR G..LATIN CAPITAL LETTER TURNED INSULAR G + {0xA77F, 0xA77F, prLower}, // L& LATIN SMALL LETTER TURNED INSULAR G + {0xA780, 0xA780, prUpper}, // L& LATIN CAPITAL LETTER TURNED L + {0xA781, 0xA781, prLower}, // L& LATIN SMALL LETTER TURNED L + {0xA782, 0xA782, prUpper}, // L& LATIN CAPITAL LETTER INSULAR R + {0xA783, 0xA783, prLower}, // L& LATIN SMALL LETTER INSULAR R + {0xA784, 0xA784, prUpper}, // L& LATIN CAPITAL LETTER INSULAR S + {0xA785, 0xA785, prLower}, // L& LATIN SMALL LETTER INSULAR S + {0xA786, 0xA786, prUpper}, // L& LATIN CAPITAL LETTER INSULAR T + {0xA787, 0xA787, prLower}, // L& LATIN SMALL LETTER INSULAR T + {0xA788, 0xA788, prOLetter}, // Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT + {0xA78B, 0xA78B, prUpper}, // L& LATIN CAPITAL LETTER SALTILLO + {0xA78C, 0xA78C, prLower}, // L& LATIN SMALL LETTER SALTILLO + {0xA78D, 0xA78D, prUpper}, // L& LATIN CAPITAL LETTER TURNED H + {0xA78E, 0xA78E, prLower}, // L& LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT + {0xA78F, 0xA78F, prOLetter}, // Lo LATIN LETTER SINOLOGICAL DOT + {0xA790, 0xA790, prUpper}, // L& LATIN CAPITAL LETTER N WITH DESCENDER + {0xA791, 0xA791, prLower}, // L& LATIN SMALL LETTER N WITH DESCENDER + {0xA792, 0xA792, prUpper}, // L& LATIN CAPITAL LETTER C WITH BAR + {0xA793, 0xA795, prLower}, // L& [3] LATIN SMALL LETTER C WITH BAR..LATIN SMALL LETTER H WITH PALATAL HOOK + {0xA796, 0xA796, prUpper}, // L& LATIN CAPITAL LETTER B WITH FLOURISH + {0xA797, 0xA797, prLower}, // L& LATIN SMALL LETTER B WITH FLOURISH + {0xA798, 0xA798, prUpper}, // L& LATIN CAPITAL LETTER F WITH STROKE + {0xA799, 0xA799, prLower}, // L& LATIN SMALL LETTER F WITH STROKE + {0xA79A, 0xA79A, prUpper}, // L& LATIN CAPITAL LETTER VOLAPUK AE + {0xA79B, 0xA79B, prLower}, // L& LATIN SMALL LETTER VOLAPUK AE + {0xA79C, 0xA79C, prUpper}, // L& LATIN CAPITAL LETTER VOLAPUK OE + {0xA79D, 0xA79D, prLower}, // L& LATIN SMALL LETTER VOLAPUK OE + {0xA79E, 0xA79E, prUpper}, // L& LATIN CAPITAL LETTER VOLAPUK UE + {0xA79F, 0xA79F, prLower}, // L& LATIN SMALL LETTER VOLAPUK UE + {0xA7A0, 0xA7A0, prUpper}, // L& LATIN CAPITAL LETTER G WITH OBLIQUE STROKE + {0xA7A1, 0xA7A1, prLower}, // L& LATIN SMALL LETTER G WITH OBLIQUE STROKE + {0xA7A2, 0xA7A2, prUpper}, // L& LATIN CAPITAL LETTER K WITH OBLIQUE STROKE + {0xA7A3, 0xA7A3, prLower}, // L& LATIN SMALL LETTER K WITH OBLIQUE STROKE + {0xA7A4, 0xA7A4, prUpper}, // L& LATIN CAPITAL LETTER N WITH OBLIQUE STROKE + {0xA7A5, 0xA7A5, prLower}, // L& LATIN SMALL LETTER N WITH OBLIQUE STROKE + {0xA7A6, 0xA7A6, prUpper}, // L& LATIN CAPITAL LETTER R WITH OBLIQUE STROKE + {0xA7A7, 0xA7A7, prLower}, // L& LATIN SMALL LETTER R WITH OBLIQUE STROKE + {0xA7A8, 0xA7A8, prUpper}, // L& LATIN CAPITAL LETTER S WITH OBLIQUE STROKE + {0xA7A9, 0xA7A9, prLower}, // L& LATIN SMALL LETTER S WITH OBLIQUE STROKE + {0xA7AA, 0xA7AE, prUpper}, // L& [5] LATIN CAPITAL LETTER H WITH HOOK..LATIN CAPITAL LETTER SMALL CAPITAL I + {0xA7AF, 0xA7AF, prLower}, // L& LATIN LETTER SMALL CAPITAL Q + {0xA7B0, 0xA7B4, prUpper}, // L& [5] LATIN CAPITAL LETTER TURNED K..LATIN CAPITAL LETTER BETA + {0xA7B5, 0xA7B5, prLower}, // L& LATIN SMALL LETTER BETA + {0xA7B6, 0xA7B6, prUpper}, // L& LATIN CAPITAL LETTER OMEGA + {0xA7B7, 0xA7B7, prLower}, // L& LATIN SMALL LETTER OMEGA + {0xA7B8, 0xA7B8, prUpper}, // L& LATIN CAPITAL LETTER U WITH STROKE + {0xA7B9, 0xA7B9, prLower}, // L& LATIN SMALL LETTER U WITH STROKE + {0xA7BA, 0xA7BA, prUpper}, // L& LATIN CAPITAL LETTER GLOTTAL A + {0xA7BB, 0xA7BB, prLower}, // L& LATIN SMALL LETTER GLOTTAL A + {0xA7BC, 0xA7BC, prUpper}, // L& LATIN CAPITAL LETTER GLOTTAL I + {0xA7BD, 0xA7BD, prLower}, // L& LATIN SMALL LETTER GLOTTAL I + {0xA7BE, 0xA7BE, prUpper}, // L& LATIN CAPITAL LETTER GLOTTAL U + {0xA7BF, 0xA7BF, prLower}, // L& LATIN SMALL LETTER GLOTTAL U + {0xA7C0, 0xA7C0, prUpper}, // L& LATIN CAPITAL LETTER OLD POLISH O + {0xA7C1, 0xA7C1, prLower}, // L& LATIN SMALL LETTER OLD POLISH O + {0xA7C2, 0xA7C2, prUpper}, // L& LATIN CAPITAL LETTER ANGLICANA W + {0xA7C3, 0xA7C3, prLower}, // L& LATIN SMALL LETTER ANGLICANA W + {0xA7C4, 0xA7C7, prUpper}, // L& [4] LATIN CAPITAL LETTER C WITH PALATAL HOOK..LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY + {0xA7C8, 0xA7C8, prLower}, // L& LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY + {0xA7C9, 0xA7C9, prUpper}, // L& LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY + {0xA7CA, 0xA7CA, prLower}, // L& LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY + {0xA7D0, 0xA7D0, prUpper}, // L& LATIN CAPITAL LETTER CLOSED INSULAR G + {0xA7D1, 0xA7D1, prLower}, // L& LATIN SMALL LETTER CLOSED INSULAR G + {0xA7D3, 0xA7D3, prLower}, // L& LATIN SMALL LETTER DOUBLE THORN + {0xA7D5, 0xA7D5, prLower}, // L& LATIN SMALL LETTER DOUBLE WYNN + {0xA7D6, 0xA7D6, prUpper}, // L& LATIN CAPITAL LETTER MIDDLE SCOTS S + {0xA7D7, 0xA7D7, prLower}, // L& LATIN SMALL LETTER MIDDLE SCOTS S + {0xA7D8, 0xA7D8, prUpper}, // L& LATIN CAPITAL LETTER SIGMOID S + {0xA7D9, 0xA7D9, prLower}, // L& LATIN SMALL LETTER SIGMOID S + {0xA7F2, 0xA7F4, prLower}, // Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q + {0xA7F5, 0xA7F5, prUpper}, // L& LATIN CAPITAL LETTER REVERSED HALF H + {0xA7F6, 0xA7F6, prLower}, // L& LATIN SMALL LETTER REVERSED HALF H + {0xA7F7, 0xA7F7, prOLetter}, // Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I + {0xA7F8, 0xA7F9, prLower}, // Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE + {0xA7FA, 0xA7FA, prLower}, // L& LATIN LETTER SMALL CAPITAL TURNED M + {0xA7FB, 0xA801, prOLetter}, // Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I + {0xA802, 0xA802, prExtend}, // Mn SYLOTI NAGRI SIGN DVISVARA + {0xA803, 0xA805, prOLetter}, // Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O + {0xA806, 0xA806, prExtend}, // Mn SYLOTI NAGRI SIGN HASANTA + {0xA807, 0xA80A, prOLetter}, // Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO + {0xA80B, 0xA80B, prExtend}, // Mn SYLOTI NAGRI SIGN ANUSVARA + {0xA80C, 0xA822, prOLetter}, // Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO + {0xA823, 0xA824, prExtend}, // Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I + {0xA825, 0xA826, prExtend}, // Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E + {0xA827, 0xA827, prExtend}, // Mc SYLOTI NAGRI VOWEL SIGN OO + {0xA82C, 0xA82C, prExtend}, // Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA + {0xA840, 0xA873, prOLetter}, // Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU + {0xA876, 0xA877, prSTerm}, // Po [2] PHAGS-PA MARK SHAD..PHAGS-PA MARK DOUBLE SHAD + {0xA880, 0xA881, prExtend}, // Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA + {0xA882, 0xA8B3, prOLetter}, // Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA + {0xA8B4, 0xA8C3, prExtend}, // Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU + {0xA8C4, 0xA8C5, prExtend}, // Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU + {0xA8CE, 0xA8CF, prSTerm}, // Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA + {0xA8D0, 0xA8D9, prNumeric}, // Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE + {0xA8E0, 0xA8F1, prExtend}, // Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA + {0xA8F2, 0xA8F7, prOLetter}, // Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA + {0xA8FB, 0xA8FB, prOLetter}, // Lo DEVANAGARI HEADSTROKE + {0xA8FD, 0xA8FE, prOLetter}, // Lo [2] DEVANAGARI JAIN OM..DEVANAGARI LETTER AY + {0xA8FF, 0xA8FF, prExtend}, // Mn DEVANAGARI VOWEL SIGN AY + {0xA900, 0xA909, prNumeric}, // Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE + {0xA90A, 0xA925, prOLetter}, // Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO + {0xA926, 0xA92D, prExtend}, // Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU + {0xA92F, 0xA92F, prSTerm}, // Po KAYAH LI SIGN SHYA + {0xA930, 0xA946, prOLetter}, // Lo [23] REJANG LETTER KA..REJANG LETTER A + {0xA947, 0xA951, prExtend}, // Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R + {0xA952, 0xA953, prExtend}, // Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA + {0xA960, 0xA97C, prOLetter}, // Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH + {0xA980, 0xA982, prExtend}, // Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR + {0xA983, 0xA983, prExtend}, // Mc JAVANESE SIGN WIGNYAN + {0xA984, 0xA9B2, prOLetter}, // Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA + {0xA9B3, 0xA9B3, prExtend}, // Mn JAVANESE SIGN CECAK TELU + {0xA9B4, 0xA9B5, prExtend}, // Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG + {0xA9B6, 0xA9B9, prExtend}, // Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT + {0xA9BA, 0xA9BB, prExtend}, // Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE + {0xA9BC, 0xA9BD, prExtend}, // Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET + {0xA9BE, 0xA9C0, prExtend}, // Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON + {0xA9C8, 0xA9C9, prSTerm}, // Po [2] JAVANESE PADA LINGSA..JAVANESE PADA LUNGSI + {0xA9CF, 0xA9CF, prOLetter}, // Lm JAVANESE PANGRANGKEP + {0xA9D0, 0xA9D9, prNumeric}, // Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE + {0xA9E0, 0xA9E4, prOLetter}, // Lo [5] MYANMAR LETTER SHAN GHA..MYANMAR LETTER SHAN BHA + {0xA9E5, 0xA9E5, prExtend}, // Mn MYANMAR SIGN SHAN SAW + {0xA9E6, 0xA9E6, prOLetter}, // Lm MYANMAR MODIFIER LETTER SHAN REDUPLICATION + {0xA9E7, 0xA9EF, prOLetter}, // Lo [9] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING NNA + {0xA9F0, 0xA9F9, prNumeric}, // Nd [10] MYANMAR TAI LAING DIGIT ZERO..MYANMAR TAI LAING DIGIT NINE + {0xA9FA, 0xA9FE, prOLetter}, // Lo [5] MYANMAR LETTER TAI LAING LLA..MYANMAR LETTER TAI LAING BHA + {0xAA00, 0xAA28, prOLetter}, // Lo [41] CHAM LETTER A..CHAM LETTER HA + {0xAA29, 0xAA2E, prExtend}, // Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE + {0xAA2F, 0xAA30, prExtend}, // Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI + {0xAA31, 0xAA32, prExtend}, // Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE + {0xAA33, 0xAA34, prExtend}, // Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA + {0xAA35, 0xAA36, prExtend}, // Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA + {0xAA40, 0xAA42, prOLetter}, // Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG + {0xAA43, 0xAA43, prExtend}, // Mn CHAM CONSONANT SIGN FINAL NG + {0xAA44, 0xAA4B, prOLetter}, // Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS + {0xAA4C, 0xAA4C, prExtend}, // Mn CHAM CONSONANT SIGN FINAL M + {0xAA4D, 0xAA4D, prExtend}, // Mc CHAM CONSONANT SIGN FINAL H + {0xAA50, 0xAA59, prNumeric}, // Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE + {0xAA5D, 0xAA5F, prSTerm}, // Po [3] CHAM PUNCTUATION DANDA..CHAM PUNCTUATION TRIPLE DANDA + {0xAA60, 0xAA6F, prOLetter}, // Lo [16] MYANMAR LETTER KHAMTI GA..MYANMAR LETTER KHAMTI FA + {0xAA70, 0xAA70, prOLetter}, // Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION + {0xAA71, 0xAA76, prOLetter}, // Lo [6] MYANMAR LETTER KHAMTI XA..MYANMAR LOGOGRAM KHAMTI HM + {0xAA7A, 0xAA7A, prOLetter}, // Lo MYANMAR LETTER AITON RA + {0xAA7B, 0xAA7B, prExtend}, // Mc MYANMAR SIGN PAO KAREN TONE + {0xAA7C, 0xAA7C, prExtend}, // Mn MYANMAR SIGN TAI LAING TONE-2 + {0xAA7D, 0xAA7D, prExtend}, // Mc MYANMAR SIGN TAI LAING TONE-5 + {0xAA7E, 0xAAAF, prOLetter}, // Lo [50] MYANMAR LETTER SHWE PALAUNG CHA..TAI VIET LETTER HIGH O + {0xAAB0, 0xAAB0, prExtend}, // Mn TAI VIET MAI KANG + {0xAAB1, 0xAAB1, prOLetter}, // Lo TAI VIET VOWEL AA + {0xAAB2, 0xAAB4, prExtend}, // Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U + {0xAAB5, 0xAAB6, prOLetter}, // Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O + {0xAAB7, 0xAAB8, prExtend}, // Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA + {0xAAB9, 0xAABD, prOLetter}, // Lo [5] TAI VIET VOWEL UEA..TAI VIET VOWEL AN + {0xAABE, 0xAABF, prExtend}, // Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK + {0xAAC0, 0xAAC0, prOLetter}, // Lo TAI VIET TONE MAI NUENG + {0xAAC1, 0xAAC1, prExtend}, // Mn TAI VIET TONE MAI THO + {0xAAC2, 0xAAC2, prOLetter}, // Lo TAI VIET TONE MAI SONG + {0xAADB, 0xAADC, prOLetter}, // Lo [2] TAI VIET SYMBOL KON..TAI VIET SYMBOL NUENG + {0xAADD, 0xAADD, prOLetter}, // Lm TAI VIET SYMBOL SAM + {0xAAE0, 0xAAEA, prOLetter}, // Lo [11] MEETEI MAYEK LETTER E..MEETEI MAYEK LETTER SSA + {0xAAEB, 0xAAEB, prExtend}, // Mc MEETEI MAYEK VOWEL SIGN II + {0xAAEC, 0xAAED, prExtend}, // Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI + {0xAAEE, 0xAAEF, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU + {0xAAF0, 0xAAF1, prSTerm}, // Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM + {0xAAF2, 0xAAF2, prOLetter}, // Lo MEETEI MAYEK ANJI + {0xAAF3, 0xAAF4, prOLetter}, // Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK + {0xAAF5, 0xAAF5, prExtend}, // Mc MEETEI MAYEK VOWEL SIGN VISARGA + {0xAAF6, 0xAAF6, prExtend}, // Mn MEETEI MAYEK VIRAMA + {0xAB01, 0xAB06, prOLetter}, // Lo [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO + {0xAB09, 0xAB0E, prOLetter}, // Lo [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO + {0xAB11, 0xAB16, prOLetter}, // Lo [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO + {0xAB20, 0xAB26, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO + {0xAB28, 0xAB2E, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO + {0xAB30, 0xAB5A, prLower}, // L& [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG + {0xAB5C, 0xAB5F, prLower}, // Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK + {0xAB60, 0xAB68, prLower}, // L& [9] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE + {0xAB69, 0xAB69, prLower}, // Lm MODIFIER LETTER SMALL TURNED W + {0xAB70, 0xABBF, prLower}, // L& [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA + {0xABC0, 0xABE2, prOLetter}, // Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM + {0xABE3, 0xABE4, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP + {0xABE5, 0xABE5, prExtend}, // Mn MEETEI MAYEK VOWEL SIGN ANAP + {0xABE6, 0xABE7, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP + {0xABE8, 0xABE8, prExtend}, // Mn MEETEI MAYEK VOWEL SIGN UNAP + {0xABE9, 0xABEA, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG + {0xABEB, 0xABEB, prSTerm}, // Po MEETEI MAYEK CHEIKHEI + {0xABEC, 0xABEC, prExtend}, // Mc MEETEI MAYEK LUM IYEK + {0xABED, 0xABED, prExtend}, // Mn MEETEI MAYEK APUN IYEK + {0xABF0, 0xABF9, prNumeric}, // Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE + {0xAC00, 0xD7A3, prOLetter}, // Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH + {0xD7B0, 0xD7C6, prOLetter}, // Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E + {0xD7CB, 0xD7FB, prOLetter}, // Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH + {0xF900, 0xFA6D, prOLetter}, // Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D + {0xFA70, 0xFAD9, prOLetter}, // Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 + {0xFB00, 0xFB06, prLower}, // L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST + {0xFB13, 0xFB17, prLower}, // L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH + {0xFB1D, 0xFB1D, prOLetter}, // Lo HEBREW LETTER YOD WITH HIRIQ + {0xFB1E, 0xFB1E, prExtend}, // Mn HEBREW POINT JUDEO-SPANISH VARIKA + {0xFB1F, 0xFB28, prOLetter}, // Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV + {0xFB2A, 0xFB36, prOLetter}, // Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH + {0xFB38, 0xFB3C, prOLetter}, // Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH + {0xFB3E, 0xFB3E, prOLetter}, // Lo HEBREW LETTER MEM WITH DAGESH + {0xFB40, 0xFB41, prOLetter}, // Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH + {0xFB43, 0xFB44, prOLetter}, // Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH + {0xFB46, 0xFBB1, prOLetter}, // Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM + {0xFBD3, 0xFD3D, prOLetter}, // Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM + {0xFD3E, 0xFD3E, prClose}, // Pe ORNATE LEFT PARENTHESIS + {0xFD3F, 0xFD3F, prClose}, // Ps ORNATE RIGHT PARENTHESIS + {0xFD50, 0xFD8F, prOLetter}, // Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM + {0xFD92, 0xFDC7, prOLetter}, // Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM + {0xFDF0, 0xFDFB, prOLetter}, // Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU + {0xFE00, 0xFE0F, prExtend}, // Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 + {0xFE10, 0xFE11, prSContinue}, // Po [2] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC COMMA + {0xFE13, 0xFE13, prSContinue}, // Po PRESENTATION FORM FOR VERTICAL COLON + {0xFE17, 0xFE17, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE LENTICULAR BRACKET + {0xFE18, 0xFE18, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRAKCET + {0xFE20, 0xFE2F, prExtend}, // Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + {0xFE31, 0xFE32, prSContinue}, // Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH + {0xFE35, 0xFE35, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS + {0xFE36, 0xFE36, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS + {0xFE37, 0xFE37, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET + {0xFE38, 0xFE38, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET + {0xFE39, 0xFE39, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET + {0xFE3A, 0xFE3A, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET + {0xFE3B, 0xFE3B, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET + {0xFE3C, 0xFE3C, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET + {0xFE3D, 0xFE3D, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET + {0xFE3E, 0xFE3E, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET + {0xFE3F, 0xFE3F, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET + {0xFE40, 0xFE40, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET + {0xFE41, 0xFE41, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET + {0xFE42, 0xFE42, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET + {0xFE43, 0xFE43, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET + {0xFE44, 0xFE44, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET + {0xFE47, 0xFE47, prClose}, // Ps PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET + {0xFE48, 0xFE48, prClose}, // Pe PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET + {0xFE50, 0xFE51, prSContinue}, // Po [2] SMALL COMMA..SMALL IDEOGRAPHIC COMMA + {0xFE52, 0xFE52, prATerm}, // Po SMALL FULL STOP + {0xFE55, 0xFE55, prSContinue}, // Po SMALL COLON + {0xFE56, 0xFE57, prSTerm}, // Po [2] SMALL QUESTION MARK..SMALL EXCLAMATION MARK + {0xFE58, 0xFE58, prSContinue}, // Pd SMALL EM DASH + {0xFE59, 0xFE59, prClose}, // Ps SMALL LEFT PARENTHESIS + {0xFE5A, 0xFE5A, prClose}, // Pe SMALL RIGHT PARENTHESIS + {0xFE5B, 0xFE5B, prClose}, // Ps SMALL LEFT CURLY BRACKET + {0xFE5C, 0xFE5C, prClose}, // Pe SMALL RIGHT CURLY BRACKET + {0xFE5D, 0xFE5D, prClose}, // Ps SMALL LEFT TORTOISE SHELL BRACKET + {0xFE5E, 0xFE5E, prClose}, // Pe SMALL RIGHT TORTOISE SHELL BRACKET + {0xFE63, 0xFE63, prSContinue}, // Pd SMALL HYPHEN-MINUS + {0xFE70, 0xFE74, prOLetter}, // Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM + {0xFE76, 0xFEFC, prOLetter}, // Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM + {0xFEFF, 0xFEFF, prFormat}, // Cf ZERO WIDTH NO-BREAK SPACE + {0xFF01, 0xFF01, prSTerm}, // Po FULLWIDTH EXCLAMATION MARK + {0xFF08, 0xFF08, prClose}, // Ps FULLWIDTH LEFT PARENTHESIS + {0xFF09, 0xFF09, prClose}, // Pe FULLWIDTH RIGHT PARENTHESIS + {0xFF0C, 0xFF0C, prSContinue}, // Po FULLWIDTH COMMA + {0xFF0D, 0xFF0D, prSContinue}, // Pd FULLWIDTH HYPHEN-MINUS + {0xFF0E, 0xFF0E, prATerm}, // Po FULLWIDTH FULL STOP + {0xFF10, 0xFF19, prNumeric}, // Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE + {0xFF1A, 0xFF1A, prSContinue}, // Po FULLWIDTH COLON + {0xFF1F, 0xFF1F, prSTerm}, // Po FULLWIDTH QUESTION MARK + {0xFF21, 0xFF3A, prUpper}, // L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z + {0xFF3B, 0xFF3B, prClose}, // Ps FULLWIDTH LEFT SQUARE BRACKET + {0xFF3D, 0xFF3D, prClose}, // Pe FULLWIDTH RIGHT SQUARE BRACKET + {0xFF41, 0xFF5A, prLower}, // L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z + {0xFF5B, 0xFF5B, prClose}, // Ps FULLWIDTH LEFT CURLY BRACKET + {0xFF5D, 0xFF5D, prClose}, // Pe FULLWIDTH RIGHT CURLY BRACKET + {0xFF5F, 0xFF5F, prClose}, // Ps FULLWIDTH LEFT WHITE PARENTHESIS + {0xFF60, 0xFF60, prClose}, // Pe FULLWIDTH RIGHT WHITE PARENTHESIS + {0xFF61, 0xFF61, prSTerm}, // Po HALFWIDTH IDEOGRAPHIC FULL STOP + {0xFF62, 0xFF62, prClose}, // Ps HALFWIDTH LEFT CORNER BRACKET + {0xFF63, 0xFF63, prClose}, // Pe HALFWIDTH RIGHT CORNER BRACKET + {0xFF64, 0xFF64, prSContinue}, // Po HALFWIDTH IDEOGRAPHIC COMMA + {0xFF66, 0xFF6F, prOLetter}, // Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU + {0xFF70, 0xFF70, prOLetter}, // Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK + {0xFF71, 0xFF9D, prOLetter}, // Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N + {0xFF9E, 0xFF9F, prExtend}, // Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK + {0xFFA0, 0xFFBE, prOLetter}, // Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH + {0xFFC2, 0xFFC7, prOLetter}, // Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E + {0xFFCA, 0xFFCF, prOLetter}, // Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE + {0xFFD2, 0xFFD7, prOLetter}, // Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU + {0xFFDA, 0xFFDC, prOLetter}, // Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I + {0xFFF9, 0xFFFB, prFormat}, // Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR + {0x10000, 0x1000B, prOLetter}, // Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE + {0x1000D, 0x10026, prOLetter}, // Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO + {0x10028, 0x1003A, prOLetter}, // Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO + {0x1003C, 0x1003D, prOLetter}, // Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE + {0x1003F, 0x1004D, prOLetter}, // Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO + {0x10050, 0x1005D, prOLetter}, // Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 + {0x10080, 0x100FA, prOLetter}, // Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 + {0x10140, 0x10174, prOLetter}, // Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS + {0x101FD, 0x101FD, prExtend}, // Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE + {0x10280, 0x1029C, prOLetter}, // Lo [29] LYCIAN LETTER A..LYCIAN LETTER X + {0x102A0, 0x102D0, prOLetter}, // Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 + {0x102E0, 0x102E0, prExtend}, // Mn COPTIC EPACT THOUSANDS MARK + {0x10300, 0x1031F, prOLetter}, // Lo [32] OLD ITALIC LETTER A..OLD ITALIC LETTER ESS + {0x1032D, 0x10340, prOLetter}, // Lo [20] OLD ITALIC LETTER YE..GOTHIC LETTER PAIRTHRA + {0x10341, 0x10341, prOLetter}, // Nl GOTHIC LETTER NINETY + {0x10342, 0x10349, prOLetter}, // Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL + {0x1034A, 0x1034A, prOLetter}, // Nl GOTHIC LETTER NINE HUNDRED + {0x10350, 0x10375, prOLetter}, // Lo [38] OLD PERMIC LETTER AN..OLD PERMIC LETTER IA + {0x10376, 0x1037A, prExtend}, // Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII + {0x10380, 0x1039D, prOLetter}, // Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU + {0x103A0, 0x103C3, prOLetter}, // Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA + {0x103C8, 0x103CF, prOLetter}, // Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH + {0x103D1, 0x103D5, prOLetter}, // Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED + {0x10400, 0x10427, prUpper}, // L& [40] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER EW + {0x10428, 0x1044F, prLower}, // L& [40] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER EW + {0x10450, 0x1049D, prOLetter}, // Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO + {0x104A0, 0x104A9, prNumeric}, // Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE + {0x104B0, 0x104D3, prUpper}, // L& [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA + {0x104D8, 0x104FB, prLower}, // L& [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA + {0x10500, 0x10527, prOLetter}, // Lo [40] ELBASAN LETTER A..ELBASAN LETTER KHE + {0x10530, 0x10563, prOLetter}, // Lo [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW + {0x10570, 0x1057A, prUpper}, // L& [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA + {0x1057C, 0x1058A, prUpper}, // L& [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE + {0x1058C, 0x10592, prUpper}, // L& [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE + {0x10594, 0x10595, prUpper}, // L& [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE + {0x10597, 0x105A1, prLower}, // L& [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA + {0x105A3, 0x105B1, prLower}, // L& [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE + {0x105B3, 0x105B9, prLower}, // L& [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE + {0x105BB, 0x105BC, prLower}, // L& [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE + {0x10600, 0x10736, prOLetter}, // Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 + {0x10740, 0x10755, prOLetter}, // Lo [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE + {0x10760, 0x10767, prOLetter}, // Lo [8] LINEAR A SIGN A800..LINEAR A SIGN A807 + {0x10780, 0x10780, prLower}, // Lm MODIFIER LETTER SMALL CAPITAL AA + {0x10781, 0x10782, prOLetter}, // Lm [2] MODIFIER LETTER SUPERSCRIPT TRIANGULAR COLON..MODIFIER LETTER SUPERSCRIPT HALF TRIANGULAR COLON + {0x10783, 0x10785, prLower}, // Lm [3] MODIFIER LETTER SMALL AE..MODIFIER LETTER SMALL B WITH HOOK + {0x10787, 0x107B0, prLower}, // Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK + {0x107B2, 0x107BA, prLower}, // Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL + {0x10800, 0x10805, prOLetter}, // Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA + {0x10808, 0x10808, prOLetter}, // Lo CYPRIOT SYLLABLE JO + {0x1080A, 0x10835, prOLetter}, // Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO + {0x10837, 0x10838, prOLetter}, // Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE + {0x1083C, 0x1083C, prOLetter}, // Lo CYPRIOT SYLLABLE ZA + {0x1083F, 0x10855, prOLetter}, // Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW + {0x10860, 0x10876, prOLetter}, // Lo [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW + {0x10880, 0x1089E, prOLetter}, // Lo [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW + {0x108E0, 0x108F2, prOLetter}, // Lo [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH + {0x108F4, 0x108F5, prOLetter}, // Lo [2] HATRAN LETTER SHIN..HATRAN LETTER TAW + {0x10900, 0x10915, prOLetter}, // Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU + {0x10920, 0x10939, prOLetter}, // Lo [26] LYDIAN LETTER A..LYDIAN LETTER C + {0x10980, 0x109B7, prOLetter}, // Lo [56] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC CURSIVE LETTER DA + {0x109BE, 0x109BF, prOLetter}, // Lo [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN + {0x10A00, 0x10A00, prOLetter}, // Lo KHAROSHTHI LETTER A + {0x10A01, 0x10A03, prExtend}, // Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R + {0x10A05, 0x10A06, prExtend}, // Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O + {0x10A0C, 0x10A0F, prExtend}, // Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA + {0x10A10, 0x10A13, prOLetter}, // Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA + {0x10A15, 0x10A17, prOLetter}, // Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA + {0x10A19, 0x10A35, prOLetter}, // Lo [29] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER VHA + {0x10A38, 0x10A3A, prExtend}, // Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW + {0x10A3F, 0x10A3F, prExtend}, // Mn KHAROSHTHI VIRAMA + {0x10A56, 0x10A57, prSTerm}, // Po [2] KHAROSHTHI PUNCTUATION DANDA..KHAROSHTHI PUNCTUATION DOUBLE DANDA + {0x10A60, 0x10A7C, prOLetter}, // Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH + {0x10A80, 0x10A9C, prOLetter}, // Lo [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH + {0x10AC0, 0x10AC7, prOLetter}, // Lo [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW + {0x10AC9, 0x10AE4, prOLetter}, // Lo [28] MANICHAEAN LETTER ZAYIN..MANICHAEAN LETTER TAW + {0x10AE5, 0x10AE6, prExtend}, // Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW + {0x10B00, 0x10B35, prOLetter}, // Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE + {0x10B40, 0x10B55, prOLetter}, // Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW + {0x10B60, 0x10B72, prOLetter}, // Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW + {0x10B80, 0x10B91, prOLetter}, // Lo [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW + {0x10C00, 0x10C48, prOLetter}, // Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH + {0x10C80, 0x10CB2, prUpper}, // L& [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US + {0x10CC0, 0x10CF2, prLower}, // L& [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US + {0x10D00, 0x10D23, prOLetter}, // Lo [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA + {0x10D24, 0x10D27, prExtend}, // Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI + {0x10D30, 0x10D39, prNumeric}, // Nd [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE + {0x10E80, 0x10EA9, prOLetter}, // Lo [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET + {0x10EAB, 0x10EAC, prExtend}, // Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK + {0x10EB0, 0x10EB1, prOLetter}, // Lo [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE + {0x10EFD, 0x10EFF, prExtend}, // Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA + {0x10F00, 0x10F1C, prOLetter}, // Lo [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL + {0x10F27, 0x10F27, prOLetter}, // Lo OLD SOGDIAN LIGATURE AYIN-DALETH + {0x10F30, 0x10F45, prOLetter}, // Lo [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN + {0x10F46, 0x10F50, prExtend}, // Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW + {0x10F55, 0x10F59, prSTerm}, // Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT + {0x10F70, 0x10F81, prOLetter}, // Lo [18] OLD UYGHUR LETTER ALEPH..OLD UYGHUR LETTER LESH + {0x10F82, 0x10F85, prExtend}, // Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW + {0x10F86, 0x10F89, prSTerm}, // Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS + {0x10FB0, 0x10FC4, prOLetter}, // Lo [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW + {0x10FE0, 0x10FF6, prOLetter}, // Lo [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH + {0x11000, 0x11000, prExtend}, // Mc BRAHMI SIGN CANDRABINDU + {0x11001, 0x11001, prExtend}, // Mn BRAHMI SIGN ANUSVARA + {0x11002, 0x11002, prExtend}, // Mc BRAHMI SIGN VISARGA + {0x11003, 0x11037, prOLetter}, // Lo [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI LETTER OLD TAMIL NNNA + {0x11038, 0x11046, prExtend}, // Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA + {0x11047, 0x11048, prSTerm}, // Po [2] BRAHMI DANDA..BRAHMI DOUBLE DANDA + {0x11066, 0x1106F, prNumeric}, // Nd [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE + {0x11070, 0x11070, prExtend}, // Mn BRAHMI SIGN OLD TAMIL VIRAMA + {0x11071, 0x11072, prOLetter}, // Lo [2] BRAHMI LETTER OLD TAMIL SHORT E..BRAHMI LETTER OLD TAMIL SHORT O + {0x11073, 0x11074, prExtend}, // Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O + {0x11075, 0x11075, prOLetter}, // Lo BRAHMI LETTER OLD TAMIL LLA + {0x1107F, 0x11081, prExtend}, // Mn [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA + {0x11082, 0x11082, prExtend}, // Mc KAITHI SIGN VISARGA + {0x11083, 0x110AF, prOLetter}, // Lo [45] KAITHI LETTER A..KAITHI LETTER HA + {0x110B0, 0x110B2, prExtend}, // Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II + {0x110B3, 0x110B6, prExtend}, // Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI + {0x110B7, 0x110B8, prExtend}, // Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU + {0x110B9, 0x110BA, prExtend}, // Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA + {0x110BD, 0x110BD, prFormat}, // Cf KAITHI NUMBER SIGN + {0x110BE, 0x110C1, prSTerm}, // Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA + {0x110C2, 0x110C2, prExtend}, // Mn KAITHI VOWEL SIGN VOCALIC R + {0x110CD, 0x110CD, prFormat}, // Cf KAITHI NUMBER SIGN ABOVE + {0x110D0, 0x110E8, prOLetter}, // Lo [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE + {0x110F0, 0x110F9, prNumeric}, // Nd [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE + {0x11100, 0x11102, prExtend}, // Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA + {0x11103, 0x11126, prOLetter}, // Lo [36] CHAKMA LETTER AA..CHAKMA LETTER HAA + {0x11127, 0x1112B, prExtend}, // Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU + {0x1112C, 0x1112C, prExtend}, // Mc CHAKMA VOWEL SIGN E + {0x1112D, 0x11134, prExtend}, // Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA + {0x11136, 0x1113F, prNumeric}, // Nd [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE + {0x11141, 0x11143, prSTerm}, // Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK + {0x11144, 0x11144, prOLetter}, // Lo CHAKMA LETTER LHAA + {0x11145, 0x11146, prExtend}, // Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI + {0x11147, 0x11147, prOLetter}, // Lo CHAKMA LETTER VAA + {0x11150, 0x11172, prOLetter}, // Lo [35] MAHAJANI LETTER A..MAHAJANI LETTER RRA + {0x11173, 0x11173, prExtend}, // Mn MAHAJANI SIGN NUKTA + {0x11176, 0x11176, prOLetter}, // Lo MAHAJANI LIGATURE SHRI + {0x11180, 0x11181, prExtend}, // Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA + {0x11182, 0x11182, prExtend}, // Mc SHARADA SIGN VISARGA + {0x11183, 0x111B2, prOLetter}, // Lo [48] SHARADA LETTER A..SHARADA LETTER HA + {0x111B3, 0x111B5, prExtend}, // Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II + {0x111B6, 0x111BE, prExtend}, // Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O + {0x111BF, 0x111C0, prExtend}, // Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA + {0x111C1, 0x111C4, prOLetter}, // Lo [4] SHARADA SIGN AVAGRAHA..SHARADA OM + {0x111C5, 0x111C6, prSTerm}, // Po [2] SHARADA DANDA..SHARADA DOUBLE DANDA + {0x111C9, 0x111CC, prExtend}, // Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK + {0x111CD, 0x111CD, prSTerm}, // Po SHARADA SUTRA MARK + {0x111CE, 0x111CE, prExtend}, // Mc SHARADA VOWEL SIGN PRISHTHAMATRA E + {0x111CF, 0x111CF, prExtend}, // Mn SHARADA SIGN INVERTED CANDRABINDU + {0x111D0, 0x111D9, prNumeric}, // Nd [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE + {0x111DA, 0x111DA, prOLetter}, // Lo SHARADA EKAM + {0x111DC, 0x111DC, prOLetter}, // Lo SHARADA HEADSTROKE + {0x111DE, 0x111DF, prSTerm}, // Po [2] SHARADA SECTION MARK-1..SHARADA SECTION MARK-2 + {0x11200, 0x11211, prOLetter}, // Lo [18] KHOJKI LETTER A..KHOJKI LETTER JJA + {0x11213, 0x1122B, prOLetter}, // Lo [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA + {0x1122C, 0x1122E, prExtend}, // Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II + {0x1122F, 0x11231, prExtend}, // Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI + {0x11232, 0x11233, prExtend}, // Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU + {0x11234, 0x11234, prExtend}, // Mn KHOJKI SIGN ANUSVARA + {0x11235, 0x11235, prExtend}, // Mc KHOJKI SIGN VIRAMA + {0x11236, 0x11237, prExtend}, // Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA + {0x11238, 0x11239, prSTerm}, // Po [2] KHOJKI DANDA..KHOJKI DOUBLE DANDA + {0x1123B, 0x1123C, prSTerm}, // Po [2] KHOJKI SECTION MARK..KHOJKI DOUBLE SECTION MARK + {0x1123E, 0x1123E, prExtend}, // Mn KHOJKI SIGN SUKUN + {0x1123F, 0x11240, prOLetter}, // Lo [2] KHOJKI LETTER QA..KHOJKI LETTER SHORT I + {0x11241, 0x11241, prExtend}, // Mn KHOJKI VOWEL SIGN VOCALIC R + {0x11280, 0x11286, prOLetter}, // Lo [7] MULTANI LETTER A..MULTANI LETTER GA + {0x11288, 0x11288, prOLetter}, // Lo MULTANI LETTER GHA + {0x1128A, 0x1128D, prOLetter}, // Lo [4] MULTANI LETTER CA..MULTANI LETTER JJA + {0x1128F, 0x1129D, prOLetter}, // Lo [15] MULTANI LETTER NYA..MULTANI LETTER BA + {0x1129F, 0x112A8, prOLetter}, // Lo [10] MULTANI LETTER BHA..MULTANI LETTER RHA + {0x112A9, 0x112A9, prSTerm}, // Po MULTANI SECTION MARK + {0x112B0, 0x112DE, prOLetter}, // Lo [47] KHUDAWADI LETTER A..KHUDAWADI LETTER HA + {0x112DF, 0x112DF, prExtend}, // Mn KHUDAWADI SIGN ANUSVARA + {0x112E0, 0x112E2, prExtend}, // Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II + {0x112E3, 0x112EA, prExtend}, // Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA + {0x112F0, 0x112F9, prNumeric}, // Nd [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE + {0x11300, 0x11301, prExtend}, // Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU + {0x11302, 0x11303, prExtend}, // Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA + {0x11305, 0x1130C, prOLetter}, // Lo [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L + {0x1130F, 0x11310, prOLetter}, // Lo [2] GRANTHA LETTER EE..GRANTHA LETTER AI + {0x11313, 0x11328, prOLetter}, // Lo [22] GRANTHA LETTER OO..GRANTHA LETTER NA + {0x1132A, 0x11330, prOLetter}, // Lo [7] GRANTHA LETTER PA..GRANTHA LETTER RA + {0x11332, 0x11333, prOLetter}, // Lo [2] GRANTHA LETTER LA..GRANTHA LETTER LLA + {0x11335, 0x11339, prOLetter}, // Lo [5] GRANTHA LETTER VA..GRANTHA LETTER HA + {0x1133B, 0x1133C, prExtend}, // Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA + {0x1133D, 0x1133D, prOLetter}, // Lo GRANTHA SIGN AVAGRAHA + {0x1133E, 0x1133F, prExtend}, // Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I + {0x11340, 0x11340, prExtend}, // Mn GRANTHA VOWEL SIGN II + {0x11341, 0x11344, prExtend}, // Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR + {0x11347, 0x11348, prExtend}, // Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI + {0x1134B, 0x1134D, prExtend}, // Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA + {0x11350, 0x11350, prOLetter}, // Lo GRANTHA OM + {0x11357, 0x11357, prExtend}, // Mc GRANTHA AU LENGTH MARK + {0x1135D, 0x11361, prOLetter}, // Lo [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL + {0x11362, 0x11363, prExtend}, // Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL + {0x11366, 0x1136C, prExtend}, // Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX + {0x11370, 0x11374, prExtend}, // Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA + {0x11400, 0x11434, prOLetter}, // Lo [53] NEWA LETTER A..NEWA LETTER HA + {0x11435, 0x11437, prExtend}, // Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II + {0x11438, 0x1143F, prExtend}, // Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI + {0x11440, 0x11441, prExtend}, // Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU + {0x11442, 0x11444, prExtend}, // Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA + {0x11445, 0x11445, prExtend}, // Mc NEWA SIGN VISARGA + {0x11446, 0x11446, prExtend}, // Mn NEWA SIGN NUKTA + {0x11447, 0x1144A, prOLetter}, // Lo [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI + {0x1144B, 0x1144C, prSTerm}, // Po [2] NEWA DANDA..NEWA DOUBLE DANDA + {0x11450, 0x11459, prNumeric}, // Nd [10] NEWA DIGIT ZERO..NEWA DIGIT NINE + {0x1145E, 0x1145E, prExtend}, // Mn NEWA SANDHI MARK + {0x1145F, 0x11461, prOLetter}, // Lo [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA + {0x11480, 0x114AF, prOLetter}, // Lo [48] TIRHUTA ANJI..TIRHUTA LETTER HA + {0x114B0, 0x114B2, prExtend}, // Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II + {0x114B3, 0x114B8, prExtend}, // Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL + {0x114B9, 0x114B9, prExtend}, // Mc TIRHUTA VOWEL SIGN E + {0x114BA, 0x114BA, prExtend}, // Mn TIRHUTA VOWEL SIGN SHORT E + {0x114BB, 0x114BE, prExtend}, // Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU + {0x114BF, 0x114C0, prExtend}, // Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA + {0x114C1, 0x114C1, prExtend}, // Mc TIRHUTA SIGN VISARGA + {0x114C2, 0x114C3, prExtend}, // Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA + {0x114C4, 0x114C5, prOLetter}, // Lo [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA GVANG + {0x114C7, 0x114C7, prOLetter}, // Lo TIRHUTA OM + {0x114D0, 0x114D9, prNumeric}, // Nd [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE + {0x11580, 0x115AE, prOLetter}, // Lo [47] SIDDHAM LETTER A..SIDDHAM LETTER HA + {0x115AF, 0x115B1, prExtend}, // Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II + {0x115B2, 0x115B5, prExtend}, // Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR + {0x115B8, 0x115BB, prExtend}, // Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU + {0x115BC, 0x115BD, prExtend}, // Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA + {0x115BE, 0x115BE, prExtend}, // Mc SIDDHAM SIGN VISARGA + {0x115BF, 0x115C0, prExtend}, // Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA + {0x115C2, 0x115C3, prSTerm}, // Po [2] SIDDHAM DANDA..SIDDHAM DOUBLE DANDA + {0x115C9, 0x115D7, prSTerm}, // Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES + {0x115D8, 0x115DB, prOLetter}, // Lo [4] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM LETTER ALTERNATE U + {0x115DC, 0x115DD, prExtend}, // Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU + {0x11600, 0x1162F, prOLetter}, // Lo [48] MODI LETTER A..MODI LETTER LLA + {0x11630, 0x11632, prExtend}, // Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II + {0x11633, 0x1163A, prExtend}, // Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI + {0x1163B, 0x1163C, prExtend}, // Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU + {0x1163D, 0x1163D, prExtend}, // Mn MODI SIGN ANUSVARA + {0x1163E, 0x1163E, prExtend}, // Mc MODI SIGN VISARGA + {0x1163F, 0x11640, prExtend}, // Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA + {0x11641, 0x11642, prSTerm}, // Po [2] MODI DANDA..MODI DOUBLE DANDA + {0x11644, 0x11644, prOLetter}, // Lo MODI SIGN HUVA + {0x11650, 0x11659, prNumeric}, // Nd [10] MODI DIGIT ZERO..MODI DIGIT NINE + {0x11680, 0x116AA, prOLetter}, // Lo [43] TAKRI LETTER A..TAKRI LETTER RRA + {0x116AB, 0x116AB, prExtend}, // Mn TAKRI SIGN ANUSVARA + {0x116AC, 0x116AC, prExtend}, // Mc TAKRI SIGN VISARGA + {0x116AD, 0x116AD, prExtend}, // Mn TAKRI VOWEL SIGN AA + {0x116AE, 0x116AF, prExtend}, // Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II + {0x116B0, 0x116B5, prExtend}, // Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU + {0x116B6, 0x116B6, prExtend}, // Mc TAKRI SIGN VIRAMA + {0x116B7, 0x116B7, prExtend}, // Mn TAKRI SIGN NUKTA + {0x116B8, 0x116B8, prOLetter}, // Lo TAKRI LETTER ARCHAIC KHA + {0x116C0, 0x116C9, prNumeric}, // Nd [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE + {0x11700, 0x1171A, prOLetter}, // Lo [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA + {0x1171D, 0x1171F, prExtend}, // Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA + {0x11720, 0x11721, prExtend}, // Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA + {0x11722, 0x11725, prExtend}, // Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU + {0x11726, 0x11726, prExtend}, // Mc AHOM VOWEL SIGN E + {0x11727, 0x1172B, prExtend}, // Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER + {0x11730, 0x11739, prNumeric}, // Nd [10] AHOM DIGIT ZERO..AHOM DIGIT NINE + {0x1173C, 0x1173E, prSTerm}, // Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI + {0x11740, 0x11746, prOLetter}, // Lo [7] AHOM LETTER CA..AHOM LETTER LLA + {0x11800, 0x1182B, prOLetter}, // Lo [44] DOGRA LETTER A..DOGRA LETTER RRA + {0x1182C, 0x1182E, prExtend}, // Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II + {0x1182F, 0x11837, prExtend}, // Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA + {0x11838, 0x11838, prExtend}, // Mc DOGRA SIGN VISARGA + {0x11839, 0x1183A, prExtend}, // Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA + {0x118A0, 0x118BF, prUpper}, // L& [32] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI CAPITAL LETTER VIYO + {0x118C0, 0x118DF, prLower}, // L& [32] WARANG CITI SMALL LETTER NGAA..WARANG CITI SMALL LETTER VIYO + {0x118E0, 0x118E9, prNumeric}, // Nd [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE + {0x118FF, 0x11906, prOLetter}, // Lo [8] WARANG CITI OM..DIVES AKURU LETTER E + {0x11909, 0x11909, prOLetter}, // Lo DIVES AKURU LETTER O + {0x1190C, 0x11913, prOLetter}, // Lo [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA + {0x11915, 0x11916, prOLetter}, // Lo [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA + {0x11918, 0x1192F, prOLetter}, // Lo [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA + {0x11930, 0x11935, prExtend}, // Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E + {0x11937, 0x11938, prExtend}, // Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O + {0x1193B, 0x1193C, prExtend}, // Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU + {0x1193D, 0x1193D, prExtend}, // Mc DIVES AKURU SIGN HALANTA + {0x1193E, 0x1193E, prExtend}, // Mn DIVES AKURU VIRAMA + {0x1193F, 0x1193F, prOLetter}, // Lo DIVES AKURU PREFIXED NASAL SIGN + {0x11940, 0x11940, prExtend}, // Mc DIVES AKURU MEDIAL YA + {0x11941, 0x11941, prOLetter}, // Lo DIVES AKURU INITIAL RA + {0x11942, 0x11942, prExtend}, // Mc DIVES AKURU MEDIAL RA + {0x11943, 0x11943, prExtend}, // Mn DIVES AKURU SIGN NUKTA + {0x11944, 0x11944, prSTerm}, // Po DIVES AKURU DOUBLE DANDA + {0x11946, 0x11946, prSTerm}, // Po DIVES AKURU END OF TEXT MARK + {0x11950, 0x11959, prNumeric}, // Nd [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE + {0x119A0, 0x119A7, prOLetter}, // Lo [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR + {0x119AA, 0x119D0, prOLetter}, // Lo [39] NANDINAGARI LETTER E..NANDINAGARI LETTER RRA + {0x119D1, 0x119D3, prExtend}, // Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II + {0x119D4, 0x119D7, prExtend}, // Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR + {0x119DA, 0x119DB, prExtend}, // Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI + {0x119DC, 0x119DF, prExtend}, // Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA + {0x119E0, 0x119E0, prExtend}, // Mn NANDINAGARI SIGN VIRAMA + {0x119E1, 0x119E1, prOLetter}, // Lo NANDINAGARI SIGN AVAGRAHA + {0x119E3, 0x119E3, prOLetter}, // Lo NANDINAGARI HEADSTROKE + {0x119E4, 0x119E4, prExtend}, // Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E + {0x11A00, 0x11A00, prOLetter}, // Lo ZANABAZAR SQUARE LETTER A + {0x11A01, 0x11A0A, prExtend}, // Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK + {0x11A0B, 0x11A32, prOLetter}, // Lo [40] ZANABAZAR SQUARE LETTER KA..ZANABAZAR SQUARE LETTER KSSA + {0x11A33, 0x11A38, prExtend}, // Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA + {0x11A39, 0x11A39, prExtend}, // Mc ZANABAZAR SQUARE SIGN VISARGA + {0x11A3A, 0x11A3A, prOLetter}, // Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA + {0x11A3B, 0x11A3E, prExtend}, // Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA + {0x11A42, 0x11A43, prSTerm}, // Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD + {0x11A47, 0x11A47, prExtend}, // Mn ZANABAZAR SQUARE SUBJOINER + {0x11A50, 0x11A50, prOLetter}, // Lo SOYOMBO LETTER A + {0x11A51, 0x11A56, prExtend}, // Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE + {0x11A57, 0x11A58, prExtend}, // Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU + {0x11A59, 0x11A5B, prExtend}, // Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK + {0x11A5C, 0x11A89, prOLetter}, // Lo [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA + {0x11A8A, 0x11A96, prExtend}, // Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA + {0x11A97, 0x11A97, prExtend}, // Mc SOYOMBO SIGN VISARGA + {0x11A98, 0x11A99, prExtend}, // Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER + {0x11A9B, 0x11A9C, prSTerm}, // Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD + {0x11A9D, 0x11A9D, prOLetter}, // Lo SOYOMBO MARK PLUTA + {0x11AB0, 0x11AF8, prOLetter}, // Lo [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL + {0x11C00, 0x11C08, prOLetter}, // Lo [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L + {0x11C0A, 0x11C2E, prOLetter}, // Lo [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA + {0x11C2F, 0x11C2F, prExtend}, // Mc BHAIKSUKI VOWEL SIGN AA + {0x11C30, 0x11C36, prExtend}, // Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L + {0x11C38, 0x11C3D, prExtend}, // Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA + {0x11C3E, 0x11C3E, prExtend}, // Mc BHAIKSUKI SIGN VISARGA + {0x11C3F, 0x11C3F, prExtend}, // Mn BHAIKSUKI SIGN VIRAMA + {0x11C40, 0x11C40, prOLetter}, // Lo BHAIKSUKI SIGN AVAGRAHA + {0x11C41, 0x11C42, prSTerm}, // Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA + {0x11C50, 0x11C59, prNumeric}, // Nd [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE + {0x11C72, 0x11C8F, prOLetter}, // Lo [30] MARCHEN LETTER KA..MARCHEN LETTER A + {0x11C92, 0x11CA7, prExtend}, // Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA + {0x11CA9, 0x11CA9, prExtend}, // Mc MARCHEN SUBJOINED LETTER YA + {0x11CAA, 0x11CB0, prExtend}, // Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA + {0x11CB1, 0x11CB1, prExtend}, // Mc MARCHEN VOWEL SIGN I + {0x11CB2, 0x11CB3, prExtend}, // Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E + {0x11CB4, 0x11CB4, prExtend}, // Mc MARCHEN VOWEL SIGN O + {0x11CB5, 0x11CB6, prExtend}, // Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU + {0x11D00, 0x11D06, prOLetter}, // Lo [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E + {0x11D08, 0x11D09, prOLetter}, // Lo [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O + {0x11D0B, 0x11D30, prOLetter}, // Lo [38] MASARAM GONDI LETTER AU..MASARAM GONDI LETTER TRA + {0x11D31, 0x11D36, prExtend}, // Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R + {0x11D3A, 0x11D3A, prExtend}, // Mn MASARAM GONDI VOWEL SIGN E + {0x11D3C, 0x11D3D, prExtend}, // Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O + {0x11D3F, 0x11D45, prExtend}, // Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA + {0x11D46, 0x11D46, prOLetter}, // Lo MASARAM GONDI REPHA + {0x11D47, 0x11D47, prExtend}, // Mn MASARAM GONDI RA-KARA + {0x11D50, 0x11D59, prNumeric}, // Nd [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE + {0x11D60, 0x11D65, prOLetter}, // Lo [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU + {0x11D67, 0x11D68, prOLetter}, // Lo [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI + {0x11D6A, 0x11D89, prOLetter}, // Lo [32] GUNJALA GONDI LETTER OO..GUNJALA GONDI LETTER SA + {0x11D8A, 0x11D8E, prExtend}, // Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU + {0x11D90, 0x11D91, prExtend}, // Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI + {0x11D93, 0x11D94, prExtend}, // Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU + {0x11D95, 0x11D95, prExtend}, // Mn GUNJALA GONDI SIGN ANUSVARA + {0x11D96, 0x11D96, prExtend}, // Mc GUNJALA GONDI SIGN VISARGA + {0x11D97, 0x11D97, prExtend}, // Mn GUNJALA GONDI VIRAMA + {0x11D98, 0x11D98, prOLetter}, // Lo GUNJALA GONDI OM + {0x11DA0, 0x11DA9, prNumeric}, // Nd [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE + {0x11EE0, 0x11EF2, prOLetter}, // Lo [19] MAKASAR LETTER KA..MAKASAR ANGKA + {0x11EF3, 0x11EF4, prExtend}, // Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U + {0x11EF5, 0x11EF6, prExtend}, // Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O + {0x11EF7, 0x11EF8, prSTerm}, // Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION + {0x11F00, 0x11F01, prExtend}, // Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA + {0x11F02, 0x11F02, prOLetter}, // Lo KAWI SIGN REPHA + {0x11F03, 0x11F03, prExtend}, // Mc KAWI SIGN VISARGA + {0x11F04, 0x11F10, prOLetter}, // Lo [13] KAWI LETTER A..KAWI LETTER O + {0x11F12, 0x11F33, prOLetter}, // Lo [34] KAWI LETTER KA..KAWI LETTER JNYA + {0x11F34, 0x11F35, prExtend}, // Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA + {0x11F36, 0x11F3A, prExtend}, // Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R + {0x11F3E, 0x11F3F, prExtend}, // Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI + {0x11F40, 0x11F40, prExtend}, // Mn KAWI VOWEL SIGN EU + {0x11F41, 0x11F41, prExtend}, // Mc KAWI SIGN KILLER + {0x11F42, 0x11F42, prExtend}, // Mn KAWI CONJOINER + {0x11F43, 0x11F44, prSTerm}, // Po [2] KAWI DANDA..KAWI DOUBLE DANDA + {0x11F50, 0x11F59, prNumeric}, // Nd [10] KAWI DIGIT ZERO..KAWI DIGIT NINE + {0x11FB0, 0x11FB0, prOLetter}, // Lo LISU LETTER YHA + {0x12000, 0x12399, prOLetter}, // Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U + {0x12400, 0x1246E, prOLetter}, // Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM + {0x12480, 0x12543, prOLetter}, // Lo [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU + {0x12F90, 0x12FF0, prOLetter}, // Lo [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 + {0x13000, 0x1342F, prOLetter}, // Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D + {0x13430, 0x1343F, prFormat}, // Cf [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE + {0x13440, 0x13440, prExtend}, // Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY + {0x13441, 0x13446, prOLetter}, // Lo [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN + {0x13447, 0x13455, prExtend}, // Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED + {0x14400, 0x14646, prOLetter}, // Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 + {0x16800, 0x16A38, prOLetter}, // Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ + {0x16A40, 0x16A5E, prOLetter}, // Lo [31] MRO LETTER TA..MRO LETTER TEK + {0x16A60, 0x16A69, prNumeric}, // Nd [10] MRO DIGIT ZERO..MRO DIGIT NINE + {0x16A6E, 0x16A6F, prSTerm}, // Po [2] MRO DANDA..MRO DOUBLE DANDA + {0x16A70, 0x16ABE, prOLetter}, // Lo [79] TANGSA LETTER OZ..TANGSA LETTER ZA + {0x16AC0, 0x16AC9, prNumeric}, // Nd [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE + {0x16AD0, 0x16AED, prOLetter}, // Lo [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I + {0x16AF0, 0x16AF4, prExtend}, // Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE + {0x16AF5, 0x16AF5, prSTerm}, // Po BASSA VAH FULL STOP + {0x16B00, 0x16B2F, prOLetter}, // Lo [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU + {0x16B30, 0x16B36, prExtend}, // Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM + {0x16B37, 0x16B38, prSTerm}, // Po [2] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS TSHAB CEEB + {0x16B40, 0x16B43, prOLetter}, // Lm [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM + {0x16B44, 0x16B44, prSTerm}, // Po PAHAWH HMONG SIGN XAUS + {0x16B50, 0x16B59, prNumeric}, // Nd [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE + {0x16B63, 0x16B77, prOLetter}, // Lo [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS + {0x16B7D, 0x16B8F, prOLetter}, // Lo [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ + {0x16E40, 0x16E5F, prUpper}, // L& [32] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN CAPITAL LETTER Y + {0x16E60, 0x16E7F, prLower}, // L& [32] MEDEFAIDRIN SMALL LETTER M..MEDEFAIDRIN SMALL LETTER Y + {0x16E98, 0x16E98, prSTerm}, // Po MEDEFAIDRIN FULL STOP + {0x16F00, 0x16F4A, prOLetter}, // Lo [75] MIAO LETTER PA..MIAO LETTER RTE + {0x16F4F, 0x16F4F, prExtend}, // Mn MIAO SIGN CONSONANT MODIFIER BAR + {0x16F50, 0x16F50, prOLetter}, // Lo MIAO LETTER NASALIZATION + {0x16F51, 0x16F87, prExtend}, // Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI + {0x16F8F, 0x16F92, prExtend}, // Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW + {0x16F93, 0x16F9F, prOLetter}, // Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 + {0x16FE0, 0x16FE1, prOLetter}, // Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK + {0x16FE3, 0x16FE3, prOLetter}, // Lm OLD CHINESE ITERATION MARK + {0x16FE4, 0x16FE4, prExtend}, // Mn KHITAN SMALL SCRIPT FILLER + {0x16FF0, 0x16FF1, prExtend}, // Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY + {0x17000, 0x187F7, prOLetter}, // Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 + {0x18800, 0x18CD5, prOLetter}, // Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5 + {0x18D00, 0x18D08, prOLetter}, // Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 + {0x1AFF0, 0x1AFF3, prOLetter}, // Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 + {0x1AFF5, 0x1AFFB, prOLetter}, // Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 + {0x1AFFD, 0x1AFFE, prOLetter}, // Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 + {0x1B000, 0x1B122, prOLetter}, // Lo [291] KATAKANA LETTER ARCHAIC E..KATAKANA LETTER ARCHAIC WU + {0x1B132, 0x1B132, prOLetter}, // Lo HIRAGANA LETTER SMALL KO + {0x1B150, 0x1B152, prOLetter}, // Lo [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO + {0x1B155, 0x1B155, prOLetter}, // Lo KATAKANA LETTER SMALL KO + {0x1B164, 0x1B167, prOLetter}, // Lo [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N + {0x1B170, 0x1B2FB, prOLetter}, // Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB + {0x1BC00, 0x1BC6A, prOLetter}, // Lo [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M + {0x1BC70, 0x1BC7C, prOLetter}, // Lo [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK + {0x1BC80, 0x1BC88, prOLetter}, // Lo [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL + {0x1BC90, 0x1BC99, prOLetter}, // Lo [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW + {0x1BC9D, 0x1BC9E, prExtend}, // Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK + {0x1BC9F, 0x1BC9F, prSTerm}, // Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP + {0x1BCA0, 0x1BCA3, prFormat}, // Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP + {0x1CF00, 0x1CF2D, prExtend}, // Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT + {0x1CF30, 0x1CF46, prExtend}, // Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG + {0x1D165, 0x1D166, prExtend}, // Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM + {0x1D167, 0x1D169, prExtend}, // Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 + {0x1D16D, 0x1D172, prExtend}, // Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 + {0x1D173, 0x1D17A, prFormat}, // Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE + {0x1D17B, 0x1D182, prExtend}, // Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE + {0x1D185, 0x1D18B, prExtend}, // Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE + {0x1D1AA, 0x1D1AD, prExtend}, // Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO + {0x1D242, 0x1D244, prExtend}, // Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + {0x1D400, 0x1D419, prUpper}, // L& [26] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL BOLD CAPITAL Z + {0x1D41A, 0x1D433, prLower}, // L& [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z + {0x1D434, 0x1D44D, prUpper}, // L& [26] MATHEMATICAL ITALIC CAPITAL A..MATHEMATICAL ITALIC CAPITAL Z + {0x1D44E, 0x1D454, prLower}, // L& [7] MATHEMATICAL ITALIC SMALL A..MATHEMATICAL ITALIC SMALL G + {0x1D456, 0x1D467, prLower}, // L& [18] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL Z + {0x1D468, 0x1D481, prUpper}, // L& [26] MATHEMATICAL BOLD ITALIC CAPITAL A..MATHEMATICAL BOLD ITALIC CAPITAL Z + {0x1D482, 0x1D49B, prLower}, // L& [26] MATHEMATICAL BOLD ITALIC SMALL A..MATHEMATICAL BOLD ITALIC SMALL Z + {0x1D49C, 0x1D49C, prUpper}, // L& MATHEMATICAL SCRIPT CAPITAL A + {0x1D49E, 0x1D49F, prUpper}, // L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D + {0x1D4A2, 0x1D4A2, prUpper}, // L& MATHEMATICAL SCRIPT CAPITAL G + {0x1D4A5, 0x1D4A6, prUpper}, // L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K + {0x1D4A9, 0x1D4AC, prUpper}, // L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q + {0x1D4AE, 0x1D4B5, prUpper}, // L& [8] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT CAPITAL Z + {0x1D4B6, 0x1D4B9, prLower}, // L& [4] MATHEMATICAL SCRIPT SMALL A..MATHEMATICAL SCRIPT SMALL D + {0x1D4BB, 0x1D4BB, prLower}, // L& MATHEMATICAL SCRIPT SMALL F + {0x1D4BD, 0x1D4C3, prLower}, // L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N + {0x1D4C5, 0x1D4CF, prLower}, // L& [11] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL SCRIPT SMALL Z + {0x1D4D0, 0x1D4E9, prUpper}, // L& [26] MATHEMATICAL BOLD SCRIPT CAPITAL A..MATHEMATICAL BOLD SCRIPT CAPITAL Z + {0x1D4EA, 0x1D503, prLower}, // L& [26] MATHEMATICAL BOLD SCRIPT SMALL A..MATHEMATICAL BOLD SCRIPT SMALL Z + {0x1D504, 0x1D505, prUpper}, // L& [2] MATHEMATICAL FRAKTUR CAPITAL A..MATHEMATICAL FRAKTUR CAPITAL B + {0x1D507, 0x1D50A, prUpper}, // L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G + {0x1D50D, 0x1D514, prUpper}, // L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q + {0x1D516, 0x1D51C, prUpper}, // L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y + {0x1D51E, 0x1D537, prLower}, // L& [26] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL FRAKTUR SMALL Z + {0x1D538, 0x1D539, prUpper}, // L& [2] MATHEMATICAL DOUBLE-STRUCK CAPITAL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B + {0x1D53B, 0x1D53E, prUpper}, // L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G + {0x1D540, 0x1D544, prUpper}, // L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M + {0x1D546, 0x1D546, prUpper}, // L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O + {0x1D54A, 0x1D550, prUpper}, // L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y + {0x1D552, 0x1D56B, prLower}, // L& [26] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL DOUBLE-STRUCK SMALL Z + {0x1D56C, 0x1D585, prUpper}, // L& [26] MATHEMATICAL BOLD FRAKTUR CAPITAL A..MATHEMATICAL BOLD FRAKTUR CAPITAL Z + {0x1D586, 0x1D59F, prLower}, // L& [26] MATHEMATICAL BOLD FRAKTUR SMALL A..MATHEMATICAL BOLD FRAKTUR SMALL Z + {0x1D5A0, 0x1D5B9, prUpper}, // L& [26] MATHEMATICAL SANS-SERIF CAPITAL A..MATHEMATICAL SANS-SERIF CAPITAL Z + {0x1D5BA, 0x1D5D3, prLower}, // L& [26] MATHEMATICAL SANS-SERIF SMALL A..MATHEMATICAL SANS-SERIF SMALL Z + {0x1D5D4, 0x1D5ED, prUpper}, // L& [26] MATHEMATICAL SANS-SERIF BOLD CAPITAL A..MATHEMATICAL SANS-SERIF BOLD CAPITAL Z + {0x1D5EE, 0x1D607, prLower}, // L& [26] MATHEMATICAL SANS-SERIF BOLD SMALL A..MATHEMATICAL SANS-SERIF BOLD SMALL Z + {0x1D608, 0x1D621, prUpper}, // L& [26] MATHEMATICAL SANS-SERIF ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF ITALIC CAPITAL Z + {0x1D622, 0x1D63B, prLower}, // L& [26] MATHEMATICAL SANS-SERIF ITALIC SMALL A..MATHEMATICAL SANS-SERIF ITALIC SMALL Z + {0x1D63C, 0x1D655, prUpper}, // L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Z + {0x1D656, 0x1D66F, prLower}, // L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z + {0x1D670, 0x1D689, prUpper}, // L& [26] MATHEMATICAL MONOSPACE CAPITAL A..MATHEMATICAL MONOSPACE CAPITAL Z + {0x1D68A, 0x1D6A5, prLower}, // L& [28] MATHEMATICAL MONOSPACE SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J + {0x1D6A8, 0x1D6C0, prUpper}, // L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA + {0x1D6C2, 0x1D6DA, prLower}, // L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA + {0x1D6DC, 0x1D6E1, prLower}, // L& [6] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL BOLD PI SYMBOL + {0x1D6E2, 0x1D6FA, prUpper}, // L& [25] MATHEMATICAL ITALIC CAPITAL ALPHA..MATHEMATICAL ITALIC CAPITAL OMEGA + {0x1D6FC, 0x1D714, prLower}, // L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA + {0x1D716, 0x1D71B, prLower}, // L& [6] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL ITALIC PI SYMBOL + {0x1D71C, 0x1D734, prUpper}, // L& [25] MATHEMATICAL BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA + {0x1D736, 0x1D74E, prLower}, // L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA + {0x1D750, 0x1D755, prLower}, // L& [6] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC PI SYMBOL + {0x1D756, 0x1D76E, prUpper}, // L& [25] MATHEMATICAL SANS-SERIF BOLD CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA + {0x1D770, 0x1D788, prLower}, // L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA + {0x1D78A, 0x1D78F, prLower}, // L& [6] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD PI SYMBOL + {0x1D790, 0x1D7A8, prUpper}, // L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA + {0x1D7AA, 0x1D7C2, prLower}, // L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA + {0x1D7C4, 0x1D7C9, prLower}, // L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL + {0x1D7CA, 0x1D7CA, prUpper}, // L& MATHEMATICAL BOLD CAPITAL DIGAMMA + {0x1D7CB, 0x1D7CB, prLower}, // L& MATHEMATICAL BOLD SMALL DIGAMMA + {0x1D7CE, 0x1D7FF, prNumeric}, // Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + {0x1DA00, 0x1DA36, prExtend}, // Mn [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN + {0x1DA3B, 0x1DA6C, prExtend}, // Mn [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT + {0x1DA75, 0x1DA75, prExtend}, // Mn SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS + {0x1DA84, 0x1DA84, prExtend}, // Mn SIGNWRITING LOCATION HEAD NECK + {0x1DA88, 0x1DA88, prSTerm}, // Po SIGNWRITING FULL STOP + {0x1DA9B, 0x1DA9F, prExtend}, // Mn [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 + {0x1DAA1, 0x1DAAF, prExtend}, // Mn [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 + {0x1DF00, 0x1DF09, prLower}, // L& [10] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK + {0x1DF0A, 0x1DF0A, prOLetter}, // Lo LATIN LETTER RETROFLEX CLICK WITH RETROFLEX HOOK + {0x1DF0B, 0x1DF1E, prLower}, // L& [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL + {0x1DF25, 0x1DF2A, prLower}, // L& [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK + {0x1E000, 0x1E006, prExtend}, // Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE + {0x1E008, 0x1E018, prExtend}, // Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU + {0x1E01B, 0x1E021, prExtend}, // Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI + {0x1E023, 0x1E024, prExtend}, // Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS + {0x1E026, 0x1E02A, prExtend}, // Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA + {0x1E030, 0x1E06D, prLower}, // Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE + {0x1E08F, 0x1E08F, prExtend}, // Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + {0x1E100, 0x1E12C, prOLetter}, // Lo [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W + {0x1E130, 0x1E136, prExtend}, // Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D + {0x1E137, 0x1E13D, prOLetter}, // Lm [7] NYIAKENG PUACHUE HMONG SIGN FOR PERSON..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER + {0x1E140, 0x1E149, prNumeric}, // Nd [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE + {0x1E14E, 0x1E14E, prOLetter}, // Lo NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ + {0x1E290, 0x1E2AD, prOLetter}, // Lo [30] TOTO LETTER PA..TOTO LETTER A + {0x1E2AE, 0x1E2AE, prExtend}, // Mn TOTO SIGN RISING TONE + {0x1E2C0, 0x1E2EB, prOLetter}, // Lo [44] WANCHO LETTER AA..WANCHO LETTER YIH + {0x1E2EC, 0x1E2EF, prExtend}, // Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI + {0x1E2F0, 0x1E2F9, prNumeric}, // Nd [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE + {0x1E4D0, 0x1E4EA, prOLetter}, // Lo [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL + {0x1E4EB, 0x1E4EB, prOLetter}, // Lm NAG MUNDARI SIGN OJOD + {0x1E4EC, 0x1E4EF, prExtend}, // Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH + {0x1E4F0, 0x1E4F9, prNumeric}, // Nd [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE + {0x1E7E0, 0x1E7E6, prOLetter}, // Lo [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO + {0x1E7E8, 0x1E7EB, prOLetter}, // Lo [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE + {0x1E7ED, 0x1E7EE, prOLetter}, // Lo [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE + {0x1E7F0, 0x1E7FE, prOLetter}, // Lo [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE + {0x1E800, 0x1E8C4, prOLetter}, // Lo [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON + {0x1E8D0, 0x1E8D6, prExtend}, // Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + {0x1E900, 0x1E921, prUpper}, // L& [34] ADLAM CAPITAL LETTER ALIF..ADLAM CAPITAL LETTER SHA + {0x1E922, 0x1E943, prLower}, // L& [34] ADLAM SMALL LETTER ALIF..ADLAM SMALL LETTER SHA + {0x1E944, 0x1E94A, prExtend}, // Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA + {0x1E94B, 0x1E94B, prOLetter}, // Lm ADLAM NASALIZATION MARK + {0x1E950, 0x1E959, prNumeric}, // Nd [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE + {0x1EE00, 0x1EE03, prOLetter}, // Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL + {0x1EE05, 0x1EE1F, prOLetter}, // Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF + {0x1EE21, 0x1EE22, prOLetter}, // Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM + {0x1EE24, 0x1EE24, prOLetter}, // Lo ARABIC MATHEMATICAL INITIAL HEH + {0x1EE27, 0x1EE27, prOLetter}, // Lo ARABIC MATHEMATICAL INITIAL HAH + {0x1EE29, 0x1EE32, prOLetter}, // Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF + {0x1EE34, 0x1EE37, prOLetter}, // Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH + {0x1EE39, 0x1EE39, prOLetter}, // Lo ARABIC MATHEMATICAL INITIAL DAD + {0x1EE3B, 0x1EE3B, prOLetter}, // Lo ARABIC MATHEMATICAL INITIAL GHAIN + {0x1EE42, 0x1EE42, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED JEEM + {0x1EE47, 0x1EE47, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED HAH + {0x1EE49, 0x1EE49, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED YEH + {0x1EE4B, 0x1EE4B, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED LAM + {0x1EE4D, 0x1EE4F, prOLetter}, // Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN + {0x1EE51, 0x1EE52, prOLetter}, // Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF + {0x1EE54, 0x1EE54, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED SHEEN + {0x1EE57, 0x1EE57, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED KHAH + {0x1EE59, 0x1EE59, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED DAD + {0x1EE5B, 0x1EE5B, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED GHAIN + {0x1EE5D, 0x1EE5D, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON + {0x1EE5F, 0x1EE5F, prOLetter}, // Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF + {0x1EE61, 0x1EE62, prOLetter}, // Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM + {0x1EE64, 0x1EE64, prOLetter}, // Lo ARABIC MATHEMATICAL STRETCHED HEH + {0x1EE67, 0x1EE6A, prOLetter}, // Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF + {0x1EE6C, 0x1EE72, prOLetter}, // Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF + {0x1EE74, 0x1EE77, prOLetter}, // Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH + {0x1EE79, 0x1EE7C, prOLetter}, // Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH + {0x1EE7E, 0x1EE7E, prOLetter}, // Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH + {0x1EE80, 0x1EE89, prOLetter}, // Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH + {0x1EE8B, 0x1EE9B, prOLetter}, // Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN + {0x1EEA1, 0x1EEA3, prOLetter}, // Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL + {0x1EEA5, 0x1EEA9, prOLetter}, // Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH + {0x1EEAB, 0x1EEBB, prOLetter}, // Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN + {0x1F130, 0x1F149, prUpper}, // So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z + {0x1F150, 0x1F169, prUpper}, // So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z + {0x1F170, 0x1F189, prUpper}, // So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z + {0x1F676, 0x1F678, prClose}, // So [3] SANS-SERIF HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT..SANS-SERIF HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT + {0x1FBF0, 0x1FBF9, prNumeric}, // Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE + {0x20000, 0x2A6DF, prOLetter}, // Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF + {0x2A700, 0x2B739, prOLetter}, // Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 + {0x2B740, 0x2B81D, prOLetter}, // Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D + {0x2B820, 0x2CEA1, prOLetter}, // Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 + {0x2CEB0, 0x2EBE0, prOLetter}, // Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 + {0x2F800, 0x2FA1D, prOLetter}, // Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + {0x30000, 0x3134A, prOLetter}, // Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A + {0x31350, 0x323AF, prOLetter}, // Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF + {0xE0001, 0xE0001, prFormat}, // Cf LANGUAGE TAG + {0xE0020, 0xE007F, prExtend}, // Cf [96] TAG SPACE..CANCEL TAG + {0xE0100, 0xE01EF, prExtend}, // Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 +} diff --git a/vendor/github.com/rivo/uniseg/sentencerules.go b/vendor/github.com/rivo/uniseg/sentencerules.go new file mode 100644 index 0000000000..0b29c7bdb8 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/sentencerules.go @@ -0,0 +1,276 @@ +package uniseg + +import "unicode/utf8" + +// The states of the sentence break parser. +const ( + sbAny = iota + sbCR + sbParaSep + sbATerm + sbUpper + sbLower + sbSB7 + sbSB8Close + sbSB8Sp + sbSTerm + sbSB8aClose + sbSB8aSp +) + +// sbTransitions implements the sentence break parser's state transitions. It's +// anologous to [grTransitions], see comments there for details. +// +// Unicode version 15.0.0. +func sbTransitions(state, prop int) (newState int, sentenceBreak bool, rule int) { + switch uint64(state) | uint64(prop)<<32 { + // SB3. + case sbAny | prCR<<32: + return sbCR, false, 9990 + case sbCR | prLF<<32: + return sbParaSep, false, 30 + + // SB4. + case sbAny | prSep<<32: + return sbParaSep, false, 9990 + case sbAny | prLF<<32: + return sbParaSep, false, 9990 + case sbParaSep | prAny<<32: + return sbAny, true, 40 + case sbCR | prAny<<32: + return sbAny, true, 40 + + // SB6. + case sbAny | prATerm<<32: + return sbATerm, false, 9990 + case sbATerm | prNumeric<<32: + return sbAny, false, 60 + case sbSB7 | prNumeric<<32: + return sbAny, false, 60 // Because ATerm also appears in SB7. + + // SB7. + case sbAny | prUpper<<32: + return sbUpper, false, 9990 + case sbAny | prLower<<32: + return sbLower, false, 9990 + case sbUpper | prATerm<<32: + return sbSB7, false, 70 + case sbLower | prATerm<<32: + return sbSB7, false, 70 + case sbSB7 | prUpper<<32: + return sbUpper, false, 70 + + // SB8a. + case sbAny | prSTerm<<32: + return sbSTerm, false, 9990 + case sbATerm | prSContinue<<32: + return sbAny, false, 81 + case sbATerm | prATerm<<32: + return sbATerm, false, 81 + case sbATerm | prSTerm<<32: + return sbSTerm, false, 81 + case sbSB7 | prSContinue<<32: + return sbAny, false, 81 + case sbSB7 | prATerm<<32: + return sbATerm, false, 81 + case sbSB7 | prSTerm<<32: + return sbSTerm, false, 81 + case sbSB8Close | prSContinue<<32: + return sbAny, false, 81 + case sbSB8Close | prATerm<<32: + return sbATerm, false, 81 + case sbSB8Close | prSTerm<<32: + return sbSTerm, false, 81 + case sbSB8Sp | prSContinue<<32: + return sbAny, false, 81 + case sbSB8Sp | prATerm<<32: + return sbATerm, false, 81 + case sbSB8Sp | prSTerm<<32: + return sbSTerm, false, 81 + case sbSTerm | prSContinue<<32: + return sbAny, false, 81 + case sbSTerm | prATerm<<32: + return sbATerm, false, 81 + case sbSTerm | prSTerm<<32: + return sbSTerm, false, 81 + case sbSB8aClose | prSContinue<<32: + return sbAny, false, 81 + case sbSB8aClose | prATerm<<32: + return sbATerm, false, 81 + case sbSB8aClose | prSTerm<<32: + return sbSTerm, false, 81 + case sbSB8aSp | prSContinue<<32: + return sbAny, false, 81 + case sbSB8aSp | prATerm<<32: + return sbATerm, false, 81 + case sbSB8aSp | prSTerm<<32: + return sbSTerm, false, 81 + + // SB9. + case sbATerm | prClose<<32: + return sbSB8Close, false, 90 + case sbSB7 | prClose<<32: + return sbSB8Close, false, 90 + case sbSB8Close | prClose<<32: + return sbSB8Close, false, 90 + case sbATerm | prSp<<32: + return sbSB8Sp, false, 90 + case sbSB7 | prSp<<32: + return sbSB8Sp, false, 90 + case sbSB8Close | prSp<<32: + return sbSB8Sp, false, 90 + case sbSTerm | prClose<<32: + return sbSB8aClose, false, 90 + case sbSB8aClose | prClose<<32: + return sbSB8aClose, false, 90 + case sbSTerm | prSp<<32: + return sbSB8aSp, false, 90 + case sbSB8aClose | prSp<<32: + return sbSB8aSp, false, 90 + case sbATerm | prSep<<32: + return sbParaSep, false, 90 + case sbATerm | prCR<<32: + return sbParaSep, false, 90 + case sbATerm | prLF<<32: + return sbParaSep, false, 90 + case sbSB7 | prSep<<32: + return sbParaSep, false, 90 + case sbSB7 | prCR<<32: + return sbParaSep, false, 90 + case sbSB7 | prLF<<32: + return sbParaSep, false, 90 + case sbSB8Close | prSep<<32: + return sbParaSep, false, 90 + case sbSB8Close | prCR<<32: + return sbParaSep, false, 90 + case sbSB8Close | prLF<<32: + return sbParaSep, false, 90 + case sbSTerm | prSep<<32: + return sbParaSep, false, 90 + case sbSTerm | prCR<<32: + return sbParaSep, false, 90 + case sbSTerm | prLF<<32: + return sbParaSep, false, 90 + case sbSB8aClose | prSep<<32: + return sbParaSep, false, 90 + case sbSB8aClose | prCR<<32: + return sbParaSep, false, 90 + case sbSB8aClose | prLF<<32: + return sbParaSep, false, 90 + + // SB10. + case sbSB8Sp | prSp<<32: + return sbSB8Sp, false, 100 + case sbSB8aSp | prSp<<32: + return sbSB8aSp, false, 100 + case sbSB8Sp | prSep<<32: + return sbParaSep, false, 100 + case sbSB8Sp | prCR<<32: + return sbParaSep, false, 100 + case sbSB8Sp | prLF<<32: + return sbParaSep, false, 100 + + // SB11. + case sbATerm | prAny<<32: + return sbAny, true, 110 + case sbSB7 | prAny<<32: + return sbAny, true, 110 + case sbSB8Close | prAny<<32: + return sbAny, true, 110 + case sbSB8Sp | prAny<<32: + return sbAny, true, 110 + case sbSTerm | prAny<<32: + return sbAny, true, 110 + case sbSB8aClose | prAny<<32: + return sbAny, true, 110 + case sbSB8aSp | prAny<<32: + return sbAny, true, 110 + // We'll always break after ParaSep due to SB4. + + default: + return -1, false, -1 + } +} + +// transitionSentenceBreakState determines the new state of the sentence break +// parser given the current state and the next code point. It also returns +// whether a sentence boundary was detected. If more than one code point is +// needed to determine the new state, the byte slice or the string starting +// after rune "r" can be used (whichever is not nil or empty) for further +// lookups. +func transitionSentenceBreakState(state int, r rune, b []byte, str string) (newState int, sentenceBreak bool) { + // Determine the property of the next character. + nextProperty := property(sentenceBreakCodePoints, r) + + // SB5 (Replacing Ignore Rules). + if nextProperty == prExtend || nextProperty == prFormat { + if state == sbParaSep || state == sbCR { + return sbAny, true // Make sure we don't apply SB5 to SB3 or SB4. + } + if state < 0 { + return sbAny, true // SB1. + } + return state, false + } + + // Find the applicable transition in the table. + var rule int + newState, sentenceBreak, rule = sbTransitions(state, nextProperty) + if newState < 0 { + // No specific transition found. Try the less specific ones. + anyPropState, anyPropProp, anyPropRule := sbTransitions(state, prAny) + anyStateState, anyStateProp, anyStateRule := sbTransitions(sbAny, nextProperty) + if anyPropState >= 0 && anyStateState >= 0 { + // Both apply. We'll use a mix (see comments for grTransitions). + newState, sentenceBreak, rule = anyStateState, anyStateProp, anyStateRule + if anyPropRule < anyStateRule { + sentenceBreak, rule = anyPropProp, anyPropRule + } + } else if anyPropState >= 0 { + // We only have a specific state. + newState, sentenceBreak, rule = anyPropState, anyPropProp, anyPropRule + // This branch will probably never be reached because okAnyState will + // always be true given the current transition map. But we keep it here + // for future modifications to the transition map where this may not be + // true anymore. + } else if anyStateState >= 0 { + // We only have a specific property. + newState, sentenceBreak, rule = anyStateState, anyStateProp, anyStateRule + } else { + // No known transition. SB999: Any ร— Any. + newState, sentenceBreak, rule = sbAny, false, 9990 + } + } + + // SB8. + if rule > 80 && (state == sbATerm || state == sbSB8Close || state == sbSB8Sp || state == sbSB7) { + // Check the right side of the rule. + var length int + for nextProperty != prOLetter && + nextProperty != prUpper && + nextProperty != prLower && + nextProperty != prSep && + nextProperty != prCR && + nextProperty != prLF && + nextProperty != prATerm && + nextProperty != prSTerm { + // Move on to the next rune. + if b != nil { // Byte slice version. + r, length = utf8.DecodeRune(b) + b = b[length:] + } else { // String version. + r, length = utf8.DecodeRuneInString(str) + str = str[length:] + } + if r == utf8.RuneError { + break + } + nextProperty = property(sentenceBreakCodePoints, r) + } + if nextProperty == prLower { + return sbLower, false + } + } + + return +} diff --git a/vendor/github.com/rivo/uniseg/step.go b/vendor/github.com/rivo/uniseg/step.go new file mode 100644 index 0000000000..9b72c5e596 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/step.go @@ -0,0 +1,242 @@ +package uniseg + +import "unicode/utf8" + +// The bit masks used to extract boundary information returned by [Step]. +const ( + MaskLine = 3 + MaskWord = 4 + MaskSentence = 8 +) + +// The number of bits to shift the boundary information returned by [Step] to +// obtain the monospace width of the grapheme cluster. +const ShiftWidth = 4 + +// The bit positions by which boundary flags are shifted by the [Step] function. +// These must correspond to the Mask constants. +const ( + shiftWord = 2 + shiftSentence = 3 + // shiftwWidth is ShiftWidth above. No mask as these are always the remaining bits. +) + +// The bit positions by which states are shifted by the [Step] function. These +// values must ensure state values defined for each of the boundary algorithms +// don't overlap (and that they all still fit in a single int). These must +// correspond to the Mask constants. +const ( + shiftWordState = 4 + shiftSentenceState = 9 + shiftLineState = 13 + shiftPropState = 21 // No mask as these are always the remaining bits. +) + +// The bit mask used to extract the state returned by the [Step] function, after +// shifting. These values must correspond to the shift constants. +const ( + maskGraphemeState = 0xf + maskWordState = 0x1f + maskSentenceState = 0xf + maskLineState = 0xff +) + +// Step returns the first grapheme cluster (user-perceived character) found in +// the given byte slice. It also returns information about the boundary between +// that grapheme cluster and the one following it as well as the monospace width +// of the grapheme cluster. There are three types of boundary information: word +// boundaries, sentence boundaries, and line breaks. This function is therefore +// a combination of [FirstGraphemeCluster], [FirstWord], [FirstSentence], and +// [FirstLineSegment]. +// +// The "boundaries" return value can be evaluated as follows: +// +// - boundaries&MaskWord != 0: The boundary is a word boundary. +// - boundaries&MaskWord == 0: The boundary is not a word boundary. +// - boundaries&MaskSentence != 0: The boundary is a sentence boundary. +// - boundaries&MaskSentence == 0: The boundary is not a sentence boundary. +// - boundaries&MaskLine == LineDontBreak: You must not break the line at the +// boundary. +// - boundaries&MaskLine == LineMustBreak: You must break the line at the +// boundary. +// - boundaries&MaskLine == LineCanBreak: You may or may not break the line at +// the boundary. +// - boundaries >> ShiftWidth: The width of the grapheme cluster for most +// monospace fonts where a value of 1 represents one character cell. +// +// This function can be called continuously to extract all grapheme clusters +// from a byte slice, as illustrated in the examples below. +// +// If you don't know which state to pass, for example when calling the function +// for the first time, you must pass -1. For consecutive calls, pass the state +// and rest slice returned by the previous call. +// +// The "rest" slice is the sub-slice of the original byte slice "b" starting +// after the last byte of the identified grapheme cluster. If the length of the +// "rest" slice is 0, the entire byte slice "b" has been processed. The +// "cluster" byte slice is the sub-slice of the input slice containing the +// first identified grapheme cluster. +// +// Given an empty byte slice "b", the function returns nil values. +// +// While slightly less convenient than using the Graphemes class, this function +// has much better performance and makes no allocations. It lends itself well to +// large byte slices. +// +// Note that in accordance with [UAX #14 LB3], the final segment will end with +// a mandatory line break (boundaries&MaskLine == LineMustBreak). You can choose +// to ignore this by checking if the length of the "rest" slice is 0 and calling +// [HasTrailingLineBreak] or [HasTrailingLineBreakInString] on the last rune. +// +// [UAX #14 LB3]: https://www.unicode.org/reports/tr14/#Algorithm +func Step(b []byte, state int) (cluster, rest []byte, boundaries int, newState int) { + // An empty byte slice returns nothing. + if len(b) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRune(b) + if len(b) <= length { // If we're already past the end, there is nothing else to parse. + var prop int + if state < 0 { + prop = propertyGraphemes(r) + } else { + prop = state >> shiftPropState + } + return b, nil, LineMustBreak | (1 << shiftWord) | (1 << shiftSentence) | (runeWidth(r, prop) << ShiftWidth), grAny | (wbAny << shiftWordState) | (sbAny << shiftSentenceState) | (lbAny << shiftLineState) | (prop << shiftPropState) + } + + // If we don't know the state, determine it now. + var graphemeState, wordState, sentenceState, lineState, firstProp int + remainder := b[length:] + if state < 0 { + graphemeState, firstProp, _ = transitionGraphemeState(state, r) + wordState, _ = transitionWordBreakState(state, r, remainder, "") + sentenceState, _ = transitionSentenceBreakState(state, r, remainder, "") + lineState, _ = transitionLineBreakState(state, r, remainder, "") + } else { + graphemeState = state & maskGraphemeState + wordState = (state >> shiftWordState) & maskWordState + sentenceState = (state >> shiftSentenceState) & maskSentenceState + lineState = (state >> shiftLineState) & maskLineState + firstProp = state >> shiftPropState + } + + // Transition until we find a grapheme cluster boundary. + width := runeWidth(r, firstProp) + for { + var ( + graphemeBoundary, wordBoundary, sentenceBoundary bool + lineBreak, prop int + ) + + r, l := utf8.DecodeRune(remainder) + remainder = b[length+l:] + + graphemeState, prop, graphemeBoundary = transitionGraphemeState(graphemeState, r) + wordState, wordBoundary = transitionWordBreakState(wordState, r, remainder, "") + sentenceState, sentenceBoundary = transitionSentenceBreakState(sentenceState, r, remainder, "") + lineState, lineBreak = transitionLineBreakState(lineState, r, remainder, "") + + if graphemeBoundary { + boundary := lineBreak | (width << ShiftWidth) + if wordBoundary { + boundary |= 1 << shiftWord + } + if sentenceBoundary { + boundary |= 1 << shiftSentence + } + return b[:length], b[length:], boundary, graphemeState | (wordState << shiftWordState) | (sentenceState << shiftSentenceState) | (lineState << shiftLineState) | (prop << shiftPropState) + } + + if firstProp == prExtendedPictographic { + if r == vs15 { + width = 1 + } else if r == vs16 { + width = 2 + } + } else if firstProp != prRegionalIndicator && firstProp != prL { + width += runeWidth(r, prop) + } + + length += l + if len(b) <= length { + return b, nil, LineMustBreak | (1 << shiftWord) | (1 << shiftSentence) | (width << ShiftWidth), grAny | (wbAny << shiftWordState) | (sbAny << shiftSentenceState) | (lbAny << shiftLineState) | (prop << shiftPropState) + } + } +} + +// StepString is like [Step] but its input and outputs are strings. +func StepString(str string, state int) (cluster, rest string, boundaries int, newState int) { + // An empty byte slice returns nothing. + if len(str) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRuneInString(str) + if len(str) <= length { // If we're already past the end, there is nothing else to parse. + prop := propertyGraphemes(r) + return str, "", LineMustBreak | (1 << shiftWord) | (1 << shiftSentence) | (runeWidth(r, prop) << ShiftWidth), grAny | (wbAny << shiftWordState) | (sbAny << shiftSentenceState) | (lbAny << shiftLineState) + } + + // If we don't know the state, determine it now. + var graphemeState, wordState, sentenceState, lineState, firstProp int + remainder := str[length:] + if state < 0 { + graphemeState, firstProp, _ = transitionGraphemeState(state, r) + wordState, _ = transitionWordBreakState(state, r, nil, remainder) + sentenceState, _ = transitionSentenceBreakState(state, r, nil, remainder) + lineState, _ = transitionLineBreakState(state, r, nil, remainder) + } else { + graphemeState = state & maskGraphemeState + wordState = (state >> shiftWordState) & maskWordState + sentenceState = (state >> shiftSentenceState) & maskSentenceState + lineState = (state >> shiftLineState) & maskLineState + firstProp = state >> shiftPropState + } + + // Transition until we find a grapheme cluster boundary. + width := runeWidth(r, firstProp) + for { + var ( + graphemeBoundary, wordBoundary, sentenceBoundary bool + lineBreak, prop int + ) + + r, l := utf8.DecodeRuneInString(remainder) + remainder = str[length+l:] + + graphemeState, prop, graphemeBoundary = transitionGraphemeState(graphemeState, r) + wordState, wordBoundary = transitionWordBreakState(wordState, r, nil, remainder) + sentenceState, sentenceBoundary = transitionSentenceBreakState(sentenceState, r, nil, remainder) + lineState, lineBreak = transitionLineBreakState(lineState, r, nil, remainder) + + if graphemeBoundary { + boundary := lineBreak | (width << ShiftWidth) + if wordBoundary { + boundary |= 1 << shiftWord + } + if sentenceBoundary { + boundary |= 1 << shiftSentence + } + return str[:length], str[length:], boundary, graphemeState | (wordState << shiftWordState) | (sentenceState << shiftSentenceState) | (lineState << shiftLineState) | (prop << shiftPropState) + } + + if firstProp == prExtendedPictographic { + if r == vs15 { + width = 1 + } else if r == vs16 { + width = 2 + } + } else if firstProp != prRegionalIndicator && firstProp != prL { + width += runeWidth(r, prop) + } + + length += l + if len(str) <= length { + return str, "", LineMustBreak | (1 << shiftWord) | (1 << shiftSentence) | (width << ShiftWidth), grAny | (wbAny << shiftWordState) | (sbAny << shiftSentenceState) | (lbAny << shiftLineState) | (prop << shiftPropState) + } + } +} diff --git a/vendor/github.com/rivo/uniseg/width.go b/vendor/github.com/rivo/uniseg/width.go new file mode 100644 index 0000000000..975a9f1343 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/width.go @@ -0,0 +1,61 @@ +package uniseg + +// EastAsianAmbiguousWidth specifies the monospace width for East Asian +// characters classified as Ambiguous. The default is 1 but some rare fonts +// render them with a width of 2. +var EastAsianAmbiguousWidth = 1 + +// runeWidth returns the monospace width for the given rune. The provided +// grapheme property is a value mapped by the [graphemeCodePoints] table. +// +// Every rune has a width of 1, except for runes with the following properties +// (evaluated in this order): +// +// - Control, CR, LF, Extend, ZWJ: Width of 0 +// - \u2e3a, TWO-EM DASH: Width of 3 +// - \u2e3b, THREE-EM DASH: Width of 4 +// - East-Asian width Fullwidth and Wide: Width of 2 (Ambiguous and Neutral +// have a width of 1) +// - Regional Indicator: Width of 2 +// - Extended Pictographic: Width of 2, unless Emoji Presentation is "No". +func runeWidth(r rune, graphemeProperty int) int { + switch graphemeProperty { + case prControl, prCR, prLF, prExtend, prZWJ: + return 0 + case prRegionalIndicator: + return 2 + case prExtendedPictographic: + if property(emojiPresentation, r) == prEmojiPresentation { + return 2 + } + return 1 + } + + switch r { + case 0x2e3a: + return 3 + case 0x2e3b: + return 4 + } + + switch propertyEastAsianWidth(r) { + case prW, prF: + return 2 + case prA: + return EastAsianAmbiguousWidth + } + + return 1 +} + +// StringWidth returns the monospace width for the given string, that is, the +// number of same-size cells to be occupied by the string. +func StringWidth(s string) (width int) { + state := -1 + for len(s) > 0 { + var w int + _, s, w, state = FirstGraphemeClusterInString(s, state) + width += w + } + return +} diff --git a/vendor/github.com/rivo/uniseg/word.go b/vendor/github.com/rivo/uniseg/word.go new file mode 100644 index 0000000000..34fba7f291 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/word.go @@ -0,0 +1,89 @@ +package uniseg + +import "unicode/utf8" + +// FirstWord returns the first word found in the given byte slice according to +// the rules of [Unicode Standard Annex #29, Word Boundaries]. This function can +// be called continuously to extract all words from a byte slice, as illustrated +// in the example below. +// +// If you don't know the current state, for example when calling the function +// for the first time, you must pass -1. For consecutive calls, pass the state +// and rest slice returned by the previous call. +// +// The "rest" slice is the sub-slice of the original byte slice "b" starting +// after the last byte of the identified word. If the length of the "rest" slice +// is 0, the entire byte slice "b" has been processed. The "word" byte slice is +// the sub-slice of the input slice containing the identified word. +// +// Given an empty byte slice "b", the function returns nil values. +// +// [Unicode Standard Annex #29, Word Boundaries]: http://unicode.org/reports/tr29/#Word_Boundaries +func FirstWord(b []byte, state int) (word, rest []byte, newState int) { + // An empty byte slice returns nothing. + if len(b) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRune(b) + if len(b) <= length { // If we're already past the end, there is nothing else to parse. + return b, nil, wbAny + } + + // If we don't know the state, determine it now. + if state < 0 { + state, _ = transitionWordBreakState(state, r, b[length:], "") + } + + // Transition until we find a boundary. + var boundary bool + for { + r, l := utf8.DecodeRune(b[length:]) + state, boundary = transitionWordBreakState(state, r, b[length+l:], "") + + if boundary { + return b[:length], b[length:], state + } + + length += l + if len(b) <= length { + return b, nil, wbAny + } + } +} + +// FirstWordInString is like [FirstWord] but its input and outputs are strings. +func FirstWordInString(str string, state int) (word, rest string, newState int) { + // An empty byte slice returns nothing. + if len(str) == 0 { + return + } + + // Extract the first rune. + r, length := utf8.DecodeRuneInString(str) + if len(str) <= length { // If we're already past the end, there is nothing else to parse. + return str, "", wbAny + } + + // If we don't know the state, determine it now. + if state < 0 { + state, _ = transitionWordBreakState(state, r, nil, str[length:]) + } + + // Transition until we find a boundary. + var boundary bool + for { + r, l := utf8.DecodeRuneInString(str[length:]) + state, boundary = transitionWordBreakState(state, r, nil, str[length+l:]) + + if boundary { + return str[:length], str[length:], state + } + + length += l + if len(str) <= length { + return str, "", wbAny + } + } +} diff --git a/vendor/github.com/rivo/uniseg/wordproperties.go b/vendor/github.com/rivo/uniseg/wordproperties.go new file mode 100644 index 0000000000..277ca10068 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/wordproperties.go @@ -0,0 +1,1883 @@ +// Code generated via go generate from gen_properties.go. DO NOT EDIT. + +package uniseg + +// workBreakCodePoints are taken from +// https://www.unicode.org/Public/15.0.0/ucd/auxiliary/WordBreakProperty.txt +// and +// https://unicode.org/Public/15.0.0/ucd/emoji/emoji-data.txt +// ("Extended_Pictographic" only) +// on September 5, 2023. See https://www.unicode.org/license.html for the Unicode +// license agreement. +var workBreakCodePoints = [][3]int{ + {0x000A, 0x000A, prLF}, // Cc + {0x000B, 0x000C, prNewline}, // Cc [2] .. + {0x000D, 0x000D, prCR}, // Cc + {0x0020, 0x0020, prWSegSpace}, // Zs SPACE + {0x0022, 0x0022, prDoubleQuote}, // Po QUOTATION MARK + {0x0027, 0x0027, prSingleQuote}, // Po APOSTROPHE + {0x002C, 0x002C, prMidNum}, // Po COMMA + {0x002E, 0x002E, prMidNumLet}, // Po FULL STOP + {0x0030, 0x0039, prNumeric}, // Nd [10] DIGIT ZERO..DIGIT NINE + {0x003A, 0x003A, prMidLetter}, // Po COLON + {0x003B, 0x003B, prMidNum}, // Po SEMICOLON + {0x0041, 0x005A, prALetter}, // L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z + {0x005F, 0x005F, prExtendNumLet}, // Pc LOW LINE + {0x0061, 0x007A, prALetter}, // L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z + {0x0085, 0x0085, prNewline}, // Cc + {0x00A9, 0x00A9, prExtendedPictographic}, // E0.6 [1] (ยฉ๏ธ) copyright + {0x00AA, 0x00AA, prALetter}, // Lo FEMININE ORDINAL INDICATOR + {0x00AD, 0x00AD, prFormat}, // Cf SOFT HYPHEN + {0x00AE, 0x00AE, prExtendedPictographic}, // E0.6 [1] (ยฎ๏ธ) registered + {0x00B5, 0x00B5, prALetter}, // L& MICRO SIGN + {0x00B7, 0x00B7, prMidLetter}, // Po MIDDLE DOT + {0x00BA, 0x00BA, prALetter}, // Lo MASCULINE ORDINAL INDICATOR + {0x00C0, 0x00D6, prALetter}, // L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS + {0x00D8, 0x00F6, prALetter}, // L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS + {0x00F8, 0x01BA, prALetter}, // L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL + {0x01BB, 0x01BB, prALetter}, // Lo LATIN LETTER TWO WITH STROKE + {0x01BC, 0x01BF, prALetter}, // L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN + {0x01C0, 0x01C3, prALetter}, // Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK + {0x01C4, 0x0293, prALetter}, // L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL + {0x0294, 0x0294, prALetter}, // Lo LATIN LETTER GLOTTAL STOP + {0x0295, 0x02AF, prALetter}, // L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL + {0x02B0, 0x02C1, prALetter}, // Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP + {0x02C2, 0x02C5, prALetter}, // Sk [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD + {0x02C6, 0x02D1, prALetter}, // Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON + {0x02D2, 0x02D7, prALetter}, // Sk [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN + {0x02DE, 0x02DF, prALetter}, // Sk [2] MODIFIER LETTER RHOTIC HOOK..MODIFIER LETTER CROSS ACCENT + {0x02E0, 0x02E4, prALetter}, // Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP + {0x02E5, 0x02EB, prALetter}, // Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK + {0x02EC, 0x02EC, prALetter}, // Lm MODIFIER LETTER VOICING + {0x02ED, 0x02ED, prALetter}, // Sk MODIFIER LETTER UNASPIRATED + {0x02EE, 0x02EE, prALetter}, // Lm MODIFIER LETTER DOUBLE APOSTROPHE + {0x02EF, 0x02FF, prALetter}, // Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW + {0x0300, 0x036F, prExtend}, // Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X + {0x0370, 0x0373, prALetter}, // L& [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI + {0x0374, 0x0374, prALetter}, // Lm GREEK NUMERAL SIGN + {0x0376, 0x0377, prALetter}, // L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA + {0x037A, 0x037A, prALetter}, // Lm GREEK YPOGEGRAMMENI + {0x037B, 0x037D, prALetter}, // L& [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL + {0x037E, 0x037E, prMidNum}, // Po GREEK QUESTION MARK + {0x037F, 0x037F, prALetter}, // L& GREEK CAPITAL LETTER YOT + {0x0386, 0x0386, prALetter}, // L& GREEK CAPITAL LETTER ALPHA WITH TONOS + {0x0387, 0x0387, prMidLetter}, // Po GREEK ANO TELEIA + {0x0388, 0x038A, prALetter}, // L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS + {0x038C, 0x038C, prALetter}, // L& GREEK CAPITAL LETTER OMICRON WITH TONOS + {0x038E, 0x03A1, prALetter}, // L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO + {0x03A3, 0x03F5, prALetter}, // L& [83] GREEK CAPITAL LETTER SIGMA..GREEK LUNATE EPSILON SYMBOL + {0x03F7, 0x0481, prALetter}, // L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC SMALL LETTER KOPPA + {0x0483, 0x0487, prExtend}, // Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE + {0x0488, 0x0489, prExtend}, // Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN + {0x048A, 0x052F, prALetter}, // L& [166] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EL WITH DESCENDER + {0x0531, 0x0556, prALetter}, // L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH + {0x0559, 0x0559, prALetter}, // Lm ARMENIAN MODIFIER LETTER LEFT HALF RING + {0x055A, 0x055C, prALetter}, // Po [3] ARMENIAN APOSTROPHE..ARMENIAN EXCLAMATION MARK + {0x055E, 0x055E, prALetter}, // Po ARMENIAN QUESTION MARK + {0x055F, 0x055F, prMidLetter}, // Po ARMENIAN ABBREVIATION MARK + {0x0560, 0x0588, prALetter}, // L& [41] ARMENIAN SMALL LETTER TURNED AYB..ARMENIAN SMALL LETTER YI WITH STROKE + {0x0589, 0x0589, prMidNum}, // Po ARMENIAN FULL STOP + {0x058A, 0x058A, prALetter}, // Pd ARMENIAN HYPHEN + {0x0591, 0x05BD, prExtend}, // Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG + {0x05BF, 0x05BF, prExtend}, // Mn HEBREW POINT RAFE + {0x05C1, 0x05C2, prExtend}, // Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT + {0x05C4, 0x05C5, prExtend}, // Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT + {0x05C7, 0x05C7, prExtend}, // Mn HEBREW POINT QAMATS QATAN + {0x05D0, 0x05EA, prHebrewLetter}, // Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV + {0x05EF, 0x05F2, prHebrewLetter}, // Lo [4] HEBREW YOD TRIANGLE..HEBREW LIGATURE YIDDISH DOUBLE YOD + {0x05F3, 0x05F3, prALetter}, // Po HEBREW PUNCTUATION GERESH + {0x05F4, 0x05F4, prMidLetter}, // Po HEBREW PUNCTUATION GERSHAYIM + {0x0600, 0x0605, prFormat}, // Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE + {0x060C, 0x060D, prMidNum}, // Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR + {0x0610, 0x061A, prExtend}, // Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA + {0x061C, 0x061C, prFormat}, // Cf ARABIC LETTER MARK + {0x0620, 0x063F, prALetter}, // Lo [32] ARABIC LETTER KASHMIRI YEH..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE + {0x0640, 0x0640, prALetter}, // Lm ARABIC TATWEEL + {0x0641, 0x064A, prALetter}, // Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH + {0x064B, 0x065F, prExtend}, // Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW + {0x0660, 0x0669, prNumeric}, // Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE + {0x066B, 0x066B, prNumeric}, // Po ARABIC DECIMAL SEPARATOR + {0x066C, 0x066C, prMidNum}, // Po ARABIC THOUSANDS SEPARATOR + {0x066E, 0x066F, prALetter}, // Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF + {0x0670, 0x0670, prExtend}, // Mn ARABIC LETTER SUPERSCRIPT ALEF + {0x0671, 0x06D3, prALetter}, // Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE + {0x06D5, 0x06D5, prALetter}, // Lo ARABIC LETTER AE + {0x06D6, 0x06DC, prExtend}, // Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN + {0x06DD, 0x06DD, prFormat}, // Cf ARABIC END OF AYAH + {0x06DF, 0x06E4, prExtend}, // Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA + {0x06E5, 0x06E6, prALetter}, // Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH + {0x06E7, 0x06E8, prExtend}, // Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON + {0x06EA, 0x06ED, prExtend}, // Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM + {0x06EE, 0x06EF, prALetter}, // Lo [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V + {0x06F0, 0x06F9, prNumeric}, // Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE + {0x06FA, 0x06FC, prALetter}, // Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW + {0x06FF, 0x06FF, prALetter}, // Lo ARABIC LETTER HEH WITH INVERTED V + {0x070F, 0x070F, prFormat}, // Cf SYRIAC ABBREVIATION MARK + {0x0710, 0x0710, prALetter}, // Lo SYRIAC LETTER ALAPH + {0x0711, 0x0711, prExtend}, // Mn SYRIAC LETTER SUPERSCRIPT ALAPH + {0x0712, 0x072F, prALetter}, // Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER PERSIAN DHALATH + {0x0730, 0x074A, prExtend}, // Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH + {0x074D, 0x07A5, prALetter}, // Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAANA LETTER WAAVU + {0x07A6, 0x07B0, prExtend}, // Mn [11] THAANA ABAFILI..THAANA SUKUN + {0x07B1, 0x07B1, prALetter}, // Lo THAANA LETTER NAA + {0x07C0, 0x07C9, prNumeric}, // Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE + {0x07CA, 0x07EA, prALetter}, // Lo [33] NKO LETTER A..NKO LETTER JONA RA + {0x07EB, 0x07F3, prExtend}, // Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE + {0x07F4, 0x07F5, prALetter}, // Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE + {0x07F8, 0x07F8, prMidNum}, // Po NKO COMMA + {0x07FA, 0x07FA, prALetter}, // Lm NKO LAJANYALAN + {0x07FD, 0x07FD, prExtend}, // Mn NKO DANTAYALAN + {0x0800, 0x0815, prALetter}, // Lo [22] SAMARITAN LETTER ALAF..SAMARITAN LETTER TAAF + {0x0816, 0x0819, prExtend}, // Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH + {0x081A, 0x081A, prALetter}, // Lm SAMARITAN MODIFIER LETTER EPENTHETIC YUT + {0x081B, 0x0823, prExtend}, // Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A + {0x0824, 0x0824, prALetter}, // Lm SAMARITAN MODIFIER LETTER SHORT A + {0x0825, 0x0827, prExtend}, // Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U + {0x0828, 0x0828, prALetter}, // Lm SAMARITAN MODIFIER LETTER I + {0x0829, 0x082D, prExtend}, // Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA + {0x0840, 0x0858, prALetter}, // Lo [25] MANDAIC LETTER HALQA..MANDAIC LETTER AIN + {0x0859, 0x085B, prExtend}, // Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK + {0x0860, 0x086A, prALetter}, // Lo [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA + {0x0870, 0x0887, prALetter}, // Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT + {0x0889, 0x088E, prALetter}, // Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL + {0x0890, 0x0891, prFormat}, // Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE + {0x0898, 0x089F, prExtend}, // Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA + {0x08A0, 0x08C8, prALetter}, // Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF + {0x08C9, 0x08C9, prALetter}, // Lm ARABIC SMALL FARSI YEH + {0x08CA, 0x08E1, prExtend}, // Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA + {0x08E2, 0x08E2, prFormat}, // Cf ARABIC DISPUTED END OF AYAH + {0x08E3, 0x0902, prExtend}, // Mn [32] ARABIC TURNED DAMMA BELOW..DEVANAGARI SIGN ANUSVARA + {0x0903, 0x0903, prExtend}, // Mc DEVANAGARI SIGN VISARGA + {0x0904, 0x0939, prALetter}, // Lo [54] DEVANAGARI LETTER SHORT A..DEVANAGARI LETTER HA + {0x093A, 0x093A, prExtend}, // Mn DEVANAGARI VOWEL SIGN OE + {0x093B, 0x093B, prExtend}, // Mc DEVANAGARI VOWEL SIGN OOE + {0x093C, 0x093C, prExtend}, // Mn DEVANAGARI SIGN NUKTA + {0x093D, 0x093D, prALetter}, // Lo DEVANAGARI SIGN AVAGRAHA + {0x093E, 0x0940, prExtend}, // Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II + {0x0941, 0x0948, prExtend}, // Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI + {0x0949, 0x094C, prExtend}, // Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU + {0x094D, 0x094D, prExtend}, // Mn DEVANAGARI SIGN VIRAMA + {0x094E, 0x094F, prExtend}, // Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW + {0x0950, 0x0950, prALetter}, // Lo DEVANAGARI OM + {0x0951, 0x0957, prExtend}, // Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE + {0x0958, 0x0961, prALetter}, // Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL + {0x0962, 0x0963, prExtend}, // Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL + {0x0966, 0x096F, prNumeric}, // Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE + {0x0971, 0x0971, prALetter}, // Lm DEVANAGARI SIGN HIGH SPACING DOT + {0x0972, 0x0980, prALetter}, // Lo [15] DEVANAGARI LETTER CANDRA A..BENGALI ANJI + {0x0981, 0x0981, prExtend}, // Mn BENGALI SIGN CANDRABINDU + {0x0982, 0x0983, prExtend}, // Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA + {0x0985, 0x098C, prALetter}, // Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L + {0x098F, 0x0990, prALetter}, // Lo [2] BENGALI LETTER E..BENGALI LETTER AI + {0x0993, 0x09A8, prALetter}, // Lo [22] BENGALI LETTER O..BENGALI LETTER NA + {0x09AA, 0x09B0, prALetter}, // Lo [7] BENGALI LETTER PA..BENGALI LETTER RA + {0x09B2, 0x09B2, prALetter}, // Lo BENGALI LETTER LA + {0x09B6, 0x09B9, prALetter}, // Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA + {0x09BC, 0x09BC, prExtend}, // Mn BENGALI SIGN NUKTA + {0x09BD, 0x09BD, prALetter}, // Lo BENGALI SIGN AVAGRAHA + {0x09BE, 0x09C0, prExtend}, // Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II + {0x09C1, 0x09C4, prExtend}, // Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR + {0x09C7, 0x09C8, prExtend}, // Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI + {0x09CB, 0x09CC, prExtend}, // Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU + {0x09CD, 0x09CD, prExtend}, // Mn BENGALI SIGN VIRAMA + {0x09CE, 0x09CE, prALetter}, // Lo BENGALI LETTER KHANDA TA + {0x09D7, 0x09D7, prExtend}, // Mc BENGALI AU LENGTH MARK + {0x09DC, 0x09DD, prALetter}, // Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA + {0x09DF, 0x09E1, prALetter}, // Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL + {0x09E2, 0x09E3, prExtend}, // Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL + {0x09E6, 0x09EF, prNumeric}, // Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE + {0x09F0, 0x09F1, prALetter}, // Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL + {0x09FC, 0x09FC, prALetter}, // Lo BENGALI LETTER VEDIC ANUSVARA + {0x09FE, 0x09FE, prExtend}, // Mn BENGALI SANDHI MARK + {0x0A01, 0x0A02, prExtend}, // Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI + {0x0A03, 0x0A03, prExtend}, // Mc GURMUKHI SIGN VISARGA + {0x0A05, 0x0A0A, prALetter}, // Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU + {0x0A0F, 0x0A10, prALetter}, // Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI + {0x0A13, 0x0A28, prALetter}, // Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA + {0x0A2A, 0x0A30, prALetter}, // Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA + {0x0A32, 0x0A33, prALetter}, // Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA + {0x0A35, 0x0A36, prALetter}, // Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA + {0x0A38, 0x0A39, prALetter}, // Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA + {0x0A3C, 0x0A3C, prExtend}, // Mn GURMUKHI SIGN NUKTA + {0x0A3E, 0x0A40, prExtend}, // Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II + {0x0A41, 0x0A42, prExtend}, // Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU + {0x0A47, 0x0A48, prExtend}, // Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI + {0x0A4B, 0x0A4D, prExtend}, // Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA + {0x0A51, 0x0A51, prExtend}, // Mn GURMUKHI SIGN UDAAT + {0x0A59, 0x0A5C, prALetter}, // Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA + {0x0A5E, 0x0A5E, prALetter}, // Lo GURMUKHI LETTER FA + {0x0A66, 0x0A6F, prNumeric}, // Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE + {0x0A70, 0x0A71, prExtend}, // Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK + {0x0A72, 0x0A74, prALetter}, // Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR + {0x0A75, 0x0A75, prExtend}, // Mn GURMUKHI SIGN YAKASH + {0x0A81, 0x0A82, prExtend}, // Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA + {0x0A83, 0x0A83, prExtend}, // Mc GUJARATI SIGN VISARGA + {0x0A85, 0x0A8D, prALetter}, // Lo [9] GUJARATI LETTER A..GUJARATI VOWEL CANDRA E + {0x0A8F, 0x0A91, prALetter}, // Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O + {0x0A93, 0x0AA8, prALetter}, // Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA + {0x0AAA, 0x0AB0, prALetter}, // Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA + {0x0AB2, 0x0AB3, prALetter}, // Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA + {0x0AB5, 0x0AB9, prALetter}, // Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA + {0x0ABC, 0x0ABC, prExtend}, // Mn GUJARATI SIGN NUKTA + {0x0ABD, 0x0ABD, prALetter}, // Lo GUJARATI SIGN AVAGRAHA + {0x0ABE, 0x0AC0, prExtend}, // Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II + {0x0AC1, 0x0AC5, prExtend}, // Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E + {0x0AC7, 0x0AC8, prExtend}, // Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI + {0x0AC9, 0x0AC9, prExtend}, // Mc GUJARATI VOWEL SIGN CANDRA O + {0x0ACB, 0x0ACC, prExtend}, // Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU + {0x0ACD, 0x0ACD, prExtend}, // Mn GUJARATI SIGN VIRAMA + {0x0AD0, 0x0AD0, prALetter}, // Lo GUJARATI OM + {0x0AE0, 0x0AE1, prALetter}, // Lo [2] GUJARATI LETTER VOCALIC RR..GUJARATI LETTER VOCALIC LL + {0x0AE2, 0x0AE3, prExtend}, // Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL + {0x0AE6, 0x0AEF, prNumeric}, // Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE + {0x0AF9, 0x0AF9, prALetter}, // Lo GUJARATI LETTER ZHA + {0x0AFA, 0x0AFF, prExtend}, // Mn [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE + {0x0B01, 0x0B01, prExtend}, // Mn ORIYA SIGN CANDRABINDU + {0x0B02, 0x0B03, prExtend}, // Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA + {0x0B05, 0x0B0C, prALetter}, // Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L + {0x0B0F, 0x0B10, prALetter}, // Lo [2] ORIYA LETTER E..ORIYA LETTER AI + {0x0B13, 0x0B28, prALetter}, // Lo [22] ORIYA LETTER O..ORIYA LETTER NA + {0x0B2A, 0x0B30, prALetter}, // Lo [7] ORIYA LETTER PA..ORIYA LETTER RA + {0x0B32, 0x0B33, prALetter}, // Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA + {0x0B35, 0x0B39, prALetter}, // Lo [5] ORIYA LETTER VA..ORIYA LETTER HA + {0x0B3C, 0x0B3C, prExtend}, // Mn ORIYA SIGN NUKTA + {0x0B3D, 0x0B3D, prALetter}, // Lo ORIYA SIGN AVAGRAHA + {0x0B3E, 0x0B3E, prExtend}, // Mc ORIYA VOWEL SIGN AA + {0x0B3F, 0x0B3F, prExtend}, // Mn ORIYA VOWEL SIGN I + {0x0B40, 0x0B40, prExtend}, // Mc ORIYA VOWEL SIGN II + {0x0B41, 0x0B44, prExtend}, // Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR + {0x0B47, 0x0B48, prExtend}, // Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI + {0x0B4B, 0x0B4C, prExtend}, // Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU + {0x0B4D, 0x0B4D, prExtend}, // Mn ORIYA SIGN VIRAMA + {0x0B55, 0x0B56, prExtend}, // Mn [2] ORIYA SIGN OVERLINE..ORIYA AI LENGTH MARK + {0x0B57, 0x0B57, prExtend}, // Mc ORIYA AU LENGTH MARK + {0x0B5C, 0x0B5D, prALetter}, // Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA + {0x0B5F, 0x0B61, prALetter}, // Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL + {0x0B62, 0x0B63, prExtend}, // Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL + {0x0B66, 0x0B6F, prNumeric}, // Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE + {0x0B71, 0x0B71, prALetter}, // Lo ORIYA LETTER WA + {0x0B82, 0x0B82, prExtend}, // Mn TAMIL SIGN ANUSVARA + {0x0B83, 0x0B83, prALetter}, // Lo TAMIL SIGN VISARGA + {0x0B85, 0x0B8A, prALetter}, // Lo [6] TAMIL LETTER A..TAMIL LETTER UU + {0x0B8E, 0x0B90, prALetter}, // Lo [3] TAMIL LETTER E..TAMIL LETTER AI + {0x0B92, 0x0B95, prALetter}, // Lo [4] TAMIL LETTER O..TAMIL LETTER KA + {0x0B99, 0x0B9A, prALetter}, // Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA + {0x0B9C, 0x0B9C, prALetter}, // Lo TAMIL LETTER JA + {0x0B9E, 0x0B9F, prALetter}, // Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA + {0x0BA3, 0x0BA4, prALetter}, // Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA + {0x0BA8, 0x0BAA, prALetter}, // Lo [3] TAMIL LETTER NA..TAMIL LETTER PA + {0x0BAE, 0x0BB9, prALetter}, // Lo [12] TAMIL LETTER MA..TAMIL LETTER HA + {0x0BBE, 0x0BBF, prExtend}, // Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I + {0x0BC0, 0x0BC0, prExtend}, // Mn TAMIL VOWEL SIGN II + {0x0BC1, 0x0BC2, prExtend}, // Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU + {0x0BC6, 0x0BC8, prExtend}, // Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI + {0x0BCA, 0x0BCC, prExtend}, // Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU + {0x0BCD, 0x0BCD, prExtend}, // Mn TAMIL SIGN VIRAMA + {0x0BD0, 0x0BD0, prALetter}, // Lo TAMIL OM + {0x0BD7, 0x0BD7, prExtend}, // Mc TAMIL AU LENGTH MARK + {0x0BE6, 0x0BEF, prNumeric}, // Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE + {0x0C00, 0x0C00, prExtend}, // Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE + {0x0C01, 0x0C03, prExtend}, // Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA + {0x0C04, 0x0C04, prExtend}, // Mn TELUGU SIGN COMBINING ANUSVARA ABOVE + {0x0C05, 0x0C0C, prALetter}, // Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L + {0x0C0E, 0x0C10, prALetter}, // Lo [3] TELUGU LETTER E..TELUGU LETTER AI + {0x0C12, 0x0C28, prALetter}, // Lo [23] TELUGU LETTER O..TELUGU LETTER NA + {0x0C2A, 0x0C39, prALetter}, // Lo [16] TELUGU LETTER PA..TELUGU LETTER HA + {0x0C3C, 0x0C3C, prExtend}, // Mn TELUGU SIGN NUKTA + {0x0C3D, 0x0C3D, prALetter}, // Lo TELUGU SIGN AVAGRAHA + {0x0C3E, 0x0C40, prExtend}, // Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II + {0x0C41, 0x0C44, prExtend}, // Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR + {0x0C46, 0x0C48, prExtend}, // Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI + {0x0C4A, 0x0C4D, prExtend}, // Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA + {0x0C55, 0x0C56, prExtend}, // Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK + {0x0C58, 0x0C5A, prALetter}, // Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA + {0x0C5D, 0x0C5D, prALetter}, // Lo TELUGU LETTER NAKAARA POLLU + {0x0C60, 0x0C61, prALetter}, // Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL + {0x0C62, 0x0C63, prExtend}, // Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL + {0x0C66, 0x0C6F, prNumeric}, // Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE + {0x0C80, 0x0C80, prALetter}, // Lo KANNADA SIGN SPACING CANDRABINDU + {0x0C81, 0x0C81, prExtend}, // Mn KANNADA SIGN CANDRABINDU + {0x0C82, 0x0C83, prExtend}, // Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA + {0x0C85, 0x0C8C, prALetter}, // Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L + {0x0C8E, 0x0C90, prALetter}, // Lo [3] KANNADA LETTER E..KANNADA LETTER AI + {0x0C92, 0x0CA8, prALetter}, // Lo [23] KANNADA LETTER O..KANNADA LETTER NA + {0x0CAA, 0x0CB3, prALetter}, // Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA + {0x0CB5, 0x0CB9, prALetter}, // Lo [5] KANNADA LETTER VA..KANNADA LETTER HA + {0x0CBC, 0x0CBC, prExtend}, // Mn KANNADA SIGN NUKTA + {0x0CBD, 0x0CBD, prALetter}, // Lo KANNADA SIGN AVAGRAHA + {0x0CBE, 0x0CBE, prExtend}, // Mc KANNADA VOWEL SIGN AA + {0x0CBF, 0x0CBF, prExtend}, // Mn KANNADA VOWEL SIGN I + {0x0CC0, 0x0CC4, prExtend}, // Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR + {0x0CC6, 0x0CC6, prExtend}, // Mn KANNADA VOWEL SIGN E + {0x0CC7, 0x0CC8, prExtend}, // Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI + {0x0CCA, 0x0CCB, prExtend}, // Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO + {0x0CCC, 0x0CCD, prExtend}, // Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA + {0x0CD5, 0x0CD6, prExtend}, // Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK + {0x0CDD, 0x0CDE, prALetter}, // Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA + {0x0CE0, 0x0CE1, prALetter}, // Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL + {0x0CE2, 0x0CE3, prExtend}, // Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL + {0x0CE6, 0x0CEF, prNumeric}, // Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE + {0x0CF1, 0x0CF2, prALetter}, // Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA + {0x0CF3, 0x0CF3, prExtend}, // Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT + {0x0D00, 0x0D01, prExtend}, // Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU + {0x0D02, 0x0D03, prExtend}, // Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA + {0x0D04, 0x0D0C, prALetter}, // Lo [9] MALAYALAM LETTER VEDIC ANUSVARA..MALAYALAM LETTER VOCALIC L + {0x0D0E, 0x0D10, prALetter}, // Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI + {0x0D12, 0x0D3A, prALetter}, // Lo [41] MALAYALAM LETTER O..MALAYALAM LETTER TTTA + {0x0D3B, 0x0D3C, prExtend}, // Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA + {0x0D3D, 0x0D3D, prALetter}, // Lo MALAYALAM SIGN AVAGRAHA + {0x0D3E, 0x0D40, prExtend}, // Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II + {0x0D41, 0x0D44, prExtend}, // Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR + {0x0D46, 0x0D48, prExtend}, // Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI + {0x0D4A, 0x0D4C, prExtend}, // Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU + {0x0D4D, 0x0D4D, prExtend}, // Mn MALAYALAM SIGN VIRAMA + {0x0D4E, 0x0D4E, prALetter}, // Lo MALAYALAM LETTER DOT REPH + {0x0D54, 0x0D56, prALetter}, // Lo [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL + {0x0D57, 0x0D57, prExtend}, // Mc MALAYALAM AU LENGTH MARK + {0x0D5F, 0x0D61, prALetter}, // Lo [3] MALAYALAM LETTER ARCHAIC II..MALAYALAM LETTER VOCALIC LL + {0x0D62, 0x0D63, prExtend}, // Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL + {0x0D66, 0x0D6F, prNumeric}, // Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE + {0x0D7A, 0x0D7F, prALetter}, // Lo [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K + {0x0D81, 0x0D81, prExtend}, // Mn SINHALA SIGN CANDRABINDU + {0x0D82, 0x0D83, prExtend}, // Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA + {0x0D85, 0x0D96, prALetter}, // Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA + {0x0D9A, 0x0DB1, prALetter}, // Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA + {0x0DB3, 0x0DBB, prALetter}, // Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA + {0x0DBD, 0x0DBD, prALetter}, // Lo SINHALA LETTER DANTAJA LAYANNA + {0x0DC0, 0x0DC6, prALetter}, // Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA + {0x0DCA, 0x0DCA, prExtend}, // Mn SINHALA SIGN AL-LAKUNA + {0x0DCF, 0x0DD1, prExtend}, // Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA + {0x0DD2, 0x0DD4, prExtend}, // Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA + {0x0DD6, 0x0DD6, prExtend}, // Mn SINHALA VOWEL SIGN DIGA PAA-PILLA + {0x0DD8, 0x0DDF, prExtend}, // Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA + {0x0DE6, 0x0DEF, prNumeric}, // Nd [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE + {0x0DF2, 0x0DF3, prExtend}, // Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA + {0x0E31, 0x0E31, prExtend}, // Mn THAI CHARACTER MAI HAN-AKAT + {0x0E34, 0x0E3A, prExtend}, // Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU + {0x0E47, 0x0E4E, prExtend}, // Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN + {0x0E50, 0x0E59, prNumeric}, // Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE + {0x0EB1, 0x0EB1, prExtend}, // Mn LAO VOWEL SIGN MAI KAN + {0x0EB4, 0x0EBC, prExtend}, // Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO + {0x0EC8, 0x0ECE, prExtend}, // Mn [7] LAO TONE MAI EK..LAO YAMAKKAN + {0x0ED0, 0x0ED9, prNumeric}, // Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE + {0x0F00, 0x0F00, prALetter}, // Lo TIBETAN SYLLABLE OM + {0x0F18, 0x0F19, prExtend}, // Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS + {0x0F20, 0x0F29, prNumeric}, // Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE + {0x0F35, 0x0F35, prExtend}, // Mn TIBETAN MARK NGAS BZUNG NYI ZLA + {0x0F37, 0x0F37, prExtend}, // Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS + {0x0F39, 0x0F39, prExtend}, // Mn TIBETAN MARK TSA -PHRU + {0x0F3E, 0x0F3F, prExtend}, // Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES + {0x0F40, 0x0F47, prALetter}, // Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA + {0x0F49, 0x0F6C, prALetter}, // Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER RRA + {0x0F71, 0x0F7E, prExtend}, // Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO + {0x0F7F, 0x0F7F, prExtend}, // Mc TIBETAN SIGN RNAM BCAD + {0x0F80, 0x0F84, prExtend}, // Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA + {0x0F86, 0x0F87, prExtend}, // Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS + {0x0F88, 0x0F8C, prALetter}, // Lo [5] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN INVERTED MCHU CAN + {0x0F8D, 0x0F97, prExtend}, // Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA + {0x0F99, 0x0FBC, prExtend}, // Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA + {0x0FC6, 0x0FC6, prExtend}, // Mn TIBETAN SYMBOL PADMA GDAN + {0x102B, 0x102C, prExtend}, // Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA + {0x102D, 0x1030, prExtend}, // Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU + {0x1031, 0x1031, prExtend}, // Mc MYANMAR VOWEL SIGN E + {0x1032, 0x1037, prExtend}, // Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW + {0x1038, 0x1038, prExtend}, // Mc MYANMAR SIGN VISARGA + {0x1039, 0x103A, prExtend}, // Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT + {0x103B, 0x103C, prExtend}, // Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA + {0x103D, 0x103E, prExtend}, // Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA + {0x1040, 0x1049, prNumeric}, // Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE + {0x1056, 0x1057, prExtend}, // Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR + {0x1058, 0x1059, prExtend}, // Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL + {0x105E, 0x1060, prExtend}, // Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA + {0x1062, 0x1064, prExtend}, // Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO + {0x1067, 0x106D, prExtend}, // Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 + {0x1071, 0x1074, prExtend}, // Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE + {0x1082, 0x1082, prExtend}, // Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA + {0x1083, 0x1084, prExtend}, // Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E + {0x1085, 0x1086, prExtend}, // Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y + {0x1087, 0x108C, prExtend}, // Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 + {0x108D, 0x108D, prExtend}, // Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE + {0x108F, 0x108F, prExtend}, // Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 + {0x1090, 0x1099, prNumeric}, // Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR SHAN DIGIT NINE + {0x109A, 0x109C, prExtend}, // Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A + {0x109D, 0x109D, prExtend}, // Mn MYANMAR VOWEL SIGN AITON AI + {0x10A0, 0x10C5, prALetter}, // L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE + {0x10C7, 0x10C7, prALetter}, // L& GEORGIAN CAPITAL LETTER YN + {0x10CD, 0x10CD, prALetter}, // L& GEORGIAN CAPITAL LETTER AEN + {0x10D0, 0x10FA, prALetter}, // L& [43] GEORGIAN LETTER AN..GEORGIAN LETTER AIN + {0x10FC, 0x10FC, prALetter}, // Lm MODIFIER LETTER GEORGIAN NAR + {0x10FD, 0x10FF, prALetter}, // L& [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN + {0x1100, 0x1248, prALetter}, // Lo [329] HANGUL CHOSEONG KIYEOK..ETHIOPIC SYLLABLE QWA + {0x124A, 0x124D, prALetter}, // Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE + {0x1250, 0x1256, prALetter}, // Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO + {0x1258, 0x1258, prALetter}, // Lo ETHIOPIC SYLLABLE QHWA + {0x125A, 0x125D, prALetter}, // Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE + {0x1260, 0x1288, prALetter}, // Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XWA + {0x128A, 0x128D, prALetter}, // Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE + {0x1290, 0x12B0, prALetter}, // Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KWA + {0x12B2, 0x12B5, prALetter}, // Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE + {0x12B8, 0x12BE, prALetter}, // Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO + {0x12C0, 0x12C0, prALetter}, // Lo ETHIOPIC SYLLABLE KXWA + {0x12C2, 0x12C5, prALetter}, // Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE + {0x12C8, 0x12D6, prALetter}, // Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE PHARYNGEAL O + {0x12D8, 0x1310, prALetter}, // Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE GWA + {0x1312, 0x1315, prALetter}, // Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE + {0x1318, 0x135A, prALetter}, // Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE FYA + {0x135D, 0x135F, prExtend}, // Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK + {0x1380, 0x138F, prALetter}, // Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE + {0x13A0, 0x13F5, prALetter}, // L& [86] CHEROKEE LETTER A..CHEROKEE LETTER MV + {0x13F8, 0x13FD, prALetter}, // L& [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV + {0x1401, 0x166C, prALetter}, // Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA + {0x166F, 0x167F, prALetter}, // Lo [17] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS BLACKFOOT W + {0x1680, 0x1680, prWSegSpace}, // Zs OGHAM SPACE MARK + {0x1681, 0x169A, prALetter}, // Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH + {0x16A0, 0x16EA, prALetter}, // Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X + {0x16EE, 0x16F0, prALetter}, // Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL + {0x16F1, 0x16F8, prALetter}, // Lo [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC + {0x1700, 0x1711, prALetter}, // Lo [18] TAGALOG LETTER A..TAGALOG LETTER HA + {0x1712, 0x1714, prExtend}, // Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA + {0x1715, 0x1715, prExtend}, // Mc TAGALOG SIGN PAMUDPOD + {0x171F, 0x1731, prALetter}, // Lo [19] TAGALOG LETTER ARCHAIC RA..HANUNOO LETTER HA + {0x1732, 0x1733, prExtend}, // Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U + {0x1734, 0x1734, prExtend}, // Mc HANUNOO SIGN PAMUDPOD + {0x1740, 0x1751, prALetter}, // Lo [18] BUHID LETTER A..BUHID LETTER HA + {0x1752, 0x1753, prExtend}, // Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U + {0x1760, 0x176C, prALetter}, // Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA + {0x176E, 0x1770, prALetter}, // Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA + {0x1772, 0x1773, prExtend}, // Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U + {0x17B4, 0x17B5, prExtend}, // Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA + {0x17B6, 0x17B6, prExtend}, // Mc KHMER VOWEL SIGN AA + {0x17B7, 0x17BD, prExtend}, // Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA + {0x17BE, 0x17C5, prExtend}, // Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU + {0x17C6, 0x17C6, prExtend}, // Mn KHMER SIGN NIKAHIT + {0x17C7, 0x17C8, prExtend}, // Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU + {0x17C9, 0x17D3, prExtend}, // Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT + {0x17DD, 0x17DD, prExtend}, // Mn KHMER SIGN ATTHACAN + {0x17E0, 0x17E9, prNumeric}, // Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE + {0x180B, 0x180D, prExtend}, // Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE + {0x180E, 0x180E, prFormat}, // Cf MONGOLIAN VOWEL SEPARATOR + {0x180F, 0x180F, prExtend}, // Mn MONGOLIAN FREE VARIATION SELECTOR FOUR + {0x1810, 0x1819, prNumeric}, // Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE + {0x1820, 0x1842, prALetter}, // Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI + {0x1843, 0x1843, prALetter}, // Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN + {0x1844, 0x1878, prALetter}, // Lo [53] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER CHA WITH TWO DOTS + {0x1880, 0x1884, prALetter}, // Lo [5] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER ALI GALI INVERTED UBADAMA + {0x1885, 0x1886, prExtend}, // Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA + {0x1887, 0x18A8, prALetter}, // Lo [34] MONGOLIAN LETTER ALI GALI A..MONGOLIAN LETTER MANCHU ALI GALI BHA + {0x18A9, 0x18A9, prExtend}, // Mn MONGOLIAN LETTER ALI GALI DAGALGA + {0x18AA, 0x18AA, prALetter}, // Lo MONGOLIAN LETTER MANCHU ALI GALI LHA + {0x18B0, 0x18F5, prALetter}, // Lo [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S + {0x1900, 0x191E, prALetter}, // Lo [31] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER TRA + {0x1920, 0x1922, prExtend}, // Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U + {0x1923, 0x1926, prExtend}, // Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU + {0x1927, 0x1928, prExtend}, // Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O + {0x1929, 0x192B, prExtend}, // Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA + {0x1930, 0x1931, prExtend}, // Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA + {0x1932, 0x1932, prExtend}, // Mn LIMBU SMALL LETTER ANUSVARA + {0x1933, 0x1938, prExtend}, // Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA + {0x1939, 0x193B, prExtend}, // Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I + {0x1946, 0x194F, prNumeric}, // Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE + {0x19D0, 0x19D9, prNumeric}, // Nd [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE + {0x1A00, 0x1A16, prALetter}, // Lo [23] BUGINESE LETTER KA..BUGINESE LETTER HA + {0x1A17, 0x1A18, prExtend}, // Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U + {0x1A19, 0x1A1A, prExtend}, // Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O + {0x1A1B, 0x1A1B, prExtend}, // Mn BUGINESE VOWEL SIGN AE + {0x1A55, 0x1A55, prExtend}, // Mc TAI THAM CONSONANT SIGN MEDIAL RA + {0x1A56, 0x1A56, prExtend}, // Mn TAI THAM CONSONANT SIGN MEDIAL LA + {0x1A57, 0x1A57, prExtend}, // Mc TAI THAM CONSONANT SIGN LA TANG LAI + {0x1A58, 0x1A5E, prExtend}, // Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA + {0x1A60, 0x1A60, prExtend}, // Mn TAI THAM SIGN SAKOT + {0x1A61, 0x1A61, prExtend}, // Mc TAI THAM VOWEL SIGN A + {0x1A62, 0x1A62, prExtend}, // Mn TAI THAM VOWEL SIGN MAI SAT + {0x1A63, 0x1A64, prExtend}, // Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA + {0x1A65, 0x1A6C, prExtend}, // Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW + {0x1A6D, 0x1A72, prExtend}, // Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI + {0x1A73, 0x1A7C, prExtend}, // Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN + {0x1A7F, 0x1A7F, prExtend}, // Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT + {0x1A80, 0x1A89, prNumeric}, // Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM HORA DIGIT NINE + {0x1A90, 0x1A99, prNumeric}, // Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE + {0x1AB0, 0x1ABD, prExtend}, // Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW + {0x1ABE, 0x1ABE, prExtend}, // Me COMBINING PARENTHESES OVERLAY + {0x1ABF, 0x1ACE, prExtend}, // Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T + {0x1B00, 0x1B03, prExtend}, // Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG + {0x1B04, 0x1B04, prExtend}, // Mc BALINESE SIGN BISAH + {0x1B05, 0x1B33, prALetter}, // Lo [47] BALINESE LETTER AKARA..BALINESE LETTER HA + {0x1B34, 0x1B34, prExtend}, // Mn BALINESE SIGN REREKAN + {0x1B35, 0x1B35, prExtend}, // Mc BALINESE VOWEL SIGN TEDUNG + {0x1B36, 0x1B3A, prExtend}, // Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA + {0x1B3B, 0x1B3B, prExtend}, // Mc BALINESE VOWEL SIGN RA REPA TEDUNG + {0x1B3C, 0x1B3C, prExtend}, // Mn BALINESE VOWEL SIGN LA LENGA + {0x1B3D, 0x1B41, prExtend}, // Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG + {0x1B42, 0x1B42, prExtend}, // Mn BALINESE VOWEL SIGN PEPET + {0x1B43, 0x1B44, prExtend}, // Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG + {0x1B45, 0x1B4C, prALetter}, // Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA + {0x1B50, 0x1B59, prNumeric}, // Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE + {0x1B6B, 0x1B73, prExtend}, // Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG + {0x1B80, 0x1B81, prExtend}, // Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR + {0x1B82, 0x1B82, prExtend}, // Mc SUNDANESE SIGN PANGWISAD + {0x1B83, 0x1BA0, prALetter}, // Lo [30] SUNDANESE LETTER A..SUNDANESE LETTER HA + {0x1BA1, 0x1BA1, prExtend}, // Mc SUNDANESE CONSONANT SIGN PAMINGKAL + {0x1BA2, 0x1BA5, prExtend}, // Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU + {0x1BA6, 0x1BA7, prExtend}, // Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG + {0x1BA8, 0x1BA9, prExtend}, // Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG + {0x1BAA, 0x1BAA, prExtend}, // Mc SUNDANESE SIGN PAMAAEH + {0x1BAB, 0x1BAD, prExtend}, // Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA + {0x1BAE, 0x1BAF, prALetter}, // Lo [2] SUNDANESE LETTER KHA..SUNDANESE LETTER SYA + {0x1BB0, 0x1BB9, prNumeric}, // Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DIGIT NINE + {0x1BBA, 0x1BE5, prALetter}, // Lo [44] SUNDANESE AVAGRAHA..BATAK LETTER U + {0x1BE6, 0x1BE6, prExtend}, // Mn BATAK SIGN TOMPI + {0x1BE7, 0x1BE7, prExtend}, // Mc BATAK VOWEL SIGN E + {0x1BE8, 0x1BE9, prExtend}, // Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE + {0x1BEA, 0x1BEC, prExtend}, // Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O + {0x1BED, 0x1BED, prExtend}, // Mn BATAK VOWEL SIGN KARO O + {0x1BEE, 0x1BEE, prExtend}, // Mc BATAK VOWEL SIGN U + {0x1BEF, 0x1BF1, prExtend}, // Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H + {0x1BF2, 0x1BF3, prExtend}, // Mc [2] BATAK PANGOLAT..BATAK PANONGONAN + {0x1C00, 0x1C23, prALetter}, // Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A + {0x1C24, 0x1C2B, prExtend}, // Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU + {0x1C2C, 0x1C33, prExtend}, // Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T + {0x1C34, 0x1C35, prExtend}, // Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG + {0x1C36, 0x1C37, prExtend}, // Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA + {0x1C40, 0x1C49, prNumeric}, // Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE + {0x1C4D, 0x1C4F, prALetter}, // Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA + {0x1C50, 0x1C59, prNumeric}, // Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGIT NINE + {0x1C5A, 0x1C77, prALetter}, // Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTER OH + {0x1C78, 0x1C7D, prALetter}, // Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD + {0x1C80, 0x1C88, prALetter}, // L& [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK + {0x1C90, 0x1CBA, prALetter}, // L& [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN + {0x1CBD, 0x1CBF, prALetter}, // L& [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN + {0x1CD0, 0x1CD2, prExtend}, // Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA + {0x1CD4, 0x1CE0, prExtend}, // Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA + {0x1CE1, 0x1CE1, prExtend}, // Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA + {0x1CE2, 0x1CE8, prExtend}, // Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL + {0x1CE9, 0x1CEC, prALetter}, // Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..VEDIC SIGN ANUSVARA VAMAGOMUKHA WITH TAIL + {0x1CED, 0x1CED, prExtend}, // Mn VEDIC SIGN TIRYAK + {0x1CEE, 0x1CF3, prALetter}, // Lo [6] VEDIC SIGN HEXIFORM LONG ANUSVARA..VEDIC SIGN ROTATED ARDHAVISARGA + {0x1CF4, 0x1CF4, prExtend}, // Mn VEDIC TONE CANDRA ABOVE + {0x1CF5, 0x1CF6, prALetter}, // Lo [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN UPADHMANIYA + {0x1CF7, 0x1CF7, prExtend}, // Mc VEDIC SIGN ATIKRAMA + {0x1CF8, 0x1CF9, prExtend}, // Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE + {0x1CFA, 0x1CFA, prALetter}, // Lo VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA + {0x1D00, 0x1D2B, prALetter}, // L& [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL + {0x1D2C, 0x1D6A, prALetter}, // Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI + {0x1D6B, 0x1D77, prALetter}, // L& [13] LATIN SMALL LETTER UE..LATIN SMALL LETTER TURNED G + {0x1D78, 0x1D78, prALetter}, // Lm MODIFIER LETTER CYRILLIC EN + {0x1D79, 0x1D9A, prALetter}, // L& [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK + {0x1D9B, 0x1DBF, prALetter}, // Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA + {0x1DC0, 0x1DFF, prExtend}, // Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW + {0x1E00, 0x1F15, prALetter}, // L& [278] LATIN CAPITAL LETTER A WITH RING BELOW..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA + {0x1F18, 0x1F1D, prALetter}, // L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + {0x1F20, 0x1F45, prALetter}, // L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA + {0x1F48, 0x1F4D, prALetter}, // L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + {0x1F50, 0x1F57, prALetter}, // L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI + {0x1F59, 0x1F59, prALetter}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA + {0x1F5B, 0x1F5B, prALetter}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + {0x1F5D, 0x1F5D, prALetter}, // L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + {0x1F5F, 0x1F7D, prALetter}, // L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA + {0x1F80, 0x1FB4, prALetter}, // L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + {0x1FB6, 0x1FBC, prALetter}, // L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + {0x1FBE, 0x1FBE, prALetter}, // L& GREEK PROSGEGRAMMENI + {0x1FC2, 0x1FC4, prALetter}, // L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + {0x1FC6, 0x1FCC, prALetter}, // L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + {0x1FD0, 0x1FD3, prALetter}, // L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + {0x1FD6, 0x1FDB, prALetter}, // L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA + {0x1FE0, 0x1FEC, prALetter}, // L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA + {0x1FF2, 0x1FF4, prALetter}, // L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + {0x1FF6, 0x1FFC, prALetter}, // L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + {0x2000, 0x2006, prWSegSpace}, // Zs [7] EN QUAD..SIX-PER-EM SPACE + {0x2008, 0x200A, prWSegSpace}, // Zs [3] PUNCTUATION SPACE..HAIR SPACE + {0x200C, 0x200C, prExtend}, // Cf ZERO WIDTH NON-JOINER + {0x200D, 0x200D, prZWJ}, // Cf ZERO WIDTH JOINER + {0x200E, 0x200F, prFormat}, // Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK + {0x2018, 0x2018, prMidNumLet}, // Pi LEFT SINGLE QUOTATION MARK + {0x2019, 0x2019, prMidNumLet}, // Pf RIGHT SINGLE QUOTATION MARK + {0x2024, 0x2024, prMidNumLet}, // Po ONE DOT LEADER + {0x2027, 0x2027, prMidLetter}, // Po HYPHENATION POINT + {0x2028, 0x2028, prNewline}, // Zl LINE SEPARATOR + {0x2029, 0x2029, prNewline}, // Zp PARAGRAPH SEPARATOR + {0x202A, 0x202E, prFormat}, // Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE + {0x202F, 0x202F, prExtendNumLet}, // Zs NARROW NO-BREAK SPACE + {0x203C, 0x203C, prExtendedPictographic}, // E0.6 [1] (โ€ผ๏ธ) double exclamation mark + {0x203F, 0x2040, prExtendNumLet}, // Pc [2] UNDERTIE..CHARACTER TIE + {0x2044, 0x2044, prMidNum}, // Sm FRACTION SLASH + {0x2049, 0x2049, prExtendedPictographic}, // E0.6 [1] (โ‰๏ธ) exclamation question mark + {0x2054, 0x2054, prExtendNumLet}, // Pc INVERTED UNDERTIE + {0x205F, 0x205F, prWSegSpace}, // Zs MEDIUM MATHEMATICAL SPACE + {0x2060, 0x2064, prFormat}, // Cf [5] WORD JOINER..INVISIBLE PLUS + {0x2066, 0x206F, prFormat}, // Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES + {0x2071, 0x2071, prALetter}, // Lm SUPERSCRIPT LATIN SMALL LETTER I + {0x207F, 0x207F, prALetter}, // Lm SUPERSCRIPT LATIN SMALL LETTER N + {0x2090, 0x209C, prALetter}, // Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T + {0x20D0, 0x20DC, prExtend}, // Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE + {0x20DD, 0x20E0, prExtend}, // Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH + {0x20E1, 0x20E1, prExtend}, // Mn COMBINING LEFT RIGHT ARROW ABOVE + {0x20E2, 0x20E4, prExtend}, // Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE + {0x20E5, 0x20F0, prExtend}, // Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE + {0x2102, 0x2102, prALetter}, // L& DOUBLE-STRUCK CAPITAL C + {0x2107, 0x2107, prALetter}, // L& EULER CONSTANT + {0x210A, 0x2113, prALetter}, // L& [10] SCRIPT SMALL G..SCRIPT SMALL L + {0x2115, 0x2115, prALetter}, // L& DOUBLE-STRUCK CAPITAL N + {0x2119, 0x211D, prALetter}, // L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R + {0x2122, 0x2122, prExtendedPictographic}, // E0.6 [1] (โ„ข๏ธ) trade mark + {0x2124, 0x2124, prALetter}, // L& DOUBLE-STRUCK CAPITAL Z + {0x2126, 0x2126, prALetter}, // L& OHM SIGN + {0x2128, 0x2128, prALetter}, // L& BLACK-LETTER CAPITAL Z + {0x212A, 0x212D, prALetter}, // L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C + {0x212F, 0x2134, prALetter}, // L& [6] SCRIPT SMALL E..SCRIPT SMALL O + {0x2135, 0x2138, prALetter}, // Lo [4] ALEF SYMBOL..DALET SYMBOL + {0x2139, 0x2139, prExtendedPictographic}, // E0.6 [1] (โ„น๏ธ) information + {0x2139, 0x2139, prALetter}, // L& INFORMATION SOURCE + {0x213C, 0x213F, prALetter}, // L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI + {0x2145, 0x2149, prALetter}, // L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J + {0x214E, 0x214E, prALetter}, // L& TURNED SMALL F + {0x2160, 0x2182, prALetter}, // Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL TEN THOUSAND + {0x2183, 0x2184, prALetter}, // L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED..LATIN SMALL LETTER REVERSED C + {0x2185, 0x2188, prALetter}, // Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND + {0x2194, 0x2199, prExtendedPictographic}, // E0.6 [6] (โ†”๏ธ..โ†™๏ธ) left-right arrow..down-left arrow + {0x21A9, 0x21AA, prExtendedPictographic}, // E0.6 [2] (โ†ฉ๏ธ..โ†ช๏ธ) right arrow curving left..left arrow curving right + {0x231A, 0x231B, prExtendedPictographic}, // E0.6 [2] (โŒš..โŒ›) watch..hourglass done + {0x2328, 0x2328, prExtendedPictographic}, // E1.0 [1] (โŒจ๏ธ) keyboard + {0x2388, 0x2388, prExtendedPictographic}, // E0.0 [1] (โŽˆ) HELM SYMBOL + {0x23CF, 0x23CF, prExtendedPictographic}, // E1.0 [1] (โ๏ธ) eject button + {0x23E9, 0x23EC, prExtendedPictographic}, // E0.6 [4] (โฉ..โฌ) fast-forward button..fast down button + {0x23ED, 0x23EE, prExtendedPictographic}, // E0.7 [2] (โญ๏ธ..โฎ๏ธ) next track button..last track button + {0x23EF, 0x23EF, prExtendedPictographic}, // E1.0 [1] (โฏ๏ธ) play or pause button + {0x23F0, 0x23F0, prExtendedPictographic}, // E0.6 [1] (โฐ) alarm clock + {0x23F1, 0x23F2, prExtendedPictographic}, // E1.0 [2] (โฑ๏ธ..โฒ๏ธ) stopwatch..timer clock + {0x23F3, 0x23F3, prExtendedPictographic}, // E0.6 [1] (โณ) hourglass not done + {0x23F8, 0x23FA, prExtendedPictographic}, // E0.7 [3] (โธ๏ธ..โบ๏ธ) pause button..record button + {0x24B6, 0x24E9, prALetter}, // So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z + {0x24C2, 0x24C2, prExtendedPictographic}, // E0.6 [1] (โ“‚๏ธ) circled M + {0x25AA, 0x25AB, prExtendedPictographic}, // E0.6 [2] (โ–ช๏ธ..โ–ซ๏ธ) black small square..white small square + {0x25B6, 0x25B6, prExtendedPictographic}, // E0.6 [1] (โ–ถ๏ธ) play button + {0x25C0, 0x25C0, prExtendedPictographic}, // E0.6 [1] (โ—€๏ธ) reverse button + {0x25FB, 0x25FE, prExtendedPictographic}, // E0.6 [4] (โ—ป๏ธ..โ—พ) white medium square..black medium-small square + {0x2600, 0x2601, prExtendedPictographic}, // E0.6 [2] (โ˜€๏ธ..โ˜๏ธ) sun..cloud + {0x2602, 0x2603, prExtendedPictographic}, // E0.7 [2] (โ˜‚๏ธ..โ˜ƒ๏ธ) umbrella..snowman + {0x2604, 0x2604, prExtendedPictographic}, // E1.0 [1] (โ˜„๏ธ) comet + {0x2605, 0x2605, prExtendedPictographic}, // E0.0 [1] (โ˜…) BLACK STAR + {0x2607, 0x260D, prExtendedPictographic}, // E0.0 [7] (โ˜‡..โ˜) LIGHTNING..OPPOSITION + {0x260E, 0x260E, prExtendedPictographic}, // E0.6 [1] (โ˜Ž๏ธ) telephone + {0x260F, 0x2610, prExtendedPictographic}, // E0.0 [2] (โ˜..โ˜) WHITE TELEPHONE..BALLOT BOX + {0x2611, 0x2611, prExtendedPictographic}, // E0.6 [1] (โ˜‘๏ธ) check box with check + {0x2612, 0x2612, prExtendedPictographic}, // E0.0 [1] (โ˜’) BALLOT BOX WITH X + {0x2614, 0x2615, prExtendedPictographic}, // E0.6 [2] (โ˜”..โ˜•) umbrella with rain drops..hot beverage + {0x2616, 0x2617, prExtendedPictographic}, // E0.0 [2] (โ˜–..โ˜—) WHITE SHOGI PIECE..BLACK SHOGI PIECE + {0x2618, 0x2618, prExtendedPictographic}, // E1.0 [1] (โ˜˜๏ธ) shamrock + {0x2619, 0x261C, prExtendedPictographic}, // E0.0 [4] (โ˜™..โ˜œ) REVERSED ROTATED FLORAL HEART BULLET..WHITE LEFT POINTING INDEX + {0x261D, 0x261D, prExtendedPictographic}, // E0.6 [1] (โ˜๏ธ) index pointing up + {0x261E, 0x261F, prExtendedPictographic}, // E0.0 [2] (โ˜ž..โ˜Ÿ) WHITE RIGHT POINTING INDEX..WHITE DOWN POINTING INDEX + {0x2620, 0x2620, prExtendedPictographic}, // E1.0 [1] (โ˜ ๏ธ) skull and crossbones + {0x2621, 0x2621, prExtendedPictographic}, // E0.0 [1] (โ˜ก) CAUTION SIGN + {0x2622, 0x2623, prExtendedPictographic}, // E1.0 [2] (โ˜ข๏ธ..โ˜ฃ๏ธ) radioactive..biohazard + {0x2624, 0x2625, prExtendedPictographic}, // E0.0 [2] (โ˜ค..โ˜ฅ) CADUCEUS..ANKH + {0x2626, 0x2626, prExtendedPictographic}, // E1.0 [1] (โ˜ฆ๏ธ) orthodox cross + {0x2627, 0x2629, prExtendedPictographic}, // E0.0 [3] (โ˜ง..โ˜ฉ) CHI RHO..CROSS OF JERUSALEM + {0x262A, 0x262A, prExtendedPictographic}, // E0.7 [1] (โ˜ช๏ธ) star and crescent + {0x262B, 0x262D, prExtendedPictographic}, // E0.0 [3] (โ˜ซ..โ˜ญ) FARSI SYMBOL..HAMMER AND SICKLE + {0x262E, 0x262E, prExtendedPictographic}, // E1.0 [1] (โ˜ฎ๏ธ) peace symbol + {0x262F, 0x262F, prExtendedPictographic}, // E0.7 [1] (โ˜ฏ๏ธ) yin yang + {0x2630, 0x2637, prExtendedPictographic}, // E0.0 [8] (โ˜ฐ..โ˜ท) TRIGRAM FOR HEAVEN..TRIGRAM FOR EARTH + {0x2638, 0x2639, prExtendedPictographic}, // E0.7 [2] (โ˜ธ๏ธ..โ˜น๏ธ) wheel of dharma..frowning face + {0x263A, 0x263A, prExtendedPictographic}, // E0.6 [1] (โ˜บ๏ธ) smiling face + {0x263B, 0x263F, prExtendedPictographic}, // E0.0 [5] (โ˜ป..โ˜ฟ) BLACK SMILING FACE..MERCURY + {0x2640, 0x2640, prExtendedPictographic}, // E4.0 [1] (โ™€๏ธ) female sign + {0x2641, 0x2641, prExtendedPictographic}, // E0.0 [1] (โ™) EARTH + {0x2642, 0x2642, prExtendedPictographic}, // E4.0 [1] (โ™‚๏ธ) male sign + {0x2643, 0x2647, prExtendedPictographic}, // E0.0 [5] (โ™ƒ..โ™‡) JUPITER..PLUTO + {0x2648, 0x2653, prExtendedPictographic}, // E0.6 [12] (โ™ˆ..โ™“) Aries..Pisces + {0x2654, 0x265E, prExtendedPictographic}, // E0.0 [11] (โ™”..โ™ž) WHITE CHESS KING..BLACK CHESS KNIGHT + {0x265F, 0x265F, prExtendedPictographic}, // E11.0 [1] (โ™Ÿ๏ธ) chess pawn + {0x2660, 0x2660, prExtendedPictographic}, // E0.6 [1] (โ™ ๏ธ) spade suit + {0x2661, 0x2662, prExtendedPictographic}, // E0.0 [2] (โ™ก..โ™ข) WHITE HEART SUIT..WHITE DIAMOND SUIT + {0x2663, 0x2663, prExtendedPictographic}, // E0.6 [1] (โ™ฃ๏ธ) club suit + {0x2664, 0x2664, prExtendedPictographic}, // E0.0 [1] (โ™ค) WHITE SPADE SUIT + {0x2665, 0x2666, prExtendedPictographic}, // E0.6 [2] (โ™ฅ๏ธ..โ™ฆ๏ธ) heart suit..diamond suit + {0x2667, 0x2667, prExtendedPictographic}, // E0.0 [1] (โ™ง) WHITE CLUB SUIT + {0x2668, 0x2668, prExtendedPictographic}, // E0.6 [1] (โ™จ๏ธ) hot springs + {0x2669, 0x267A, prExtendedPictographic}, // E0.0 [18] (โ™ฉ..โ™บ) QUARTER NOTE..RECYCLING SYMBOL FOR GENERIC MATERIALS + {0x267B, 0x267B, prExtendedPictographic}, // E0.6 [1] (โ™ป๏ธ) recycling symbol + {0x267C, 0x267D, prExtendedPictographic}, // E0.0 [2] (โ™ผ..โ™ฝ) RECYCLED PAPER SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL + {0x267E, 0x267E, prExtendedPictographic}, // E11.0 [1] (โ™พ๏ธ) infinity + {0x267F, 0x267F, prExtendedPictographic}, // E0.6 [1] (โ™ฟ) wheelchair symbol + {0x2680, 0x2685, prExtendedPictographic}, // E0.0 [6] (โš€..โš…) DIE FACE-1..DIE FACE-6 + {0x2690, 0x2691, prExtendedPictographic}, // E0.0 [2] (โš..โš‘) WHITE FLAG..BLACK FLAG + {0x2692, 0x2692, prExtendedPictographic}, // E1.0 [1] (โš’๏ธ) hammer and pick + {0x2693, 0x2693, prExtendedPictographic}, // E0.6 [1] (โš“) anchor + {0x2694, 0x2694, prExtendedPictographic}, // E1.0 [1] (โš”๏ธ) crossed swords + {0x2695, 0x2695, prExtendedPictographic}, // E4.0 [1] (โš•๏ธ) medical symbol + {0x2696, 0x2697, prExtendedPictographic}, // E1.0 [2] (โš–๏ธ..โš—๏ธ) balance scale..alembic + {0x2698, 0x2698, prExtendedPictographic}, // E0.0 [1] (โš˜) FLOWER + {0x2699, 0x2699, prExtendedPictographic}, // E1.0 [1] (โš™๏ธ) gear + {0x269A, 0x269A, prExtendedPictographic}, // E0.0 [1] (โšš) STAFF OF HERMES + {0x269B, 0x269C, prExtendedPictographic}, // E1.0 [2] (โš›๏ธ..โšœ๏ธ) atom symbol..fleur-de-lis + {0x269D, 0x269F, prExtendedPictographic}, // E0.0 [3] (โš..โšŸ) OUTLINED WHITE STAR..THREE LINES CONVERGING LEFT + {0x26A0, 0x26A1, prExtendedPictographic}, // E0.6 [2] (โš ๏ธ..โšก) warning..high voltage + {0x26A2, 0x26A6, prExtendedPictographic}, // E0.0 [5] (โšข..โšฆ) DOUBLED FEMALE SIGN..MALE WITH STROKE SIGN + {0x26A7, 0x26A7, prExtendedPictographic}, // E13.0 [1] (โšง๏ธ) transgender symbol + {0x26A8, 0x26A9, prExtendedPictographic}, // E0.0 [2] (โšจ..โšฉ) VERTICAL MALE WITH STROKE SIGN..HORIZONTAL MALE WITH STROKE SIGN + {0x26AA, 0x26AB, prExtendedPictographic}, // E0.6 [2] (โšช..โšซ) white circle..black circle + {0x26AC, 0x26AF, prExtendedPictographic}, // E0.0 [4] (โšฌ..โšฏ) MEDIUM SMALL WHITE CIRCLE..UNMARRIED PARTNERSHIP SYMBOL + {0x26B0, 0x26B1, prExtendedPictographic}, // E1.0 [2] (โšฐ๏ธ..โšฑ๏ธ) coffin..funeral urn + {0x26B2, 0x26BC, prExtendedPictographic}, // E0.0 [11] (โšฒ..โšผ) NEUTER..SESQUIQUADRATE + {0x26BD, 0x26BE, prExtendedPictographic}, // E0.6 [2] (โšฝ..โšพ) soccer ball..baseball + {0x26BF, 0x26C3, prExtendedPictographic}, // E0.0 [5] (โšฟ..โ›ƒ) SQUARED KEY..BLACK DRAUGHTS KING + {0x26C4, 0x26C5, prExtendedPictographic}, // E0.6 [2] (โ›„..โ›…) snowman without snow..sun behind cloud + {0x26C6, 0x26C7, prExtendedPictographic}, // E0.0 [2] (โ›†..โ›‡) RAIN..BLACK SNOWMAN + {0x26C8, 0x26C8, prExtendedPictographic}, // E0.7 [1] (โ›ˆ๏ธ) cloud with lightning and rain + {0x26C9, 0x26CD, prExtendedPictographic}, // E0.0 [5] (โ›‰..โ›) TURNED WHITE SHOGI PIECE..DISABLED CAR + {0x26CE, 0x26CE, prExtendedPictographic}, // E0.6 [1] (โ›Ž) Ophiuchus + {0x26CF, 0x26CF, prExtendedPictographic}, // E0.7 [1] (โ›๏ธ) pick + {0x26D0, 0x26D0, prExtendedPictographic}, // E0.0 [1] (โ›) CAR SLIDING + {0x26D1, 0x26D1, prExtendedPictographic}, // E0.7 [1] (โ›‘๏ธ) rescue workerโ€™s helmet + {0x26D2, 0x26D2, prExtendedPictographic}, // E0.0 [1] (โ›’) CIRCLED CROSSING LANES + {0x26D3, 0x26D3, prExtendedPictographic}, // E0.7 [1] (โ›“๏ธ) chains + {0x26D4, 0x26D4, prExtendedPictographic}, // E0.6 [1] (โ›”) no entry + {0x26D5, 0x26E8, prExtendedPictographic}, // E0.0 [20] (โ›•..โ›จ) ALTERNATE ONE-WAY LEFT WAY TRAFFIC..BLACK CROSS ON SHIELD + {0x26E9, 0x26E9, prExtendedPictographic}, // E0.7 [1] (โ›ฉ๏ธ) shinto shrine + {0x26EA, 0x26EA, prExtendedPictographic}, // E0.6 [1] (โ›ช) church + {0x26EB, 0x26EF, prExtendedPictographic}, // E0.0 [5] (โ›ซ..โ›ฏ) CASTLE..MAP SYMBOL FOR LIGHTHOUSE + {0x26F0, 0x26F1, prExtendedPictographic}, // E0.7 [2] (โ›ฐ๏ธ..โ›ฑ๏ธ) mountain..umbrella on ground + {0x26F2, 0x26F3, prExtendedPictographic}, // E0.6 [2] (โ›ฒ..โ›ณ) fountain..flag in hole + {0x26F4, 0x26F4, prExtendedPictographic}, // E0.7 [1] (โ›ด๏ธ) ferry + {0x26F5, 0x26F5, prExtendedPictographic}, // E0.6 [1] (โ›ต) sailboat + {0x26F6, 0x26F6, prExtendedPictographic}, // E0.0 [1] (โ›ถ) SQUARE FOUR CORNERS + {0x26F7, 0x26F9, prExtendedPictographic}, // E0.7 [3] (โ›ท๏ธ..โ›น๏ธ) skier..person bouncing ball + {0x26FA, 0x26FA, prExtendedPictographic}, // E0.6 [1] (โ›บ) tent + {0x26FB, 0x26FC, prExtendedPictographic}, // E0.0 [2] (โ›ป..โ›ผ) JAPANESE BANK SYMBOL..HEADSTONE GRAVEYARD SYMBOL + {0x26FD, 0x26FD, prExtendedPictographic}, // E0.6 [1] (โ›ฝ) fuel pump + {0x26FE, 0x2701, prExtendedPictographic}, // E0.0 [4] (โ›พ..โœ) CUP ON BLACK SQUARE..UPPER BLADE SCISSORS + {0x2702, 0x2702, prExtendedPictographic}, // E0.6 [1] (โœ‚๏ธ) scissors + {0x2703, 0x2704, prExtendedPictographic}, // E0.0 [2] (โœƒ..โœ„) LOWER BLADE SCISSORS..WHITE SCISSORS + {0x2705, 0x2705, prExtendedPictographic}, // E0.6 [1] (โœ…) check mark button + {0x2708, 0x270C, prExtendedPictographic}, // E0.6 [5] (โœˆ๏ธ..โœŒ๏ธ) airplane..victory hand + {0x270D, 0x270D, prExtendedPictographic}, // E0.7 [1] (โœ๏ธ) writing hand + {0x270E, 0x270E, prExtendedPictographic}, // E0.0 [1] (โœŽ) LOWER RIGHT PENCIL + {0x270F, 0x270F, prExtendedPictographic}, // E0.6 [1] (โœ๏ธ) pencil + {0x2710, 0x2711, prExtendedPictographic}, // E0.0 [2] (โœ..โœ‘) UPPER RIGHT PENCIL..WHITE NIB + {0x2712, 0x2712, prExtendedPictographic}, // E0.6 [1] (โœ’๏ธ) black nib + {0x2714, 0x2714, prExtendedPictographic}, // E0.6 [1] (โœ”๏ธ) check mark + {0x2716, 0x2716, prExtendedPictographic}, // E0.6 [1] (โœ–๏ธ) multiply + {0x271D, 0x271D, prExtendedPictographic}, // E0.7 [1] (โœ๏ธ) latin cross + {0x2721, 0x2721, prExtendedPictographic}, // E0.7 [1] (โœก๏ธ) star of David + {0x2728, 0x2728, prExtendedPictographic}, // E0.6 [1] (โœจ) sparkles + {0x2733, 0x2734, prExtendedPictographic}, // E0.6 [2] (โœณ๏ธ..โœด๏ธ) eight-spoked asterisk..eight-pointed star + {0x2744, 0x2744, prExtendedPictographic}, // E0.6 [1] (โ„๏ธ) snowflake + {0x2747, 0x2747, prExtendedPictographic}, // E0.6 [1] (โ‡๏ธ) sparkle + {0x274C, 0x274C, prExtendedPictographic}, // E0.6 [1] (โŒ) cross mark + {0x274E, 0x274E, prExtendedPictographic}, // E0.6 [1] (โŽ) cross mark button + {0x2753, 0x2755, prExtendedPictographic}, // E0.6 [3] (โ“..โ•) red question mark..white exclamation mark + {0x2757, 0x2757, prExtendedPictographic}, // E0.6 [1] (โ—) red exclamation mark + {0x2763, 0x2763, prExtendedPictographic}, // E1.0 [1] (โฃ๏ธ) heart exclamation + {0x2764, 0x2764, prExtendedPictographic}, // E0.6 [1] (โค๏ธ) red heart + {0x2765, 0x2767, prExtendedPictographic}, // E0.0 [3] (โฅ..โง) ROTATED HEAVY BLACK HEART BULLET..ROTATED FLORAL HEART BULLET + {0x2795, 0x2797, prExtendedPictographic}, // E0.6 [3] (โž•..โž—) plus..divide + {0x27A1, 0x27A1, prExtendedPictographic}, // E0.6 [1] (โžก๏ธ) right arrow + {0x27B0, 0x27B0, prExtendedPictographic}, // E0.6 [1] (โžฐ) curly loop + {0x27BF, 0x27BF, prExtendedPictographic}, // E1.0 [1] (โžฟ) double curly loop + {0x2934, 0x2935, prExtendedPictographic}, // E0.6 [2] (โคด๏ธ..โคต๏ธ) right arrow curving up..right arrow curving down + {0x2B05, 0x2B07, prExtendedPictographic}, // E0.6 [3] (โฌ…๏ธ..โฌ‡๏ธ) left arrow..down arrow + {0x2B1B, 0x2B1C, prExtendedPictographic}, // E0.6 [2] (โฌ›..โฌœ) black large square..white large square + {0x2B50, 0x2B50, prExtendedPictographic}, // E0.6 [1] (โญ) star + {0x2B55, 0x2B55, prExtendedPictographic}, // E0.6 [1] (โญ•) hollow red circle + {0x2C00, 0x2C7B, prALetter}, // L& [124] GLAGOLITIC CAPITAL LETTER AZU..LATIN LETTER SMALL CAPITAL TURNED E + {0x2C7C, 0x2C7D, prALetter}, // Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V + {0x2C7E, 0x2CE4, prALetter}, // L& [103] LATIN CAPITAL LETTER S WITH SWASH TAIL..COPTIC SYMBOL KAI + {0x2CEB, 0x2CEE, prALetter}, // L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC SMALL LETTER CRYPTOGRAMMIC GANGIA + {0x2CEF, 0x2CF1, prExtend}, // Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS + {0x2CF2, 0x2CF3, prALetter}, // L& [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI + {0x2D00, 0x2D25, prALetter}, // L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE + {0x2D27, 0x2D27, prALetter}, // L& GEORGIAN SMALL LETTER YN + {0x2D2D, 0x2D2D, prALetter}, // L& GEORGIAN SMALL LETTER AEN + {0x2D30, 0x2D67, prALetter}, // Lo [56] TIFINAGH LETTER YA..TIFINAGH LETTER YO + {0x2D6F, 0x2D6F, prALetter}, // Lm TIFINAGH MODIFIER LETTER LABIALIZATION MARK + {0x2D7F, 0x2D7F, prExtend}, // Mn TIFINAGH CONSONANT JOINER + {0x2D80, 0x2D96, prALetter}, // Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE + {0x2DA0, 0x2DA6, prALetter}, // Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO + {0x2DA8, 0x2DAE, prALetter}, // Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO + {0x2DB0, 0x2DB6, prALetter}, // Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO + {0x2DB8, 0x2DBE, prALetter}, // Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO + {0x2DC0, 0x2DC6, prALetter}, // Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO + {0x2DC8, 0x2DCE, prALetter}, // Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO + {0x2DD0, 0x2DD6, prALetter}, // Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO + {0x2DD8, 0x2DDE, prALetter}, // Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO + {0x2DE0, 0x2DFF, prExtend}, // Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS + {0x2E2F, 0x2E2F, prALetter}, // Lm VERTICAL TILDE + {0x3000, 0x3000, prWSegSpace}, // Zs IDEOGRAPHIC SPACE + {0x3005, 0x3005, prALetter}, // Lm IDEOGRAPHIC ITERATION MARK + {0x302A, 0x302D, prExtend}, // Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK + {0x302E, 0x302F, prExtend}, // Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK + {0x3030, 0x3030, prExtendedPictographic}, // E0.6 [1] (ใ€ฐ๏ธ) wavy dash + {0x3031, 0x3035, prKatakana}, // Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF + {0x303B, 0x303B, prALetter}, // Lm VERTICAL IDEOGRAPHIC ITERATION MARK + {0x303C, 0x303C, prALetter}, // Lo MASU MARK + {0x303D, 0x303D, prExtendedPictographic}, // E0.6 [1] (ใ€ฝ๏ธ) part alternation mark + {0x3099, 0x309A, prExtend}, // Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x309B, 0x309C, prKatakana}, // Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + {0x30A0, 0x30A0, prKatakana}, // Pd KATAKANA-HIRAGANA DOUBLE HYPHEN + {0x30A1, 0x30FA, prKatakana}, // Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO + {0x30FC, 0x30FE, prKatakana}, // Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK + {0x30FF, 0x30FF, prKatakana}, // Lo KATAKANA DIGRAPH KOTO + {0x3105, 0x312F, prALetter}, // Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN + {0x3131, 0x318E, prALetter}, // Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE + {0x31A0, 0x31BF, prALetter}, // Lo [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH + {0x31F0, 0x31FF, prKatakana}, // Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO + {0x3297, 0x3297, prExtendedPictographic}, // E0.6 [1] (ใŠ—๏ธ) Japanese โ€œcongratulationsโ€ button + {0x3299, 0x3299, prExtendedPictographic}, // E0.6 [1] (ใŠ™๏ธ) Japanese โ€œsecretโ€ button + {0x32D0, 0x32FE, prKatakana}, // So [47] CIRCLED KATAKANA A..CIRCLED KATAKANA WO + {0x3300, 0x3357, prKatakana}, // So [88] SQUARE APAATO..SQUARE WATTO + {0xA000, 0xA014, prALetter}, // Lo [21] YI SYLLABLE IT..YI SYLLABLE E + {0xA015, 0xA015, prALetter}, // Lm YI SYLLABLE WU + {0xA016, 0xA48C, prALetter}, // Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR + {0xA4D0, 0xA4F7, prALetter}, // Lo [40] LISU LETTER BA..LISU LETTER OE + {0xA4F8, 0xA4FD, prALetter}, // Lm [6] LISU LETTER TONE MYA TI..LISU LETTER TONE MYA JEU + {0xA500, 0xA60B, prALetter}, // Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG + {0xA60C, 0xA60C, prALetter}, // Lm VAI SYLLABLE LENGTHENER + {0xA610, 0xA61F, prALetter}, // Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL JONG + {0xA620, 0xA629, prNumeric}, // Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE + {0xA62A, 0xA62B, prALetter}, // Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO + {0xA640, 0xA66D, prALetter}, // L& [46] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER DOUBLE MONOCULAR O + {0xA66E, 0xA66E, prALetter}, // Lo CYRILLIC LETTER MULTIOCULAR O + {0xA66F, 0xA66F, prExtend}, // Mn COMBINING CYRILLIC VZMET + {0xA670, 0xA672, prExtend}, // Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN + {0xA674, 0xA67D, prExtend}, // Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK + {0xA67F, 0xA67F, prALetter}, // Lm CYRILLIC PAYEROK + {0xA680, 0xA69B, prALetter}, // L& [28] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER CROSSED O + {0xA69C, 0xA69D, prALetter}, // Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN + {0xA69E, 0xA69F, prExtend}, // Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E + {0xA6A0, 0xA6E5, prALetter}, // Lo [70] BAMUM LETTER A..BAMUM LETTER KI + {0xA6E6, 0xA6EF, prALetter}, // Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM + {0xA6F0, 0xA6F1, prExtend}, // Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS + {0xA708, 0xA716, prALetter}, // Sk [15] MODIFIER LETTER EXTRA-HIGH DOTTED TONE BAR..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR + {0xA717, 0xA71F, prALetter}, // Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK + {0xA720, 0xA721, prALetter}, // Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE + {0xA722, 0xA76F, prALetter}, // L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CON + {0xA770, 0xA770, prALetter}, // Lm MODIFIER LETTER US + {0xA771, 0xA787, prALetter}, // L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T + {0xA788, 0xA788, prALetter}, // Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT + {0xA789, 0xA78A, prALetter}, // Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN + {0xA78B, 0xA78E, prALetter}, // L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT + {0xA78F, 0xA78F, prALetter}, // Lo LATIN LETTER SINOLOGICAL DOT + {0xA790, 0xA7CA, prALetter}, // L& [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY + {0xA7D0, 0xA7D1, prALetter}, // L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G + {0xA7D3, 0xA7D3, prALetter}, // L& LATIN SMALL LETTER DOUBLE THORN + {0xA7D5, 0xA7D9, prALetter}, // L& [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S + {0xA7F2, 0xA7F4, prALetter}, // Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q + {0xA7F5, 0xA7F6, prALetter}, // L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H + {0xA7F7, 0xA7F7, prALetter}, // Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I + {0xA7F8, 0xA7F9, prALetter}, // Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE + {0xA7FA, 0xA7FA, prALetter}, // L& LATIN LETTER SMALL CAPITAL TURNED M + {0xA7FB, 0xA801, prALetter}, // Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F..SYLOTI NAGRI LETTER I + {0xA802, 0xA802, prExtend}, // Mn SYLOTI NAGRI SIGN DVISVARA + {0xA803, 0xA805, prALetter}, // Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGRI LETTER O + {0xA806, 0xA806, prExtend}, // Mn SYLOTI NAGRI SIGN HASANTA + {0xA807, 0xA80A, prALetter}, // Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAGRI LETTER GHO + {0xA80B, 0xA80B, prExtend}, // Mn SYLOTI NAGRI SIGN ANUSVARA + {0xA80C, 0xA822, prALetter}, // Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAGRI LETTER HO + {0xA823, 0xA824, prExtend}, // Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I + {0xA825, 0xA826, prExtend}, // Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E + {0xA827, 0xA827, prExtend}, // Mc SYLOTI NAGRI VOWEL SIGN OO + {0xA82C, 0xA82C, prExtend}, // Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA + {0xA840, 0xA873, prALetter}, // Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU + {0xA880, 0xA881, prExtend}, // Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA + {0xA882, 0xA8B3, prALetter}, // Lo [50] SAURASHTRA LETTER A..SAURASHTRA LETTER LLA + {0xA8B4, 0xA8C3, prExtend}, // Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU + {0xA8C4, 0xA8C5, prExtend}, // Mn [2] SAURASHTRA SIGN VIRAMA..SAURASHTRA SIGN CANDRABINDU + {0xA8D0, 0xA8D9, prNumeric}, // Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE + {0xA8E0, 0xA8F1, prExtend}, // Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA + {0xA8F2, 0xA8F7, prALetter}, // Lo [6] DEVANAGARI SIGN SPACING CANDRABINDU..DEVANAGARI SIGN CANDRABINDU AVAGRAHA + {0xA8FB, 0xA8FB, prALetter}, // Lo DEVANAGARI HEADSTROKE + {0xA8FD, 0xA8FE, prALetter}, // Lo [2] DEVANAGARI JAIN OM..DEVANAGARI LETTER AY + {0xA8FF, 0xA8FF, prExtend}, // Mn DEVANAGARI VOWEL SIGN AY + {0xA900, 0xA909, prNumeric}, // Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGIT NINE + {0xA90A, 0xA925, prALetter}, // Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTER OO + {0xA926, 0xA92D, prExtend}, // Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU + {0xA930, 0xA946, prALetter}, // Lo [23] REJANG LETTER KA..REJANG LETTER A + {0xA947, 0xA951, prExtend}, // Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R + {0xA952, 0xA953, prExtend}, // Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA + {0xA960, 0xA97C, prALetter}, // Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH + {0xA980, 0xA982, prExtend}, // Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR + {0xA983, 0xA983, prExtend}, // Mc JAVANESE SIGN WIGNYAN + {0xA984, 0xA9B2, prALetter}, // Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA + {0xA9B3, 0xA9B3, prExtend}, // Mn JAVANESE SIGN CECAK TELU + {0xA9B4, 0xA9B5, prExtend}, // Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG + {0xA9B6, 0xA9B9, prExtend}, // Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT + {0xA9BA, 0xA9BB, prExtend}, // Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE + {0xA9BC, 0xA9BD, prExtend}, // Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET + {0xA9BE, 0xA9C0, prExtend}, // Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON + {0xA9CF, 0xA9CF, prALetter}, // Lm JAVANESE PANGRANGKEP + {0xA9D0, 0xA9D9, prNumeric}, // Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE + {0xA9E5, 0xA9E5, prExtend}, // Mn MYANMAR SIGN SHAN SAW + {0xA9F0, 0xA9F9, prNumeric}, // Nd [10] MYANMAR TAI LAING DIGIT ZERO..MYANMAR TAI LAING DIGIT NINE + {0xAA00, 0xAA28, prALetter}, // Lo [41] CHAM LETTER A..CHAM LETTER HA + {0xAA29, 0xAA2E, prExtend}, // Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE + {0xAA2F, 0xAA30, prExtend}, // Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI + {0xAA31, 0xAA32, prExtend}, // Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE + {0xAA33, 0xAA34, prExtend}, // Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA + {0xAA35, 0xAA36, prExtend}, // Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA + {0xAA40, 0xAA42, prALetter}, // Lo [3] CHAM LETTER FINAL K..CHAM LETTER FINAL NG + {0xAA43, 0xAA43, prExtend}, // Mn CHAM CONSONANT SIGN FINAL NG + {0xAA44, 0xAA4B, prALetter}, // Lo [8] CHAM LETTER FINAL CH..CHAM LETTER FINAL SS + {0xAA4C, 0xAA4C, prExtend}, // Mn CHAM CONSONANT SIGN FINAL M + {0xAA4D, 0xAA4D, prExtend}, // Mc CHAM CONSONANT SIGN FINAL H + {0xAA50, 0xAA59, prNumeric}, // Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE + {0xAA7B, 0xAA7B, prExtend}, // Mc MYANMAR SIGN PAO KAREN TONE + {0xAA7C, 0xAA7C, prExtend}, // Mn MYANMAR SIGN TAI LAING TONE-2 + {0xAA7D, 0xAA7D, prExtend}, // Mc MYANMAR SIGN TAI LAING TONE-5 + {0xAAB0, 0xAAB0, prExtend}, // Mn TAI VIET MAI KANG + {0xAAB2, 0xAAB4, prExtend}, // Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U + {0xAAB7, 0xAAB8, prExtend}, // Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA + {0xAABE, 0xAABF, prExtend}, // Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK + {0xAAC1, 0xAAC1, prExtend}, // Mn TAI VIET TONE MAI THO + {0xAAE0, 0xAAEA, prALetter}, // Lo [11] MEETEI MAYEK LETTER E..MEETEI MAYEK LETTER SSA + {0xAAEB, 0xAAEB, prExtend}, // Mc MEETEI MAYEK VOWEL SIGN II + {0xAAEC, 0xAAED, prExtend}, // Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI + {0xAAEE, 0xAAEF, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU + {0xAAF2, 0xAAF2, prALetter}, // Lo MEETEI MAYEK ANJI + {0xAAF3, 0xAAF4, prALetter}, // Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK + {0xAAF5, 0xAAF5, prExtend}, // Mc MEETEI MAYEK VOWEL SIGN VISARGA + {0xAAF6, 0xAAF6, prExtend}, // Mn MEETEI MAYEK VIRAMA + {0xAB01, 0xAB06, prALetter}, // Lo [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO + {0xAB09, 0xAB0E, prALetter}, // Lo [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO + {0xAB11, 0xAB16, prALetter}, // Lo [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO + {0xAB20, 0xAB26, prALetter}, // Lo [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO + {0xAB28, 0xAB2E, prALetter}, // Lo [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO + {0xAB30, 0xAB5A, prALetter}, // L& [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG + {0xAB5B, 0xAB5B, prALetter}, // Sk MODIFIER BREVE WITH INVERTED BREVE + {0xAB5C, 0xAB5F, prALetter}, // Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK + {0xAB60, 0xAB68, prALetter}, // L& [9] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE + {0xAB69, 0xAB69, prALetter}, // Lm MODIFIER LETTER SMALL TURNED W + {0xAB70, 0xABBF, prALetter}, // L& [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA + {0xABC0, 0xABE2, prALetter}, // Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MAYEK LETTER I LONSUM + {0xABE3, 0xABE4, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP + {0xABE5, 0xABE5, prExtend}, // Mn MEETEI MAYEK VOWEL SIGN ANAP + {0xABE6, 0xABE7, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP + {0xABE8, 0xABE8, prExtend}, // Mn MEETEI MAYEK VOWEL SIGN UNAP + {0xABE9, 0xABEA, prExtend}, // Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG + {0xABEC, 0xABEC, prExtend}, // Mc MEETEI MAYEK LUM IYEK + {0xABED, 0xABED, prExtend}, // Mn MEETEI MAYEK APUN IYEK + {0xABF0, 0xABF9, prNumeric}, // Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE + {0xAC00, 0xD7A3, prALetter}, // Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH + {0xD7B0, 0xD7C6, prALetter}, // Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E + {0xD7CB, 0xD7FB, prALetter}, // Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH + {0xFB00, 0xFB06, prALetter}, // L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST + {0xFB13, 0xFB17, prALetter}, // L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH + {0xFB1D, 0xFB1D, prHebrewLetter}, // Lo HEBREW LETTER YOD WITH HIRIQ + {0xFB1E, 0xFB1E, prExtend}, // Mn HEBREW POINT JUDEO-SPANISH VARIKA + {0xFB1F, 0xFB28, prHebrewLetter}, // Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV + {0xFB2A, 0xFB36, prHebrewLetter}, // Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH + {0xFB38, 0xFB3C, prHebrewLetter}, // Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH + {0xFB3E, 0xFB3E, prHebrewLetter}, // Lo HEBREW LETTER MEM WITH DAGESH + {0xFB40, 0xFB41, prHebrewLetter}, // Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH + {0xFB43, 0xFB44, prHebrewLetter}, // Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH + {0xFB46, 0xFB4F, prHebrewLetter}, // Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATURE ALEF LAMED + {0xFB50, 0xFBB1, prALetter}, // Lo [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM + {0xFBD3, 0xFD3D, prALetter}, // Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM + {0xFD50, 0xFD8F, prALetter}, // Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM + {0xFD92, 0xFDC7, prALetter}, // Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM + {0xFDF0, 0xFDFB, prALetter}, // Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU + {0xFE00, 0xFE0F, prExtend}, // Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 + {0xFE10, 0xFE10, prMidNum}, // Po PRESENTATION FORM FOR VERTICAL COMMA + {0xFE13, 0xFE13, prMidLetter}, // Po PRESENTATION FORM FOR VERTICAL COLON + {0xFE14, 0xFE14, prMidNum}, // Po PRESENTATION FORM FOR VERTICAL SEMICOLON + {0xFE20, 0xFE2F, prExtend}, // Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + {0xFE33, 0xFE34, prExtendNumLet}, // Pc [2] PRESENTATION FORM FOR VERTICAL LOW LINE..PRESENTATION FORM FOR VERTICAL WAVY LOW LINE + {0xFE4D, 0xFE4F, prExtendNumLet}, // Pc [3] DASHED LOW LINE..WAVY LOW LINE + {0xFE50, 0xFE50, prMidNum}, // Po SMALL COMMA + {0xFE52, 0xFE52, prMidNumLet}, // Po SMALL FULL STOP + {0xFE54, 0xFE54, prMidNum}, // Po SMALL SEMICOLON + {0xFE55, 0xFE55, prMidLetter}, // Po SMALL COLON + {0xFE70, 0xFE74, prALetter}, // Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM + {0xFE76, 0xFEFC, prALetter}, // Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM + {0xFEFF, 0xFEFF, prFormat}, // Cf ZERO WIDTH NO-BREAK SPACE + {0xFF07, 0xFF07, prMidNumLet}, // Po FULLWIDTH APOSTROPHE + {0xFF0C, 0xFF0C, prMidNum}, // Po FULLWIDTH COMMA + {0xFF0E, 0xFF0E, prMidNumLet}, // Po FULLWIDTH FULL STOP + {0xFF10, 0xFF19, prNumeric}, // Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE + {0xFF1A, 0xFF1A, prMidLetter}, // Po FULLWIDTH COLON + {0xFF1B, 0xFF1B, prMidNum}, // Po FULLWIDTH SEMICOLON + {0xFF21, 0xFF3A, prALetter}, // L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z + {0xFF3F, 0xFF3F, prExtendNumLet}, // Pc FULLWIDTH LOW LINE + {0xFF41, 0xFF5A, prALetter}, // L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z + {0xFF66, 0xFF6F, prKatakana}, // Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU + {0xFF70, 0xFF70, prKatakana}, // Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK + {0xFF71, 0xFF9D, prKatakana}, // Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N + {0xFF9E, 0xFF9F, prExtend}, // Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK + {0xFFA0, 0xFFBE, prALetter}, // Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH + {0xFFC2, 0xFFC7, prALetter}, // Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E + {0xFFCA, 0xFFCF, prALetter}, // Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE + {0xFFD2, 0xFFD7, prALetter}, // Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU + {0xFFDA, 0xFFDC, prALetter}, // Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I + {0xFFF9, 0xFFFB, prFormat}, // Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR + {0x10000, 0x1000B, prALetter}, // Lo [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE + {0x1000D, 0x10026, prALetter}, // Lo [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO + {0x10028, 0x1003A, prALetter}, // Lo [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO + {0x1003C, 0x1003D, prALetter}, // Lo [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE + {0x1003F, 0x1004D, prALetter}, // Lo [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO + {0x10050, 0x1005D, prALetter}, // Lo [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 + {0x10080, 0x100FA, prALetter}, // Lo [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 + {0x10140, 0x10174, prALetter}, // Nl [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS + {0x101FD, 0x101FD, prExtend}, // Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE + {0x10280, 0x1029C, prALetter}, // Lo [29] LYCIAN LETTER A..LYCIAN LETTER X + {0x102A0, 0x102D0, prALetter}, // Lo [49] CARIAN LETTER A..CARIAN LETTER UUU3 + {0x102E0, 0x102E0, prExtend}, // Mn COPTIC EPACT THOUSANDS MARK + {0x10300, 0x1031F, prALetter}, // Lo [32] OLD ITALIC LETTER A..OLD ITALIC LETTER ESS + {0x1032D, 0x10340, prALetter}, // Lo [20] OLD ITALIC LETTER YE..GOTHIC LETTER PAIRTHRA + {0x10341, 0x10341, prALetter}, // Nl GOTHIC LETTER NINETY + {0x10342, 0x10349, prALetter}, // Lo [8] GOTHIC LETTER RAIDA..GOTHIC LETTER OTHAL + {0x1034A, 0x1034A, prALetter}, // Nl GOTHIC LETTER NINE HUNDRED + {0x10350, 0x10375, prALetter}, // Lo [38] OLD PERMIC LETTER AN..OLD PERMIC LETTER IA + {0x10376, 0x1037A, prExtend}, // Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII + {0x10380, 0x1039D, prALetter}, // Lo [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU + {0x103A0, 0x103C3, prALetter}, // Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA + {0x103C8, 0x103CF, prALetter}, // Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH + {0x103D1, 0x103D5, prALetter}, // Nl [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED + {0x10400, 0x1044F, prALetter}, // L& [80] DESERET CAPITAL LETTER LONG I..DESERET SMALL LETTER EW + {0x10450, 0x1049D, prALetter}, // Lo [78] SHAVIAN LETTER PEEP..OSMANYA LETTER OO + {0x104A0, 0x104A9, prNumeric}, // Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE + {0x104B0, 0x104D3, prALetter}, // L& [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA + {0x104D8, 0x104FB, prALetter}, // L& [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA + {0x10500, 0x10527, prALetter}, // Lo [40] ELBASAN LETTER A..ELBASAN LETTER KHE + {0x10530, 0x10563, prALetter}, // Lo [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW + {0x10570, 0x1057A, prALetter}, // L& [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA + {0x1057C, 0x1058A, prALetter}, // L& [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE + {0x1058C, 0x10592, prALetter}, // L& [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE + {0x10594, 0x10595, prALetter}, // L& [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE + {0x10597, 0x105A1, prALetter}, // L& [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA + {0x105A3, 0x105B1, prALetter}, // L& [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE + {0x105B3, 0x105B9, prALetter}, // L& [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE + {0x105BB, 0x105BC, prALetter}, // L& [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE + {0x10600, 0x10736, prALetter}, // Lo [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 + {0x10740, 0x10755, prALetter}, // Lo [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE + {0x10760, 0x10767, prALetter}, // Lo [8] LINEAR A SIGN A800..LINEAR A SIGN A807 + {0x10780, 0x10785, prALetter}, // Lm [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK + {0x10787, 0x107B0, prALetter}, // Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK + {0x107B2, 0x107BA, prALetter}, // Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL + {0x10800, 0x10805, prALetter}, // Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA + {0x10808, 0x10808, prALetter}, // Lo CYPRIOT SYLLABLE JO + {0x1080A, 0x10835, prALetter}, // Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO + {0x10837, 0x10838, prALetter}, // Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE + {0x1083C, 0x1083C, prALetter}, // Lo CYPRIOT SYLLABLE ZA + {0x1083F, 0x10855, prALetter}, // Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL ARAMAIC LETTER TAW + {0x10860, 0x10876, prALetter}, // Lo [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW + {0x10880, 0x1089E, prALetter}, // Lo [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW + {0x108E0, 0x108F2, prALetter}, // Lo [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH + {0x108F4, 0x108F5, prALetter}, // Lo [2] HATRAN LETTER SHIN..HATRAN LETTER TAW + {0x10900, 0x10915, prALetter}, // Lo [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU + {0x10920, 0x10939, prALetter}, // Lo [26] LYDIAN LETTER A..LYDIAN LETTER C + {0x10980, 0x109B7, prALetter}, // Lo [56] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC CURSIVE LETTER DA + {0x109BE, 0x109BF, prALetter}, // Lo [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN + {0x10A00, 0x10A00, prALetter}, // Lo KHAROSHTHI LETTER A + {0x10A01, 0x10A03, prExtend}, // Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R + {0x10A05, 0x10A06, prExtend}, // Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O + {0x10A0C, 0x10A0F, prExtend}, // Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA + {0x10A10, 0x10A13, prALetter}, // Lo [4] KHAROSHTHI LETTER KA..KHAROSHTHI LETTER GHA + {0x10A15, 0x10A17, prALetter}, // Lo [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA + {0x10A19, 0x10A35, prALetter}, // Lo [29] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER VHA + {0x10A38, 0x10A3A, prExtend}, // Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW + {0x10A3F, 0x10A3F, prExtend}, // Mn KHAROSHTHI VIRAMA + {0x10A60, 0x10A7C, prALetter}, // Lo [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH + {0x10A80, 0x10A9C, prALetter}, // Lo [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH + {0x10AC0, 0x10AC7, prALetter}, // Lo [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW + {0x10AC9, 0x10AE4, prALetter}, // Lo [28] MANICHAEAN LETTER ZAYIN..MANICHAEAN LETTER TAW + {0x10AE5, 0x10AE6, prExtend}, // Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW + {0x10B00, 0x10B35, prALetter}, // Lo [54] AVESTAN LETTER A..AVESTAN LETTER HE + {0x10B40, 0x10B55, prALetter}, // Lo [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW + {0x10B60, 0x10B72, prALetter}, // Lo [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW + {0x10B80, 0x10B91, prALetter}, // Lo [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW + {0x10C00, 0x10C48, prALetter}, // Lo [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH + {0x10C80, 0x10CB2, prALetter}, // L& [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US + {0x10CC0, 0x10CF2, prALetter}, // L& [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US + {0x10D00, 0x10D23, prALetter}, // Lo [36] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA MARK NA KHONNA + {0x10D24, 0x10D27, prExtend}, // Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI + {0x10D30, 0x10D39, prNumeric}, // Nd [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE + {0x10E80, 0x10EA9, prALetter}, // Lo [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET + {0x10EAB, 0x10EAC, prExtend}, // Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK + {0x10EB0, 0x10EB1, prALetter}, // Lo [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE + {0x10EFD, 0x10EFF, prExtend}, // Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA + {0x10F00, 0x10F1C, prALetter}, // Lo [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL + {0x10F27, 0x10F27, prALetter}, // Lo OLD SOGDIAN LIGATURE AYIN-DALETH + {0x10F30, 0x10F45, prALetter}, // Lo [22] SOGDIAN LETTER ALEPH..SOGDIAN INDEPENDENT SHIN + {0x10F46, 0x10F50, prExtend}, // Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW + {0x10F70, 0x10F81, prALetter}, // Lo [18] OLD UYGHUR LETTER ALEPH..OLD UYGHUR LETTER LESH + {0x10F82, 0x10F85, prExtend}, // Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW + {0x10FB0, 0x10FC4, prALetter}, // Lo [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW + {0x10FE0, 0x10FF6, prALetter}, // Lo [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH + {0x11000, 0x11000, prExtend}, // Mc BRAHMI SIGN CANDRABINDU + {0x11001, 0x11001, prExtend}, // Mn BRAHMI SIGN ANUSVARA + {0x11002, 0x11002, prExtend}, // Mc BRAHMI SIGN VISARGA + {0x11003, 0x11037, prALetter}, // Lo [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI LETTER OLD TAMIL NNNA + {0x11038, 0x11046, prExtend}, // Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA + {0x11066, 0x1106F, prNumeric}, // Nd [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE + {0x11070, 0x11070, prExtend}, // Mn BRAHMI SIGN OLD TAMIL VIRAMA + {0x11071, 0x11072, prALetter}, // Lo [2] BRAHMI LETTER OLD TAMIL SHORT E..BRAHMI LETTER OLD TAMIL SHORT O + {0x11073, 0x11074, prExtend}, // Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O + {0x11075, 0x11075, prALetter}, // Lo BRAHMI LETTER OLD TAMIL LLA + {0x1107F, 0x11081, prExtend}, // Mn [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA + {0x11082, 0x11082, prExtend}, // Mc KAITHI SIGN VISARGA + {0x11083, 0x110AF, prALetter}, // Lo [45] KAITHI LETTER A..KAITHI LETTER HA + {0x110B0, 0x110B2, prExtend}, // Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II + {0x110B3, 0x110B6, prExtend}, // Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI + {0x110B7, 0x110B8, prExtend}, // Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU + {0x110B9, 0x110BA, prExtend}, // Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA + {0x110BD, 0x110BD, prFormat}, // Cf KAITHI NUMBER SIGN + {0x110C2, 0x110C2, prExtend}, // Mn KAITHI VOWEL SIGN VOCALIC R + {0x110CD, 0x110CD, prFormat}, // Cf KAITHI NUMBER SIGN ABOVE + {0x110D0, 0x110E8, prALetter}, // Lo [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE + {0x110F0, 0x110F9, prNumeric}, // Nd [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE + {0x11100, 0x11102, prExtend}, // Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA + {0x11103, 0x11126, prALetter}, // Lo [36] CHAKMA LETTER AA..CHAKMA LETTER HAA + {0x11127, 0x1112B, prExtend}, // Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU + {0x1112C, 0x1112C, prExtend}, // Mc CHAKMA VOWEL SIGN E + {0x1112D, 0x11134, prExtend}, // Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA + {0x11136, 0x1113F, prNumeric}, // Nd [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE + {0x11144, 0x11144, prALetter}, // Lo CHAKMA LETTER LHAA + {0x11145, 0x11146, prExtend}, // Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI + {0x11147, 0x11147, prALetter}, // Lo CHAKMA LETTER VAA + {0x11150, 0x11172, prALetter}, // Lo [35] MAHAJANI LETTER A..MAHAJANI LETTER RRA + {0x11173, 0x11173, prExtend}, // Mn MAHAJANI SIGN NUKTA + {0x11176, 0x11176, prALetter}, // Lo MAHAJANI LIGATURE SHRI + {0x11180, 0x11181, prExtend}, // Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA + {0x11182, 0x11182, prExtend}, // Mc SHARADA SIGN VISARGA + {0x11183, 0x111B2, prALetter}, // Lo [48] SHARADA LETTER A..SHARADA LETTER HA + {0x111B3, 0x111B5, prExtend}, // Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II + {0x111B6, 0x111BE, prExtend}, // Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O + {0x111BF, 0x111C0, prExtend}, // Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA + {0x111C1, 0x111C4, prALetter}, // Lo [4] SHARADA SIGN AVAGRAHA..SHARADA OM + {0x111C9, 0x111CC, prExtend}, // Mn [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK + {0x111CE, 0x111CE, prExtend}, // Mc SHARADA VOWEL SIGN PRISHTHAMATRA E + {0x111CF, 0x111CF, prExtend}, // Mn SHARADA SIGN INVERTED CANDRABINDU + {0x111D0, 0x111D9, prNumeric}, // Nd [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE + {0x111DA, 0x111DA, prALetter}, // Lo SHARADA EKAM + {0x111DC, 0x111DC, prALetter}, // Lo SHARADA HEADSTROKE + {0x11200, 0x11211, prALetter}, // Lo [18] KHOJKI LETTER A..KHOJKI LETTER JJA + {0x11213, 0x1122B, prALetter}, // Lo [25] KHOJKI LETTER NYA..KHOJKI LETTER LLA + {0x1122C, 0x1122E, prExtend}, // Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II + {0x1122F, 0x11231, prExtend}, // Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI + {0x11232, 0x11233, prExtend}, // Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU + {0x11234, 0x11234, prExtend}, // Mn KHOJKI SIGN ANUSVARA + {0x11235, 0x11235, prExtend}, // Mc KHOJKI SIGN VIRAMA + {0x11236, 0x11237, prExtend}, // Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA + {0x1123E, 0x1123E, prExtend}, // Mn KHOJKI SIGN SUKUN + {0x1123F, 0x11240, prALetter}, // Lo [2] KHOJKI LETTER QA..KHOJKI LETTER SHORT I + {0x11241, 0x11241, prExtend}, // Mn KHOJKI VOWEL SIGN VOCALIC R + {0x11280, 0x11286, prALetter}, // Lo [7] MULTANI LETTER A..MULTANI LETTER GA + {0x11288, 0x11288, prALetter}, // Lo MULTANI LETTER GHA + {0x1128A, 0x1128D, prALetter}, // Lo [4] MULTANI LETTER CA..MULTANI LETTER JJA + {0x1128F, 0x1129D, prALetter}, // Lo [15] MULTANI LETTER NYA..MULTANI LETTER BA + {0x1129F, 0x112A8, prALetter}, // Lo [10] MULTANI LETTER BHA..MULTANI LETTER RHA + {0x112B0, 0x112DE, prALetter}, // Lo [47] KHUDAWADI LETTER A..KHUDAWADI LETTER HA + {0x112DF, 0x112DF, prExtend}, // Mn KHUDAWADI SIGN ANUSVARA + {0x112E0, 0x112E2, prExtend}, // Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II + {0x112E3, 0x112EA, prExtend}, // Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA + {0x112F0, 0x112F9, prNumeric}, // Nd [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE + {0x11300, 0x11301, prExtend}, // Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU + {0x11302, 0x11303, prExtend}, // Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA + {0x11305, 0x1130C, prALetter}, // Lo [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L + {0x1130F, 0x11310, prALetter}, // Lo [2] GRANTHA LETTER EE..GRANTHA LETTER AI + {0x11313, 0x11328, prALetter}, // Lo [22] GRANTHA LETTER OO..GRANTHA LETTER NA + {0x1132A, 0x11330, prALetter}, // Lo [7] GRANTHA LETTER PA..GRANTHA LETTER RA + {0x11332, 0x11333, prALetter}, // Lo [2] GRANTHA LETTER LA..GRANTHA LETTER LLA + {0x11335, 0x11339, prALetter}, // Lo [5] GRANTHA LETTER VA..GRANTHA LETTER HA + {0x1133B, 0x1133C, prExtend}, // Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA + {0x1133D, 0x1133D, prALetter}, // Lo GRANTHA SIGN AVAGRAHA + {0x1133E, 0x1133F, prExtend}, // Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I + {0x11340, 0x11340, prExtend}, // Mn GRANTHA VOWEL SIGN II + {0x11341, 0x11344, prExtend}, // Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR + {0x11347, 0x11348, prExtend}, // Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI + {0x1134B, 0x1134D, prExtend}, // Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA + {0x11350, 0x11350, prALetter}, // Lo GRANTHA OM + {0x11357, 0x11357, prExtend}, // Mc GRANTHA AU LENGTH MARK + {0x1135D, 0x11361, prALetter}, // Lo [5] GRANTHA SIGN PLUTA..GRANTHA LETTER VOCALIC LL + {0x11362, 0x11363, prExtend}, // Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL + {0x11366, 0x1136C, prExtend}, // Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX + {0x11370, 0x11374, prExtend}, // Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA + {0x11400, 0x11434, prALetter}, // Lo [53] NEWA LETTER A..NEWA LETTER HA + {0x11435, 0x11437, prExtend}, // Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II + {0x11438, 0x1143F, prExtend}, // Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI + {0x11440, 0x11441, prExtend}, // Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU + {0x11442, 0x11444, prExtend}, // Mn [3] NEWA SIGN VIRAMA..NEWA SIGN ANUSVARA + {0x11445, 0x11445, prExtend}, // Mc NEWA SIGN VISARGA + {0x11446, 0x11446, prExtend}, // Mn NEWA SIGN NUKTA + {0x11447, 0x1144A, prALetter}, // Lo [4] NEWA SIGN AVAGRAHA..NEWA SIDDHI + {0x11450, 0x11459, prNumeric}, // Nd [10] NEWA DIGIT ZERO..NEWA DIGIT NINE + {0x1145E, 0x1145E, prExtend}, // Mn NEWA SANDHI MARK + {0x1145F, 0x11461, prALetter}, // Lo [3] NEWA LETTER VEDIC ANUSVARA..NEWA SIGN UPADHMANIYA + {0x11480, 0x114AF, prALetter}, // Lo [48] TIRHUTA ANJI..TIRHUTA LETTER HA + {0x114B0, 0x114B2, prExtend}, // Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II + {0x114B3, 0x114B8, prExtend}, // Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL + {0x114B9, 0x114B9, prExtend}, // Mc TIRHUTA VOWEL SIGN E + {0x114BA, 0x114BA, prExtend}, // Mn TIRHUTA VOWEL SIGN SHORT E + {0x114BB, 0x114BE, prExtend}, // Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU + {0x114BF, 0x114C0, prExtend}, // Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA + {0x114C1, 0x114C1, prExtend}, // Mc TIRHUTA SIGN VISARGA + {0x114C2, 0x114C3, prExtend}, // Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA + {0x114C4, 0x114C5, prALetter}, // Lo [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA GVANG + {0x114C7, 0x114C7, prALetter}, // Lo TIRHUTA OM + {0x114D0, 0x114D9, prNumeric}, // Nd [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE + {0x11580, 0x115AE, prALetter}, // Lo [47] SIDDHAM LETTER A..SIDDHAM LETTER HA + {0x115AF, 0x115B1, prExtend}, // Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II + {0x115B2, 0x115B5, prExtend}, // Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR + {0x115B8, 0x115BB, prExtend}, // Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU + {0x115BC, 0x115BD, prExtend}, // Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA + {0x115BE, 0x115BE, prExtend}, // Mc SIDDHAM SIGN VISARGA + {0x115BF, 0x115C0, prExtend}, // Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA + {0x115D8, 0x115DB, prALetter}, // Lo [4] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM LETTER ALTERNATE U + {0x115DC, 0x115DD, prExtend}, // Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU + {0x11600, 0x1162F, prALetter}, // Lo [48] MODI LETTER A..MODI LETTER LLA + {0x11630, 0x11632, prExtend}, // Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II + {0x11633, 0x1163A, prExtend}, // Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI + {0x1163B, 0x1163C, prExtend}, // Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU + {0x1163D, 0x1163D, prExtend}, // Mn MODI SIGN ANUSVARA + {0x1163E, 0x1163E, prExtend}, // Mc MODI SIGN VISARGA + {0x1163F, 0x11640, prExtend}, // Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA + {0x11644, 0x11644, prALetter}, // Lo MODI SIGN HUVA + {0x11650, 0x11659, prNumeric}, // Nd [10] MODI DIGIT ZERO..MODI DIGIT NINE + {0x11680, 0x116AA, prALetter}, // Lo [43] TAKRI LETTER A..TAKRI LETTER RRA + {0x116AB, 0x116AB, prExtend}, // Mn TAKRI SIGN ANUSVARA + {0x116AC, 0x116AC, prExtend}, // Mc TAKRI SIGN VISARGA + {0x116AD, 0x116AD, prExtend}, // Mn TAKRI VOWEL SIGN AA + {0x116AE, 0x116AF, prExtend}, // Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II + {0x116B0, 0x116B5, prExtend}, // Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU + {0x116B6, 0x116B6, prExtend}, // Mc TAKRI SIGN VIRAMA + {0x116B7, 0x116B7, prExtend}, // Mn TAKRI SIGN NUKTA + {0x116B8, 0x116B8, prALetter}, // Lo TAKRI LETTER ARCHAIC KHA + {0x116C0, 0x116C9, prNumeric}, // Nd [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE + {0x1171D, 0x1171F, prExtend}, // Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA + {0x11720, 0x11721, prExtend}, // Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA + {0x11722, 0x11725, prExtend}, // Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU + {0x11726, 0x11726, prExtend}, // Mc AHOM VOWEL SIGN E + {0x11727, 0x1172B, prExtend}, // Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER + {0x11730, 0x11739, prNumeric}, // Nd [10] AHOM DIGIT ZERO..AHOM DIGIT NINE + {0x11800, 0x1182B, prALetter}, // Lo [44] DOGRA LETTER A..DOGRA LETTER RRA + {0x1182C, 0x1182E, prExtend}, // Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II + {0x1182F, 0x11837, prExtend}, // Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA + {0x11838, 0x11838, prExtend}, // Mc DOGRA SIGN VISARGA + {0x11839, 0x1183A, prExtend}, // Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA + {0x118A0, 0x118DF, prALetter}, // L& [64] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI SMALL LETTER VIYO + {0x118E0, 0x118E9, prNumeric}, // Nd [10] WARANG CITI DIGIT ZERO..WARANG CITI DIGIT NINE + {0x118FF, 0x11906, prALetter}, // Lo [8] WARANG CITI OM..DIVES AKURU LETTER E + {0x11909, 0x11909, prALetter}, // Lo DIVES AKURU LETTER O + {0x1190C, 0x11913, prALetter}, // Lo [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA + {0x11915, 0x11916, prALetter}, // Lo [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA + {0x11918, 0x1192F, prALetter}, // Lo [24] DIVES AKURU LETTER DDA..DIVES AKURU LETTER ZA + {0x11930, 0x11935, prExtend}, // Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E + {0x11937, 0x11938, prExtend}, // Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O + {0x1193B, 0x1193C, prExtend}, // Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU + {0x1193D, 0x1193D, prExtend}, // Mc DIVES AKURU SIGN HALANTA + {0x1193E, 0x1193E, prExtend}, // Mn DIVES AKURU VIRAMA + {0x1193F, 0x1193F, prALetter}, // Lo DIVES AKURU PREFIXED NASAL SIGN + {0x11940, 0x11940, prExtend}, // Mc DIVES AKURU MEDIAL YA + {0x11941, 0x11941, prALetter}, // Lo DIVES AKURU INITIAL RA + {0x11942, 0x11942, prExtend}, // Mc DIVES AKURU MEDIAL RA + {0x11943, 0x11943, prExtend}, // Mn DIVES AKURU SIGN NUKTA + {0x11950, 0x11959, prNumeric}, // Nd [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE + {0x119A0, 0x119A7, prALetter}, // Lo [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR + {0x119AA, 0x119D0, prALetter}, // Lo [39] NANDINAGARI LETTER E..NANDINAGARI LETTER RRA + {0x119D1, 0x119D3, prExtend}, // Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II + {0x119D4, 0x119D7, prExtend}, // Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR + {0x119DA, 0x119DB, prExtend}, // Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI + {0x119DC, 0x119DF, prExtend}, // Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA + {0x119E0, 0x119E0, prExtend}, // Mn NANDINAGARI SIGN VIRAMA + {0x119E1, 0x119E1, prALetter}, // Lo NANDINAGARI SIGN AVAGRAHA + {0x119E3, 0x119E3, prALetter}, // Lo NANDINAGARI HEADSTROKE + {0x119E4, 0x119E4, prExtend}, // Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E + {0x11A00, 0x11A00, prALetter}, // Lo ZANABAZAR SQUARE LETTER A + {0x11A01, 0x11A0A, prExtend}, // Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK + {0x11A0B, 0x11A32, prALetter}, // Lo [40] ZANABAZAR SQUARE LETTER KA..ZANABAZAR SQUARE LETTER KSSA + {0x11A33, 0x11A38, prExtend}, // Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA + {0x11A39, 0x11A39, prExtend}, // Mc ZANABAZAR SQUARE SIGN VISARGA + {0x11A3A, 0x11A3A, prALetter}, // Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA + {0x11A3B, 0x11A3E, prExtend}, // Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA + {0x11A47, 0x11A47, prExtend}, // Mn ZANABAZAR SQUARE SUBJOINER + {0x11A50, 0x11A50, prALetter}, // Lo SOYOMBO LETTER A + {0x11A51, 0x11A56, prExtend}, // Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE + {0x11A57, 0x11A58, prExtend}, // Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU + {0x11A59, 0x11A5B, prExtend}, // Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK + {0x11A5C, 0x11A89, prALetter}, // Lo [46] SOYOMBO LETTER KA..SOYOMBO CLUSTER-INITIAL LETTER SA + {0x11A8A, 0x11A96, prExtend}, // Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA + {0x11A97, 0x11A97, prExtend}, // Mc SOYOMBO SIGN VISARGA + {0x11A98, 0x11A99, prExtend}, // Mn [2] SOYOMBO GEMINATION MARK..SOYOMBO SUBJOINER + {0x11A9D, 0x11A9D, prALetter}, // Lo SOYOMBO MARK PLUTA + {0x11AB0, 0x11AF8, prALetter}, // Lo [73] CANADIAN SYLLABICS NATTILIK HI..PAU CIN HAU GLOTTAL STOP FINAL + {0x11C00, 0x11C08, prALetter}, // Lo [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L + {0x11C0A, 0x11C2E, prALetter}, // Lo [37] BHAIKSUKI LETTER E..BHAIKSUKI LETTER HA + {0x11C2F, 0x11C2F, prExtend}, // Mc BHAIKSUKI VOWEL SIGN AA + {0x11C30, 0x11C36, prExtend}, // Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L + {0x11C38, 0x11C3D, prExtend}, // Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA + {0x11C3E, 0x11C3E, prExtend}, // Mc BHAIKSUKI SIGN VISARGA + {0x11C3F, 0x11C3F, prExtend}, // Mn BHAIKSUKI SIGN VIRAMA + {0x11C40, 0x11C40, prALetter}, // Lo BHAIKSUKI SIGN AVAGRAHA + {0x11C50, 0x11C59, prNumeric}, // Nd [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE + {0x11C72, 0x11C8F, prALetter}, // Lo [30] MARCHEN LETTER KA..MARCHEN LETTER A + {0x11C92, 0x11CA7, prExtend}, // Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA + {0x11CA9, 0x11CA9, prExtend}, // Mc MARCHEN SUBJOINED LETTER YA + {0x11CAA, 0x11CB0, prExtend}, // Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA + {0x11CB1, 0x11CB1, prExtend}, // Mc MARCHEN VOWEL SIGN I + {0x11CB2, 0x11CB3, prExtend}, // Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E + {0x11CB4, 0x11CB4, prExtend}, // Mc MARCHEN VOWEL SIGN O + {0x11CB5, 0x11CB6, prExtend}, // Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU + {0x11D00, 0x11D06, prALetter}, // Lo [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E + {0x11D08, 0x11D09, prALetter}, // Lo [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O + {0x11D0B, 0x11D30, prALetter}, // Lo [38] MASARAM GONDI LETTER AU..MASARAM GONDI LETTER TRA + {0x11D31, 0x11D36, prExtend}, // Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R + {0x11D3A, 0x11D3A, prExtend}, // Mn MASARAM GONDI VOWEL SIGN E + {0x11D3C, 0x11D3D, prExtend}, // Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O + {0x11D3F, 0x11D45, prExtend}, // Mn [7] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI VIRAMA + {0x11D46, 0x11D46, prALetter}, // Lo MASARAM GONDI REPHA + {0x11D47, 0x11D47, prExtend}, // Mn MASARAM GONDI RA-KARA + {0x11D50, 0x11D59, prNumeric}, // Nd [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE + {0x11D60, 0x11D65, prALetter}, // Lo [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU + {0x11D67, 0x11D68, prALetter}, // Lo [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI + {0x11D6A, 0x11D89, prALetter}, // Lo [32] GUNJALA GONDI LETTER OO..GUNJALA GONDI LETTER SA + {0x11D8A, 0x11D8E, prExtend}, // Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU + {0x11D90, 0x11D91, prExtend}, // Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI + {0x11D93, 0x11D94, prExtend}, // Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU + {0x11D95, 0x11D95, prExtend}, // Mn GUNJALA GONDI SIGN ANUSVARA + {0x11D96, 0x11D96, prExtend}, // Mc GUNJALA GONDI SIGN VISARGA + {0x11D97, 0x11D97, prExtend}, // Mn GUNJALA GONDI VIRAMA + {0x11D98, 0x11D98, prALetter}, // Lo GUNJALA GONDI OM + {0x11DA0, 0x11DA9, prNumeric}, // Nd [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE + {0x11EE0, 0x11EF2, prALetter}, // Lo [19] MAKASAR LETTER KA..MAKASAR ANGKA + {0x11EF3, 0x11EF4, prExtend}, // Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U + {0x11EF5, 0x11EF6, prExtend}, // Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O + {0x11F00, 0x11F01, prExtend}, // Mn [2] KAWI SIGN CANDRABINDU..KAWI SIGN ANUSVARA + {0x11F02, 0x11F02, prALetter}, // Lo KAWI SIGN REPHA + {0x11F03, 0x11F03, prExtend}, // Mc KAWI SIGN VISARGA + {0x11F04, 0x11F10, prALetter}, // Lo [13] KAWI LETTER A..KAWI LETTER O + {0x11F12, 0x11F33, prALetter}, // Lo [34] KAWI LETTER KA..KAWI LETTER JNYA + {0x11F34, 0x11F35, prExtend}, // Mc [2] KAWI VOWEL SIGN AA..KAWI VOWEL SIGN ALTERNATE AA + {0x11F36, 0x11F3A, prExtend}, // Mn [5] KAWI VOWEL SIGN I..KAWI VOWEL SIGN VOCALIC R + {0x11F3E, 0x11F3F, prExtend}, // Mc [2] KAWI VOWEL SIGN E..KAWI VOWEL SIGN AI + {0x11F40, 0x11F40, prExtend}, // Mn KAWI VOWEL SIGN EU + {0x11F41, 0x11F41, prExtend}, // Mc KAWI SIGN KILLER + {0x11F42, 0x11F42, prExtend}, // Mn KAWI CONJOINER + {0x11F50, 0x11F59, prNumeric}, // Nd [10] KAWI DIGIT ZERO..KAWI DIGIT NINE + {0x11FB0, 0x11FB0, prALetter}, // Lo LISU LETTER YHA + {0x12000, 0x12399, prALetter}, // Lo [922] CUNEIFORM SIGN A..CUNEIFORM SIGN U U + {0x12400, 0x1246E, prALetter}, // Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM + {0x12480, 0x12543, prALetter}, // Lo [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU + {0x12F90, 0x12FF0, prALetter}, // Lo [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 + {0x13000, 0x1342F, prALetter}, // Lo [1072] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH V011D + {0x13430, 0x1343F, prFormat}, // Cf [16] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE + {0x13440, 0x13440, prExtend}, // Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY + {0x13441, 0x13446, prALetter}, // Lo [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN + {0x13447, 0x13455, prExtend}, // Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED + {0x14400, 0x14646, prALetter}, // Lo [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 + {0x16800, 0x16A38, prALetter}, // Lo [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ + {0x16A40, 0x16A5E, prALetter}, // Lo [31] MRO LETTER TA..MRO LETTER TEK + {0x16A60, 0x16A69, prNumeric}, // Nd [10] MRO DIGIT ZERO..MRO DIGIT NINE + {0x16A70, 0x16ABE, prALetter}, // Lo [79] TANGSA LETTER OZ..TANGSA LETTER ZA + {0x16AC0, 0x16AC9, prNumeric}, // Nd [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE + {0x16AD0, 0x16AED, prALetter}, // Lo [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I + {0x16AF0, 0x16AF4, prExtend}, // Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE + {0x16B00, 0x16B2F, prALetter}, // Lo [48] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG CONSONANT CAU + {0x16B30, 0x16B36, prExtend}, // Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM + {0x16B40, 0x16B43, prALetter}, // Lm [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM + {0x16B50, 0x16B59, prNumeric}, // Nd [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE + {0x16B63, 0x16B77, prALetter}, // Lo [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS + {0x16B7D, 0x16B8F, prALetter}, // Lo [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ + {0x16E40, 0x16E7F, prALetter}, // L& [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y + {0x16F00, 0x16F4A, prALetter}, // Lo [75] MIAO LETTER PA..MIAO LETTER RTE + {0x16F4F, 0x16F4F, prExtend}, // Mn MIAO SIGN CONSONANT MODIFIER BAR + {0x16F50, 0x16F50, prALetter}, // Lo MIAO LETTER NASALIZATION + {0x16F51, 0x16F87, prExtend}, // Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI + {0x16F8F, 0x16F92, prExtend}, // Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW + {0x16F93, 0x16F9F, prALetter}, // Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 + {0x16FE0, 0x16FE1, prALetter}, // Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK + {0x16FE3, 0x16FE3, prALetter}, // Lm OLD CHINESE ITERATION MARK + {0x16FE4, 0x16FE4, prExtend}, // Mn KHITAN SMALL SCRIPT FILLER + {0x16FF0, 0x16FF1, prExtend}, // Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY + {0x1AFF0, 0x1AFF3, prKatakana}, // Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 + {0x1AFF5, 0x1AFFB, prKatakana}, // Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 + {0x1AFFD, 0x1AFFE, prKatakana}, // Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 + {0x1B000, 0x1B000, prKatakana}, // Lo KATAKANA LETTER ARCHAIC E + {0x1B120, 0x1B122, prKatakana}, // Lo [3] KATAKANA LETTER ARCHAIC YI..KATAKANA LETTER ARCHAIC WU + {0x1B155, 0x1B155, prKatakana}, // Lo KATAKANA LETTER SMALL KO + {0x1B164, 0x1B167, prKatakana}, // Lo [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N + {0x1BC00, 0x1BC6A, prALetter}, // Lo [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M + {0x1BC70, 0x1BC7C, prALetter}, // Lo [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK + {0x1BC80, 0x1BC88, prALetter}, // Lo [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL + {0x1BC90, 0x1BC99, prALetter}, // Lo [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW + {0x1BC9D, 0x1BC9E, prExtend}, // Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK + {0x1BCA0, 0x1BCA3, prFormat}, // Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP + {0x1CF00, 0x1CF2D, prExtend}, // Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT + {0x1CF30, 0x1CF46, prExtend}, // Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG + {0x1D165, 0x1D166, prExtend}, // Mc [2] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING SPRECHGESANG STEM + {0x1D167, 0x1D169, prExtend}, // Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 + {0x1D16D, 0x1D172, prExtend}, // Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 + {0x1D173, 0x1D17A, prFormat}, // Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE + {0x1D17B, 0x1D182, prExtend}, // Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE + {0x1D185, 0x1D18B, prExtend}, // Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE + {0x1D1AA, 0x1D1AD, prExtend}, // Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO + {0x1D242, 0x1D244, prExtend}, // Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + {0x1D400, 0x1D454, prALetter}, // L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G + {0x1D456, 0x1D49C, prALetter}, // L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A + {0x1D49E, 0x1D49F, prALetter}, // L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D + {0x1D4A2, 0x1D4A2, prALetter}, // L& MATHEMATICAL SCRIPT CAPITAL G + {0x1D4A5, 0x1D4A6, prALetter}, // L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K + {0x1D4A9, 0x1D4AC, prALetter}, // L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q + {0x1D4AE, 0x1D4B9, prALetter}, // L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D + {0x1D4BB, 0x1D4BB, prALetter}, // L& MATHEMATICAL SCRIPT SMALL F + {0x1D4BD, 0x1D4C3, prALetter}, // L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N + {0x1D4C5, 0x1D505, prALetter}, // L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B + {0x1D507, 0x1D50A, prALetter}, // L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G + {0x1D50D, 0x1D514, prALetter}, // L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q + {0x1D516, 0x1D51C, prALetter}, // L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y + {0x1D51E, 0x1D539, prALetter}, // L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B + {0x1D53B, 0x1D53E, prALetter}, // L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G + {0x1D540, 0x1D544, prALetter}, // L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M + {0x1D546, 0x1D546, prALetter}, // L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O + {0x1D54A, 0x1D550, prALetter}, // L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y + {0x1D552, 0x1D6A5, prALetter}, // L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J + {0x1D6A8, 0x1D6C0, prALetter}, // L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA + {0x1D6C2, 0x1D6DA, prALetter}, // L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA + {0x1D6DC, 0x1D6FA, prALetter}, // L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA + {0x1D6FC, 0x1D714, prALetter}, // L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA + {0x1D716, 0x1D734, prALetter}, // L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA + {0x1D736, 0x1D74E, prALetter}, // L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA + {0x1D750, 0x1D76E, prALetter}, // L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA + {0x1D770, 0x1D788, prALetter}, // L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA + {0x1D78A, 0x1D7A8, prALetter}, // L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA + {0x1D7AA, 0x1D7C2, prALetter}, // L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA + {0x1D7C4, 0x1D7CB, prALetter}, // L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA + {0x1D7CE, 0x1D7FF, prNumeric}, // Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + {0x1DA00, 0x1DA36, prExtend}, // Mn [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN + {0x1DA3B, 0x1DA6C, prExtend}, // Mn [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT + {0x1DA75, 0x1DA75, prExtend}, // Mn SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS + {0x1DA84, 0x1DA84, prExtend}, // Mn SIGNWRITING LOCATION HEAD NECK + {0x1DA9B, 0x1DA9F, prExtend}, // Mn [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 + {0x1DAA1, 0x1DAAF, prExtend}, // Mn [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 + {0x1DF00, 0x1DF09, prALetter}, // L& [10] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK + {0x1DF0A, 0x1DF0A, prALetter}, // Lo LATIN LETTER RETROFLEX CLICK WITH RETROFLEX HOOK + {0x1DF0B, 0x1DF1E, prALetter}, // L& [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL + {0x1DF25, 0x1DF2A, prALetter}, // L& [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK + {0x1E000, 0x1E006, prExtend}, // Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE + {0x1E008, 0x1E018, prExtend}, // Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU + {0x1E01B, 0x1E021, prExtend}, // Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI + {0x1E023, 0x1E024, prExtend}, // Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS + {0x1E026, 0x1E02A, prExtend}, // Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA + {0x1E030, 0x1E06D, prALetter}, // Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE + {0x1E08F, 0x1E08F, prExtend}, // Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + {0x1E100, 0x1E12C, prALetter}, // Lo [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W + {0x1E130, 0x1E136, prExtend}, // Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D + {0x1E137, 0x1E13D, prALetter}, // Lm [7] NYIAKENG PUACHUE HMONG SIGN FOR PERSON..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER + {0x1E140, 0x1E149, prNumeric}, // Nd [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE + {0x1E14E, 0x1E14E, prALetter}, // Lo NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ + {0x1E290, 0x1E2AD, prALetter}, // Lo [30] TOTO LETTER PA..TOTO LETTER A + {0x1E2AE, 0x1E2AE, prExtend}, // Mn TOTO SIGN RISING TONE + {0x1E2C0, 0x1E2EB, prALetter}, // Lo [44] WANCHO LETTER AA..WANCHO LETTER YIH + {0x1E2EC, 0x1E2EF, prExtend}, // Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI + {0x1E2F0, 0x1E2F9, prNumeric}, // Nd [10] WANCHO DIGIT ZERO..WANCHO DIGIT NINE + {0x1E4D0, 0x1E4EA, prALetter}, // Lo [27] NAG MUNDARI LETTER O..NAG MUNDARI LETTER ELL + {0x1E4EB, 0x1E4EB, prALetter}, // Lm NAG MUNDARI SIGN OJOD + {0x1E4EC, 0x1E4EF, prExtend}, // Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH + {0x1E4F0, 0x1E4F9, prNumeric}, // Nd [10] NAG MUNDARI DIGIT ZERO..NAG MUNDARI DIGIT NINE + {0x1E7E0, 0x1E7E6, prALetter}, // Lo [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO + {0x1E7E8, 0x1E7EB, prALetter}, // Lo [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE + {0x1E7ED, 0x1E7EE, prALetter}, // Lo [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE + {0x1E7F0, 0x1E7FE, prALetter}, // Lo [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE + {0x1E800, 0x1E8C4, prALetter}, // Lo [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON + {0x1E8D0, 0x1E8D6, prExtend}, // Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + {0x1E900, 0x1E943, prALetter}, // L& [68] ADLAM CAPITAL LETTER ALIF..ADLAM SMALL LETTER SHA + {0x1E944, 0x1E94A, prExtend}, // Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA + {0x1E94B, 0x1E94B, prALetter}, // Lm ADLAM NASALIZATION MARK + {0x1E950, 0x1E959, prNumeric}, // Nd [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE + {0x1EE00, 0x1EE03, prALetter}, // Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL + {0x1EE05, 0x1EE1F, prALetter}, // Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF + {0x1EE21, 0x1EE22, prALetter}, // Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM + {0x1EE24, 0x1EE24, prALetter}, // Lo ARABIC MATHEMATICAL INITIAL HEH + {0x1EE27, 0x1EE27, prALetter}, // Lo ARABIC MATHEMATICAL INITIAL HAH + {0x1EE29, 0x1EE32, prALetter}, // Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF + {0x1EE34, 0x1EE37, prALetter}, // Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH + {0x1EE39, 0x1EE39, prALetter}, // Lo ARABIC MATHEMATICAL INITIAL DAD + {0x1EE3B, 0x1EE3B, prALetter}, // Lo ARABIC MATHEMATICAL INITIAL GHAIN + {0x1EE42, 0x1EE42, prALetter}, // Lo ARABIC MATHEMATICAL TAILED JEEM + {0x1EE47, 0x1EE47, prALetter}, // Lo ARABIC MATHEMATICAL TAILED HAH + {0x1EE49, 0x1EE49, prALetter}, // Lo ARABIC MATHEMATICAL TAILED YEH + {0x1EE4B, 0x1EE4B, prALetter}, // Lo ARABIC MATHEMATICAL TAILED LAM + {0x1EE4D, 0x1EE4F, prALetter}, // Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN + {0x1EE51, 0x1EE52, prALetter}, // Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF + {0x1EE54, 0x1EE54, prALetter}, // Lo ARABIC MATHEMATICAL TAILED SHEEN + {0x1EE57, 0x1EE57, prALetter}, // Lo ARABIC MATHEMATICAL TAILED KHAH + {0x1EE59, 0x1EE59, prALetter}, // Lo ARABIC MATHEMATICAL TAILED DAD + {0x1EE5B, 0x1EE5B, prALetter}, // Lo ARABIC MATHEMATICAL TAILED GHAIN + {0x1EE5D, 0x1EE5D, prALetter}, // Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON + {0x1EE5F, 0x1EE5F, prALetter}, // Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF + {0x1EE61, 0x1EE62, prALetter}, // Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM + {0x1EE64, 0x1EE64, prALetter}, // Lo ARABIC MATHEMATICAL STRETCHED HEH + {0x1EE67, 0x1EE6A, prALetter}, // Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF + {0x1EE6C, 0x1EE72, prALetter}, // Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF + {0x1EE74, 0x1EE77, prALetter}, // Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH + {0x1EE79, 0x1EE7C, prALetter}, // Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH + {0x1EE7E, 0x1EE7E, prALetter}, // Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH + {0x1EE80, 0x1EE89, prALetter}, // Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH + {0x1EE8B, 0x1EE9B, prALetter}, // Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN + {0x1EEA1, 0x1EEA3, prALetter}, // Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL + {0x1EEA5, 0x1EEA9, prALetter}, // Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH + {0x1EEAB, 0x1EEBB, prALetter}, // Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN + {0x1F000, 0x1F003, prExtendedPictographic}, // E0.0 [4] (๐Ÿ€€..๐Ÿ€ƒ) MAHJONG TILE EAST WIND..MAHJONG TILE NORTH WIND + {0x1F004, 0x1F004, prExtendedPictographic}, // E0.6 [1] (๐Ÿ€„) mahjong red dragon + {0x1F005, 0x1F0CE, prExtendedPictographic}, // E0.0 [202] (๐Ÿ€…..๐ŸƒŽ) MAHJONG TILE GREEN DRAGON..PLAYING CARD KING OF DIAMONDS + {0x1F0CF, 0x1F0CF, prExtendedPictographic}, // E0.6 [1] (๐Ÿƒ) joker + {0x1F0D0, 0x1F0FF, prExtendedPictographic}, // E0.0 [48] (๐Ÿƒ..๐Ÿƒฟ) .. + {0x1F10D, 0x1F10F, prExtendedPictographic}, // E0.0 [3] (๐Ÿ„..๐Ÿ„) CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH + {0x1F12F, 0x1F12F, prExtendedPictographic}, // E0.0 [1] (๐Ÿ„ฏ) COPYLEFT SYMBOL + {0x1F130, 0x1F149, prALetter}, // So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z + {0x1F150, 0x1F169, prALetter}, // So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z + {0x1F16C, 0x1F16F, prExtendedPictographic}, // E0.0 [4] (๐Ÿ…ฌ..๐Ÿ…ฏ) RAISED MR SIGN..CIRCLED HUMAN FIGURE + {0x1F170, 0x1F189, prALetter}, // So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z + {0x1F170, 0x1F171, prExtendedPictographic}, // E0.6 [2] (๐Ÿ…ฐ๏ธ..๐Ÿ…ฑ๏ธ) A button (blood type)..B button (blood type) + {0x1F17E, 0x1F17F, prExtendedPictographic}, // E0.6 [2] (๐Ÿ…พ๏ธ..๐Ÿ…ฟ๏ธ) O button (blood type)..P button + {0x1F18E, 0x1F18E, prExtendedPictographic}, // E0.6 [1] (๐Ÿ†Ž) AB button (blood type) + {0x1F191, 0x1F19A, prExtendedPictographic}, // E0.6 [10] (๐Ÿ†‘..๐Ÿ†š) CL button..VS button + {0x1F1AD, 0x1F1E5, prExtendedPictographic}, // E0.0 [57] (๐Ÿ†ญ..๐Ÿ‡ฅ) MASK WORK SYMBOL.. + {0x1F1E6, 0x1F1FF, prRegionalIndicator}, // So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + {0x1F201, 0x1F202, prExtendedPictographic}, // E0.6 [2] (๐Ÿˆ..๐Ÿˆ‚๏ธ) Japanese โ€œhereโ€ button..Japanese โ€œservice chargeโ€ button + {0x1F203, 0x1F20F, prExtendedPictographic}, // E0.0 [13] (๐Ÿˆƒ..๐Ÿˆ) .. + {0x1F21A, 0x1F21A, prExtendedPictographic}, // E0.6 [1] (๐Ÿˆš) Japanese โ€œfree of chargeโ€ button + {0x1F22F, 0x1F22F, prExtendedPictographic}, // E0.6 [1] (๐Ÿˆฏ) Japanese โ€œreservedโ€ button + {0x1F232, 0x1F23A, prExtendedPictographic}, // E0.6 [9] (๐Ÿˆฒ..๐Ÿˆบ) Japanese โ€œprohibitedโ€ button..Japanese โ€œopen for businessโ€ button + {0x1F23C, 0x1F23F, prExtendedPictographic}, // E0.0 [4] (๐Ÿˆผ..๐Ÿˆฟ) .. + {0x1F249, 0x1F24F, prExtendedPictographic}, // E0.0 [7] (๐Ÿ‰‰..๐Ÿ‰) .. + {0x1F250, 0x1F251, prExtendedPictographic}, // E0.6 [2] (๐Ÿ‰..๐Ÿ‰‘) Japanese โ€œbargainโ€ button..Japanese โ€œacceptableโ€ button + {0x1F252, 0x1F2FF, prExtendedPictographic}, // E0.0 [174] (๐Ÿ‰’..๐Ÿ‹ฟ) .. + {0x1F300, 0x1F30C, prExtendedPictographic}, // E0.6 [13] (๐ŸŒ€..๐ŸŒŒ) cyclone..milky way + {0x1F30D, 0x1F30E, prExtendedPictographic}, // E0.7 [2] (๐ŸŒ..๐ŸŒŽ) globe showing Europe-Africa..globe showing Americas + {0x1F30F, 0x1F30F, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ) globe showing Asia-Australia + {0x1F310, 0x1F310, prExtendedPictographic}, // E1.0 [1] (๐ŸŒ) globe with meridians + {0x1F311, 0x1F311, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ‘) new moon + {0x1F312, 0x1F312, prExtendedPictographic}, // E1.0 [1] (๐ŸŒ’) waxing crescent moon + {0x1F313, 0x1F315, prExtendedPictographic}, // E0.6 [3] (๐ŸŒ“..๐ŸŒ•) first quarter moon..full moon + {0x1F316, 0x1F318, prExtendedPictographic}, // E1.0 [3] (๐ŸŒ–..๐ŸŒ˜) waning gibbous moon..waning crescent moon + {0x1F319, 0x1F319, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ™) crescent moon + {0x1F31A, 0x1F31A, prExtendedPictographic}, // E1.0 [1] (๐ŸŒš) new moon face + {0x1F31B, 0x1F31B, prExtendedPictographic}, // E0.6 [1] (๐ŸŒ›) first quarter moon face + {0x1F31C, 0x1F31C, prExtendedPictographic}, // E0.7 [1] (๐ŸŒœ) last quarter moon face + {0x1F31D, 0x1F31E, prExtendedPictographic}, // E1.0 [2] (๐ŸŒ..๐ŸŒž) full moon face..sun with face + {0x1F31F, 0x1F320, prExtendedPictographic}, // E0.6 [2] (๐ŸŒŸ..๐ŸŒ ) glowing star..shooting star + {0x1F321, 0x1F321, prExtendedPictographic}, // E0.7 [1] (๐ŸŒก๏ธ) thermometer + {0x1F322, 0x1F323, prExtendedPictographic}, // E0.0 [2] (๐ŸŒข..๐ŸŒฃ) BLACK DROPLET..WHITE SUN + {0x1F324, 0x1F32C, prExtendedPictographic}, // E0.7 [9] (๐ŸŒค๏ธ..๐ŸŒฌ๏ธ) sun behind small cloud..wind face + {0x1F32D, 0x1F32F, prExtendedPictographic}, // E1.0 [3] (๐ŸŒญ..๐ŸŒฏ) hot dog..burrito + {0x1F330, 0x1F331, prExtendedPictographic}, // E0.6 [2] (๐ŸŒฐ..๐ŸŒฑ) chestnut..seedling + {0x1F332, 0x1F333, prExtendedPictographic}, // E1.0 [2] (๐ŸŒฒ..๐ŸŒณ) evergreen tree..deciduous tree + {0x1F334, 0x1F335, prExtendedPictographic}, // E0.6 [2] (๐ŸŒด..๐ŸŒต) palm tree..cactus + {0x1F336, 0x1F336, prExtendedPictographic}, // E0.7 [1] (๐ŸŒถ๏ธ) hot pepper + {0x1F337, 0x1F34A, prExtendedPictographic}, // E0.6 [20] (๐ŸŒท..๐ŸŠ) tulip..tangerine + {0x1F34B, 0x1F34B, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‹) lemon + {0x1F34C, 0x1F34F, prExtendedPictographic}, // E0.6 [4] (๐ŸŒ..๐Ÿ) banana..green apple + {0x1F350, 0x1F350, prExtendedPictographic}, // E1.0 [1] (๐Ÿ) pear + {0x1F351, 0x1F37B, prExtendedPictographic}, // E0.6 [43] (๐Ÿ‘..๐Ÿป) peach..clinking beer mugs + {0x1F37C, 0x1F37C, prExtendedPictographic}, // E1.0 [1] (๐Ÿผ) baby bottle + {0x1F37D, 0x1F37D, prExtendedPictographic}, // E0.7 [1] (๐Ÿฝ๏ธ) fork and knife with plate + {0x1F37E, 0x1F37F, prExtendedPictographic}, // E1.0 [2] (๐Ÿพ..๐Ÿฟ) bottle with popping cork..popcorn + {0x1F380, 0x1F393, prExtendedPictographic}, // E0.6 [20] (๐ŸŽ€..๐ŸŽ“) ribbon..graduation cap + {0x1F394, 0x1F395, prExtendedPictographic}, // E0.0 [2] (๐ŸŽ”..๐ŸŽ•) HEART WITH TIP ON THE LEFT..BOUQUET OF FLOWERS + {0x1F396, 0x1F397, prExtendedPictographic}, // E0.7 [2] (๐ŸŽ–๏ธ..๐ŸŽ—๏ธ) military medal..reminder ribbon + {0x1F398, 0x1F398, prExtendedPictographic}, // E0.0 [1] (๐ŸŽ˜) MUSICAL KEYBOARD WITH JACKS + {0x1F399, 0x1F39B, prExtendedPictographic}, // E0.7 [3] (๐ŸŽ™๏ธ..๐ŸŽ›๏ธ) studio microphone..control knobs + {0x1F39C, 0x1F39D, prExtendedPictographic}, // E0.0 [2] (๐ŸŽœ..๐ŸŽ) BEAMED ASCENDING MUSICAL NOTES..BEAMED DESCENDING MUSICAL NOTES + {0x1F39E, 0x1F39F, prExtendedPictographic}, // E0.7 [2] (๐ŸŽž๏ธ..๐ŸŽŸ๏ธ) film frames..admission tickets + {0x1F3A0, 0x1F3C4, prExtendedPictographic}, // E0.6 [37] (๐ŸŽ ..๐Ÿ„) carousel horse..person surfing + {0x1F3C5, 0x1F3C5, prExtendedPictographic}, // E1.0 [1] (๐Ÿ…) sports medal + {0x1F3C6, 0x1F3C6, prExtendedPictographic}, // E0.6 [1] (๐Ÿ†) trophy + {0x1F3C7, 0x1F3C7, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‡) horse racing + {0x1F3C8, 0x1F3C8, prExtendedPictographic}, // E0.6 [1] (๐Ÿˆ) american football + {0x1F3C9, 0x1F3C9, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‰) rugby football + {0x1F3CA, 0x1F3CA, prExtendedPictographic}, // E0.6 [1] (๐ŸŠ) person swimming + {0x1F3CB, 0x1F3CE, prExtendedPictographic}, // E0.7 [4] (๐Ÿ‹๏ธ..๐ŸŽ๏ธ) person lifting weights..racing car + {0x1F3CF, 0x1F3D3, prExtendedPictographic}, // E1.0 [5] (๐Ÿ..๐Ÿ“) cricket game..ping pong + {0x1F3D4, 0x1F3DF, prExtendedPictographic}, // E0.7 [12] (๐Ÿ”๏ธ..๐ŸŸ๏ธ) snow-capped mountain..stadium + {0x1F3E0, 0x1F3E3, prExtendedPictographic}, // E0.6 [4] (๐Ÿ ..๐Ÿฃ) house..Japanese post office + {0x1F3E4, 0x1F3E4, prExtendedPictographic}, // E1.0 [1] (๐Ÿค) post office + {0x1F3E5, 0x1F3F0, prExtendedPictographic}, // E0.6 [12] (๐Ÿฅ..๐Ÿฐ) hospital..castle + {0x1F3F1, 0x1F3F2, prExtendedPictographic}, // E0.0 [2] (๐Ÿฑ..๐Ÿฒ) WHITE PENNANT..BLACK PENNANT + {0x1F3F3, 0x1F3F3, prExtendedPictographic}, // E0.7 [1] (๐Ÿณ๏ธ) white flag + {0x1F3F4, 0x1F3F4, prExtendedPictographic}, // E1.0 [1] (๐Ÿด) black flag + {0x1F3F5, 0x1F3F5, prExtendedPictographic}, // E0.7 [1] (๐Ÿต๏ธ) rosette + {0x1F3F6, 0x1F3F6, prExtendedPictographic}, // E0.0 [1] (๐Ÿถ) BLACK ROSETTE + {0x1F3F7, 0x1F3F7, prExtendedPictographic}, // E0.7 [1] (๐Ÿท๏ธ) label + {0x1F3F8, 0x1F3FA, prExtendedPictographic}, // E1.0 [3] (๐Ÿธ..๐Ÿบ) badminton..amphora + {0x1F3FB, 0x1F3FF, prExtend}, // Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 + {0x1F400, 0x1F407, prExtendedPictographic}, // E1.0 [8] (๐Ÿ€..๐Ÿ‡) rat..rabbit + {0x1F408, 0x1F408, prExtendedPictographic}, // E0.7 [1] (๐Ÿˆ) cat + {0x1F409, 0x1F40B, prExtendedPictographic}, // E1.0 [3] (๐Ÿ‰..๐Ÿ‹) dragon..whale + {0x1F40C, 0x1F40E, prExtendedPictographic}, // E0.6 [3] (๐ŸŒ..๐ŸŽ) snail..horse + {0x1F40F, 0x1F410, prExtendedPictographic}, // E1.0 [2] (๐Ÿ..๐Ÿ) ram..goat + {0x1F411, 0x1F412, prExtendedPictographic}, // E0.6 [2] (๐Ÿ‘..๐Ÿ’) ewe..monkey + {0x1F413, 0x1F413, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“) rooster + {0x1F414, 0x1F414, prExtendedPictographic}, // E0.6 [1] (๐Ÿ”) chicken + {0x1F415, 0x1F415, prExtendedPictographic}, // E0.7 [1] (๐Ÿ•) dog + {0x1F416, 0x1F416, prExtendedPictographic}, // E1.0 [1] (๐Ÿ–) pig + {0x1F417, 0x1F429, prExtendedPictographic}, // E0.6 [19] (๐Ÿ—..๐Ÿฉ) boar..poodle + {0x1F42A, 0x1F42A, prExtendedPictographic}, // E1.0 [1] (๐Ÿช) camel + {0x1F42B, 0x1F43E, prExtendedPictographic}, // E0.6 [20] (๐Ÿซ..๐Ÿพ) two-hump camel..paw prints + {0x1F43F, 0x1F43F, prExtendedPictographic}, // E0.7 [1] (๐Ÿฟ๏ธ) chipmunk + {0x1F440, 0x1F440, prExtendedPictographic}, // E0.6 [1] (๐Ÿ‘€) eyes + {0x1F441, 0x1F441, prExtendedPictographic}, // E0.7 [1] (๐Ÿ‘๏ธ) eye + {0x1F442, 0x1F464, prExtendedPictographic}, // E0.6 [35] (๐Ÿ‘‚..๐Ÿ‘ค) ear..bust in silhouette + {0x1F465, 0x1F465, prExtendedPictographic}, // E1.0 [1] (๐Ÿ‘ฅ) busts in silhouette + {0x1F466, 0x1F46B, prExtendedPictographic}, // E0.6 [6] (๐Ÿ‘ฆ..๐Ÿ‘ซ) boy..woman and man holding hands + {0x1F46C, 0x1F46D, prExtendedPictographic}, // E1.0 [2] (๐Ÿ‘ฌ..๐Ÿ‘ญ) men holding hands..women holding hands + {0x1F46E, 0x1F4AC, prExtendedPictographic}, // E0.6 [63] (๐Ÿ‘ฎ..๐Ÿ’ฌ) police officer..speech balloon + {0x1F4AD, 0x1F4AD, prExtendedPictographic}, // E1.0 [1] (๐Ÿ’ญ) thought balloon + {0x1F4AE, 0x1F4B5, prExtendedPictographic}, // E0.6 [8] (๐Ÿ’ฎ..๐Ÿ’ต) white flower..dollar banknote + {0x1F4B6, 0x1F4B7, prExtendedPictographic}, // E1.0 [2] (๐Ÿ’ถ..๐Ÿ’ท) euro banknote..pound banknote + {0x1F4B8, 0x1F4EB, prExtendedPictographic}, // E0.6 [52] (๐Ÿ’ธ..๐Ÿ“ซ) money with wings..closed mailbox with raised flag + {0x1F4EC, 0x1F4ED, prExtendedPictographic}, // E0.7 [2] (๐Ÿ“ฌ..๐Ÿ“ญ) open mailbox with raised flag..open mailbox with lowered flag + {0x1F4EE, 0x1F4EE, prExtendedPictographic}, // E0.6 [1] (๐Ÿ“ฎ) postbox + {0x1F4EF, 0x1F4EF, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“ฏ) postal horn + {0x1F4F0, 0x1F4F4, prExtendedPictographic}, // E0.6 [5] (๐Ÿ“ฐ..๐Ÿ“ด) newspaper..mobile phone off + {0x1F4F5, 0x1F4F5, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“ต) no mobile phones + {0x1F4F6, 0x1F4F7, prExtendedPictographic}, // E0.6 [2] (๐Ÿ“ถ..๐Ÿ“ท) antenna bars..camera + {0x1F4F8, 0x1F4F8, prExtendedPictographic}, // E1.0 [1] (๐Ÿ“ธ) camera with flash + {0x1F4F9, 0x1F4FC, prExtendedPictographic}, // E0.6 [4] (๐Ÿ“น..๐Ÿ“ผ) video camera..videocassette + {0x1F4FD, 0x1F4FD, prExtendedPictographic}, // E0.7 [1] (๐Ÿ“ฝ๏ธ) film projector + {0x1F4FE, 0x1F4FE, prExtendedPictographic}, // E0.0 [1] (๐Ÿ“พ) PORTABLE STEREO + {0x1F4FF, 0x1F502, prExtendedPictographic}, // E1.0 [4] (๐Ÿ“ฟ..๐Ÿ”‚) prayer beads..repeat single button + {0x1F503, 0x1F503, prExtendedPictographic}, // E0.6 [1] (๐Ÿ”ƒ) clockwise vertical arrows + {0x1F504, 0x1F507, prExtendedPictographic}, // E1.0 [4] (๐Ÿ”„..๐Ÿ”‡) counterclockwise arrows button..muted speaker + {0x1F508, 0x1F508, prExtendedPictographic}, // E0.7 [1] (๐Ÿ”ˆ) speaker low volume + {0x1F509, 0x1F509, prExtendedPictographic}, // E1.0 [1] (๐Ÿ”‰) speaker medium volume + {0x1F50A, 0x1F514, prExtendedPictographic}, // E0.6 [11] (๐Ÿ”Š..๐Ÿ””) speaker high volume..bell + {0x1F515, 0x1F515, prExtendedPictographic}, // E1.0 [1] (๐Ÿ”•) bell with slash + {0x1F516, 0x1F52B, prExtendedPictographic}, // E0.6 [22] (๐Ÿ”–..๐Ÿ”ซ) bookmark..water pistol + {0x1F52C, 0x1F52D, prExtendedPictographic}, // E1.0 [2] (๐Ÿ”ฌ..๐Ÿ”ญ) microscope..telescope + {0x1F52E, 0x1F53D, prExtendedPictographic}, // E0.6 [16] (๐Ÿ”ฎ..๐Ÿ”ฝ) crystal ball..downwards button + {0x1F546, 0x1F548, prExtendedPictographic}, // E0.0 [3] (๐Ÿ•†..๐Ÿ•ˆ) WHITE LATIN CROSS..CELTIC CROSS + {0x1F549, 0x1F54A, prExtendedPictographic}, // E0.7 [2] (๐Ÿ•‰๏ธ..๐Ÿ•Š๏ธ) om..dove + {0x1F54B, 0x1F54E, prExtendedPictographic}, // E1.0 [4] (๐Ÿ•‹..๐Ÿ•Ž) kaaba..menorah + {0x1F54F, 0x1F54F, prExtendedPictographic}, // E0.0 [1] (๐Ÿ•) BOWL OF HYGIEIA + {0x1F550, 0x1F55B, prExtendedPictographic}, // E0.6 [12] (๐Ÿ•..๐Ÿ•›) one oโ€™clock..twelve oโ€™clock + {0x1F55C, 0x1F567, prExtendedPictographic}, // E0.7 [12] (๐Ÿ•œ..๐Ÿ•ง) one-thirty..twelve-thirty + {0x1F568, 0x1F56E, prExtendedPictographic}, // E0.0 [7] (๐Ÿ•จ..๐Ÿ•ฎ) RIGHT SPEAKER..BOOK + {0x1F56F, 0x1F570, prExtendedPictographic}, // E0.7 [2] (๐Ÿ•ฏ๏ธ..๐Ÿ•ฐ๏ธ) candle..mantelpiece clock + {0x1F571, 0x1F572, prExtendedPictographic}, // E0.0 [2] (๐Ÿ•ฑ..๐Ÿ•ฒ) BLACK SKULL AND CROSSBONES..NO PIRACY + {0x1F573, 0x1F579, prExtendedPictographic}, // E0.7 [7] (๐Ÿ•ณ๏ธ..๐Ÿ•น๏ธ) hole..joystick + {0x1F57A, 0x1F57A, prExtendedPictographic}, // E3.0 [1] (๐Ÿ•บ) man dancing + {0x1F57B, 0x1F586, prExtendedPictographic}, // E0.0 [12] (๐Ÿ•ป..๐Ÿ–†) LEFT HAND TELEPHONE RECEIVER..PEN OVER STAMPED ENVELOPE + {0x1F587, 0x1F587, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–‡๏ธ) linked paperclips + {0x1F588, 0x1F589, prExtendedPictographic}, // E0.0 [2] (๐Ÿ–ˆ..๐Ÿ–‰) BLACK PUSHPIN..LOWER LEFT PENCIL + {0x1F58A, 0x1F58D, prExtendedPictographic}, // E0.7 [4] (๐Ÿ–Š๏ธ..๐Ÿ–๏ธ) pen..crayon + {0x1F58E, 0x1F58F, prExtendedPictographic}, // E0.0 [2] (๐Ÿ–Ž..๐Ÿ–) LEFT WRITING HAND..TURNED OK HAND SIGN + {0x1F590, 0x1F590, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–๏ธ) hand with fingers splayed + {0x1F591, 0x1F594, prExtendedPictographic}, // E0.0 [4] (๐Ÿ–‘..๐Ÿ–”) REVERSED RAISED HAND WITH FINGERS SPLAYED..REVERSED VICTORY HAND + {0x1F595, 0x1F596, prExtendedPictographic}, // E1.0 [2] (๐Ÿ–•..๐Ÿ––) middle finger..vulcan salute + {0x1F597, 0x1F5A3, prExtendedPictographic}, // E0.0 [13] (๐Ÿ–—..๐Ÿ–ฃ) WHITE DOWN POINTING LEFT HAND INDEX..BLACK DOWN POINTING BACKHAND INDEX + {0x1F5A4, 0x1F5A4, prExtendedPictographic}, // E3.0 [1] (๐Ÿ–ค) black heart + {0x1F5A5, 0x1F5A5, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–ฅ๏ธ) desktop computer + {0x1F5A6, 0x1F5A7, prExtendedPictographic}, // E0.0 [2] (๐Ÿ–ฆ..๐Ÿ–ง) KEYBOARD AND MOUSE..THREE NETWORKED COMPUTERS + {0x1F5A8, 0x1F5A8, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–จ๏ธ) printer + {0x1F5A9, 0x1F5B0, prExtendedPictographic}, // E0.0 [8] (๐Ÿ–ฉ..๐Ÿ–ฐ) POCKET CALCULATOR..TWO BUTTON MOUSE + {0x1F5B1, 0x1F5B2, prExtendedPictographic}, // E0.7 [2] (๐Ÿ–ฑ๏ธ..๐Ÿ–ฒ๏ธ) computer mouse..trackball + {0x1F5B3, 0x1F5BB, prExtendedPictographic}, // E0.0 [9] (๐Ÿ–ณ..๐Ÿ–ป) OLD PERSONAL COMPUTER..DOCUMENT WITH PICTURE + {0x1F5BC, 0x1F5BC, prExtendedPictographic}, // E0.7 [1] (๐Ÿ–ผ๏ธ) framed picture + {0x1F5BD, 0x1F5C1, prExtendedPictographic}, // E0.0 [5] (๐Ÿ–ฝ..๐Ÿ—) FRAME WITH TILES..OPEN FOLDER + {0x1F5C2, 0x1F5C4, prExtendedPictographic}, // E0.7 [3] (๐Ÿ—‚๏ธ..๐Ÿ—„๏ธ) card index dividers..file cabinet + {0x1F5C5, 0x1F5D0, prExtendedPictographic}, // E0.0 [12] (๐Ÿ—…..๐Ÿ—) EMPTY NOTE..PAGES + {0x1F5D1, 0x1F5D3, prExtendedPictographic}, // E0.7 [3] (๐Ÿ—‘๏ธ..๐Ÿ—“๏ธ) wastebasket..spiral calendar + {0x1F5D4, 0x1F5DB, prExtendedPictographic}, // E0.0 [8] (๐Ÿ—”..๐Ÿ—›) DESKTOP WINDOW..DECREASE FONT SIZE SYMBOL + {0x1F5DC, 0x1F5DE, prExtendedPictographic}, // E0.7 [3] (๐Ÿ—œ๏ธ..๐Ÿ—ž๏ธ) clamp..rolled-up newspaper + {0x1F5DF, 0x1F5E0, prExtendedPictographic}, // E0.0 [2] (๐Ÿ—Ÿ..๐Ÿ— ) PAGE WITH CIRCLED TEXT..STOCK CHART + {0x1F5E1, 0x1F5E1, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ก๏ธ) dagger + {0x1F5E2, 0x1F5E2, prExtendedPictographic}, // E0.0 [1] (๐Ÿ—ข) LIPS + {0x1F5E3, 0x1F5E3, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ฃ๏ธ) speaking head + {0x1F5E4, 0x1F5E7, prExtendedPictographic}, // E0.0 [4] (๐Ÿ—ค..๐Ÿ—ง) THREE RAYS ABOVE..THREE RAYS RIGHT + {0x1F5E8, 0x1F5E8, prExtendedPictographic}, // E2.0 [1] (๐Ÿ—จ๏ธ) left speech bubble + {0x1F5E9, 0x1F5EE, prExtendedPictographic}, // E0.0 [6] (๐Ÿ—ฉ..๐Ÿ—ฎ) RIGHT SPEECH BUBBLE..LEFT ANGER BUBBLE + {0x1F5EF, 0x1F5EF, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ฏ๏ธ) right anger bubble + {0x1F5F0, 0x1F5F2, prExtendedPictographic}, // E0.0 [3] (๐Ÿ—ฐ..๐Ÿ—ฒ) MOOD BUBBLE..LIGHTNING MOOD + {0x1F5F3, 0x1F5F3, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—ณ๏ธ) ballot box with ballot + {0x1F5F4, 0x1F5F9, prExtendedPictographic}, // E0.0 [6] (๐Ÿ—ด..๐Ÿ—น) BALLOT SCRIPT X..BALLOT BOX WITH BOLD CHECK + {0x1F5FA, 0x1F5FA, prExtendedPictographic}, // E0.7 [1] (๐Ÿ—บ๏ธ) world map + {0x1F5FB, 0x1F5FF, prExtendedPictographic}, // E0.6 [5] (๐Ÿ—ป..๐Ÿ—ฟ) mount fuji..moai + {0x1F600, 0x1F600, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜€) grinning face + {0x1F601, 0x1F606, prExtendedPictographic}, // E0.6 [6] (๐Ÿ˜..๐Ÿ˜†) beaming face with smiling eyes..grinning squinting face + {0x1F607, 0x1F608, prExtendedPictographic}, // E1.0 [2] (๐Ÿ˜‡..๐Ÿ˜ˆ) smiling face with halo..smiling face with horns + {0x1F609, 0x1F60D, prExtendedPictographic}, // E0.6 [5] (๐Ÿ˜‰..๐Ÿ˜) winking face..smiling face with heart-eyes + {0x1F60E, 0x1F60E, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜Ž) smiling face with sunglasses + {0x1F60F, 0x1F60F, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜) smirking face + {0x1F610, 0x1F610, prExtendedPictographic}, // E0.7 [1] (๐Ÿ˜) neutral face + {0x1F611, 0x1F611, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜‘) expressionless face + {0x1F612, 0x1F614, prExtendedPictographic}, // E0.6 [3] (๐Ÿ˜’..๐Ÿ˜”) unamused face..pensive face + {0x1F615, 0x1F615, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜•) confused face + {0x1F616, 0x1F616, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜–) confounded face + {0x1F617, 0x1F617, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜—) kissing face + {0x1F618, 0x1F618, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜˜) face blowing a kiss + {0x1F619, 0x1F619, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜™) kissing face with smiling eyes + {0x1F61A, 0x1F61A, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜š) kissing face with closed eyes + {0x1F61B, 0x1F61B, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜›) face with tongue + {0x1F61C, 0x1F61E, prExtendedPictographic}, // E0.6 [3] (๐Ÿ˜œ..๐Ÿ˜ž) winking face with tongue..disappointed face + {0x1F61F, 0x1F61F, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜Ÿ) worried face + {0x1F620, 0x1F625, prExtendedPictographic}, // E0.6 [6] (๐Ÿ˜ ..๐Ÿ˜ฅ) angry face..sad but relieved face + {0x1F626, 0x1F627, prExtendedPictographic}, // E1.0 [2] (๐Ÿ˜ฆ..๐Ÿ˜ง) frowning face with open mouth..anguished face + {0x1F628, 0x1F62B, prExtendedPictographic}, // E0.6 [4] (๐Ÿ˜จ..๐Ÿ˜ซ) fearful face..tired face + {0x1F62C, 0x1F62C, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜ฌ) grimacing face + {0x1F62D, 0x1F62D, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜ญ) loudly crying face + {0x1F62E, 0x1F62F, prExtendedPictographic}, // E1.0 [2] (๐Ÿ˜ฎ..๐Ÿ˜ฏ) face with open mouth..hushed face + {0x1F630, 0x1F633, prExtendedPictographic}, // E0.6 [4] (๐Ÿ˜ฐ..๐Ÿ˜ณ) anxious face with sweat..flushed face + {0x1F634, 0x1F634, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜ด) sleeping face + {0x1F635, 0x1F635, prExtendedPictographic}, // E0.6 [1] (๐Ÿ˜ต) face with crossed-out eyes + {0x1F636, 0x1F636, prExtendedPictographic}, // E1.0 [1] (๐Ÿ˜ถ) face without mouth + {0x1F637, 0x1F640, prExtendedPictographic}, // E0.6 [10] (๐Ÿ˜ท..๐Ÿ™€) face with medical mask..weary cat + {0x1F641, 0x1F644, prExtendedPictographic}, // E1.0 [4] (๐Ÿ™..๐Ÿ™„) slightly frowning face..face with rolling eyes + {0x1F645, 0x1F64F, prExtendedPictographic}, // E0.6 [11] (๐Ÿ™…..๐Ÿ™) person gesturing NO..folded hands + {0x1F680, 0x1F680, prExtendedPictographic}, // E0.6 [1] (๐Ÿš€) rocket + {0x1F681, 0x1F682, prExtendedPictographic}, // E1.0 [2] (๐Ÿš..๐Ÿš‚) helicopter..locomotive + {0x1F683, 0x1F685, prExtendedPictographic}, // E0.6 [3] (๐Ÿšƒ..๐Ÿš…) railway car..bullet train + {0x1F686, 0x1F686, prExtendedPictographic}, // E1.0 [1] (๐Ÿš†) train + {0x1F687, 0x1F687, prExtendedPictographic}, // E0.6 [1] (๐Ÿš‡) metro + {0x1F688, 0x1F688, prExtendedPictographic}, // E1.0 [1] (๐Ÿšˆ) light rail + {0x1F689, 0x1F689, prExtendedPictographic}, // E0.6 [1] (๐Ÿš‰) station + {0x1F68A, 0x1F68B, prExtendedPictographic}, // E1.0 [2] (๐ŸšŠ..๐Ÿš‹) tram..tram car + {0x1F68C, 0x1F68C, prExtendedPictographic}, // E0.6 [1] (๐ŸšŒ) bus + {0x1F68D, 0x1F68D, prExtendedPictographic}, // E0.7 [1] (๐Ÿš) oncoming bus + {0x1F68E, 0x1F68E, prExtendedPictographic}, // E1.0 [1] (๐ŸšŽ) trolleybus + {0x1F68F, 0x1F68F, prExtendedPictographic}, // E0.6 [1] (๐Ÿš) bus stop + {0x1F690, 0x1F690, prExtendedPictographic}, // E1.0 [1] (๐Ÿš) minibus + {0x1F691, 0x1F693, prExtendedPictographic}, // E0.6 [3] (๐Ÿš‘..๐Ÿš“) ambulance..police car + {0x1F694, 0x1F694, prExtendedPictographic}, // E0.7 [1] (๐Ÿš”) oncoming police car + {0x1F695, 0x1F695, prExtendedPictographic}, // E0.6 [1] (๐Ÿš•) taxi + {0x1F696, 0x1F696, prExtendedPictographic}, // E1.0 [1] (๐Ÿš–) oncoming taxi + {0x1F697, 0x1F697, prExtendedPictographic}, // E0.6 [1] (๐Ÿš—) automobile + {0x1F698, 0x1F698, prExtendedPictographic}, // E0.7 [1] (๐Ÿš˜) oncoming automobile + {0x1F699, 0x1F69A, prExtendedPictographic}, // E0.6 [2] (๐Ÿš™..๐Ÿšš) sport utility vehicle..delivery truck + {0x1F69B, 0x1F6A1, prExtendedPictographic}, // E1.0 [7] (๐Ÿš›..๐Ÿšก) articulated lorry..aerial tramway + {0x1F6A2, 0x1F6A2, prExtendedPictographic}, // E0.6 [1] (๐Ÿšข) ship + {0x1F6A3, 0x1F6A3, prExtendedPictographic}, // E1.0 [1] (๐Ÿšฃ) person rowing boat + {0x1F6A4, 0x1F6A5, prExtendedPictographic}, // E0.6 [2] (๐Ÿšค..๐Ÿšฅ) speedboat..horizontal traffic light + {0x1F6A6, 0x1F6A6, prExtendedPictographic}, // E1.0 [1] (๐Ÿšฆ) vertical traffic light + {0x1F6A7, 0x1F6AD, prExtendedPictographic}, // E0.6 [7] (๐Ÿšง..๐Ÿšญ) construction..no smoking + {0x1F6AE, 0x1F6B1, prExtendedPictographic}, // E1.0 [4] (๐Ÿšฎ..๐Ÿšฑ) litter in bin sign..non-potable water + {0x1F6B2, 0x1F6B2, prExtendedPictographic}, // E0.6 [1] (๐Ÿšฒ) bicycle + {0x1F6B3, 0x1F6B5, prExtendedPictographic}, // E1.0 [3] (๐Ÿšณ..๐Ÿšต) no bicycles..person mountain biking + {0x1F6B6, 0x1F6B6, prExtendedPictographic}, // E0.6 [1] (๐Ÿšถ) person walking + {0x1F6B7, 0x1F6B8, prExtendedPictographic}, // E1.0 [2] (๐Ÿšท..๐Ÿšธ) no pedestrians..children crossing + {0x1F6B9, 0x1F6BE, prExtendedPictographic}, // E0.6 [6] (๐Ÿšน..๐Ÿšพ) menโ€™s room..water closet + {0x1F6BF, 0x1F6BF, prExtendedPictographic}, // E1.0 [1] (๐Ÿšฟ) shower + {0x1F6C0, 0x1F6C0, prExtendedPictographic}, // E0.6 [1] (๐Ÿ›€) person taking bath + {0x1F6C1, 0x1F6C5, prExtendedPictographic}, // E1.0 [5] (๐Ÿ›..๐Ÿ›…) bathtub..left luggage + {0x1F6C6, 0x1F6CA, prExtendedPictographic}, // E0.0 [5] (๐Ÿ›†..๐Ÿ›Š) TRIANGLE WITH ROUNDED CORNERS..GIRLS SYMBOL + {0x1F6CB, 0x1F6CB, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›‹๏ธ) couch and lamp + {0x1F6CC, 0x1F6CC, prExtendedPictographic}, // E1.0 [1] (๐Ÿ›Œ) person in bed + {0x1F6CD, 0x1F6CF, prExtendedPictographic}, // E0.7 [3] (๐Ÿ›๏ธ..๐Ÿ›๏ธ) shopping bags..bed + {0x1F6D0, 0x1F6D0, prExtendedPictographic}, // E1.0 [1] (๐Ÿ›) place of worship + {0x1F6D1, 0x1F6D2, prExtendedPictographic}, // E3.0 [2] (๐Ÿ›‘..๐Ÿ›’) stop sign..shopping cart + {0x1F6D3, 0x1F6D4, prExtendedPictographic}, // E0.0 [2] (๐Ÿ›“..๐Ÿ›”) STUPA..PAGODA + {0x1F6D5, 0x1F6D5, prExtendedPictographic}, // E12.0 [1] (๐Ÿ›•) hindu temple + {0x1F6D6, 0x1F6D7, prExtendedPictographic}, // E13.0 [2] (๐Ÿ›–..๐Ÿ›—) hut..elevator + {0x1F6D8, 0x1F6DB, prExtendedPictographic}, // E0.0 [4] (๐Ÿ›˜..๐Ÿ››) .. + {0x1F6DC, 0x1F6DC, prExtendedPictographic}, // E15.0 [1] (๐Ÿ›œ) wireless + {0x1F6DD, 0x1F6DF, prExtendedPictographic}, // E14.0 [3] (๐Ÿ›..๐Ÿ›Ÿ) playground slide..ring buoy + {0x1F6E0, 0x1F6E5, prExtendedPictographic}, // E0.7 [6] (๐Ÿ› ๏ธ..๐Ÿ›ฅ๏ธ) hammer and wrench..motor boat + {0x1F6E6, 0x1F6E8, prExtendedPictographic}, // E0.0 [3] (๐Ÿ›ฆ..๐Ÿ›จ) UP-POINTING MILITARY AIRPLANE..UP-POINTING SMALL AIRPLANE + {0x1F6E9, 0x1F6E9, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›ฉ๏ธ) small airplane + {0x1F6EA, 0x1F6EA, prExtendedPictographic}, // E0.0 [1] (๐Ÿ›ช) NORTHEAST-POINTING AIRPLANE + {0x1F6EB, 0x1F6EC, prExtendedPictographic}, // E1.0 [2] (๐Ÿ›ซ..๐Ÿ›ฌ) airplane departure..airplane arrival + {0x1F6ED, 0x1F6EF, prExtendedPictographic}, // E0.0 [3] (๐Ÿ›ญ..๐Ÿ›ฏ) .. + {0x1F6F0, 0x1F6F0, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›ฐ๏ธ) satellite + {0x1F6F1, 0x1F6F2, prExtendedPictographic}, // E0.0 [2] (๐Ÿ›ฑ..๐Ÿ›ฒ) ONCOMING FIRE ENGINE..DIESEL LOCOMOTIVE + {0x1F6F3, 0x1F6F3, prExtendedPictographic}, // E0.7 [1] (๐Ÿ›ณ๏ธ) passenger ship + {0x1F6F4, 0x1F6F6, prExtendedPictographic}, // E3.0 [3] (๐Ÿ›ด..๐Ÿ›ถ) kick scooter..canoe + {0x1F6F7, 0x1F6F8, prExtendedPictographic}, // E5.0 [2] (๐Ÿ›ท..๐Ÿ›ธ) sled..flying saucer + {0x1F6F9, 0x1F6F9, prExtendedPictographic}, // E11.0 [1] (๐Ÿ›น) skateboard + {0x1F6FA, 0x1F6FA, prExtendedPictographic}, // E12.0 [1] (๐Ÿ›บ) auto rickshaw + {0x1F6FB, 0x1F6FC, prExtendedPictographic}, // E13.0 [2] (๐Ÿ›ป..๐Ÿ›ผ) pickup truck..roller skate + {0x1F6FD, 0x1F6FF, prExtendedPictographic}, // E0.0 [3] (๐Ÿ›ฝ..๐Ÿ›ฟ) .. + {0x1F774, 0x1F77F, prExtendedPictographic}, // E0.0 [12] (๐Ÿด..๐Ÿฟ) LOT OF FORTUNE..ORCUS + {0x1F7D5, 0x1F7DF, prExtendedPictographic}, // E0.0 [11] (๐ŸŸ•..๐ŸŸŸ) CIRCLED TRIANGLE.. + {0x1F7E0, 0x1F7EB, prExtendedPictographic}, // E12.0 [12] (๐ŸŸ ..๐ŸŸซ) orange circle..brown square + {0x1F7EC, 0x1F7EF, prExtendedPictographic}, // E0.0 [4] (๐ŸŸฌ..๐ŸŸฏ) .. + {0x1F7F0, 0x1F7F0, prExtendedPictographic}, // E14.0 [1] (๐ŸŸฐ) heavy equals sign + {0x1F7F1, 0x1F7FF, prExtendedPictographic}, // E0.0 [15] (๐ŸŸฑ..๐ŸŸฟ) .. + {0x1F80C, 0x1F80F, prExtendedPictographic}, // E0.0 [4] (๐Ÿ Œ..๐Ÿ ) .. + {0x1F848, 0x1F84F, prExtendedPictographic}, // E0.0 [8] (๐Ÿกˆ..๐Ÿก) .. + {0x1F85A, 0x1F85F, prExtendedPictographic}, // E0.0 [6] (๐Ÿกš..๐ŸกŸ) .. + {0x1F888, 0x1F88F, prExtendedPictographic}, // E0.0 [8] (๐Ÿขˆ..๐Ÿข) .. + {0x1F8AE, 0x1F8FF, prExtendedPictographic}, // E0.0 [82] (๐Ÿขฎ..๐Ÿฃฟ) .. + {0x1F90C, 0x1F90C, prExtendedPictographic}, // E13.0 [1] (๐ŸคŒ) pinched fingers + {0x1F90D, 0x1F90F, prExtendedPictographic}, // E12.0 [3] (๐Ÿค..๐Ÿค) white heart..pinching hand + {0x1F910, 0x1F918, prExtendedPictographic}, // E1.0 [9] (๐Ÿค..๐Ÿค˜) zipper-mouth face..sign of the horns + {0x1F919, 0x1F91E, prExtendedPictographic}, // E3.0 [6] (๐Ÿค™..๐Ÿคž) call me hand..crossed fingers + {0x1F91F, 0x1F91F, prExtendedPictographic}, // E5.0 [1] (๐ŸคŸ) love-you gesture + {0x1F920, 0x1F927, prExtendedPictographic}, // E3.0 [8] (๐Ÿค ..๐Ÿคง) cowboy hat face..sneezing face + {0x1F928, 0x1F92F, prExtendedPictographic}, // E5.0 [8] (๐Ÿคจ..๐Ÿคฏ) face with raised eyebrow..exploding head + {0x1F930, 0x1F930, prExtendedPictographic}, // E3.0 [1] (๐Ÿคฐ) pregnant woman + {0x1F931, 0x1F932, prExtendedPictographic}, // E5.0 [2] (๐Ÿคฑ..๐Ÿคฒ) breast-feeding..palms up together + {0x1F933, 0x1F93A, prExtendedPictographic}, // E3.0 [8] (๐Ÿคณ..๐Ÿคบ) selfie..person fencing + {0x1F93C, 0x1F93E, prExtendedPictographic}, // E3.0 [3] (๐Ÿคผ..๐Ÿคพ) people wrestling..person playing handball + {0x1F93F, 0x1F93F, prExtendedPictographic}, // E12.0 [1] (๐Ÿคฟ) diving mask + {0x1F940, 0x1F945, prExtendedPictographic}, // E3.0 [6] (๐Ÿฅ€..๐Ÿฅ…) wilted flower..goal net + {0x1F947, 0x1F94B, prExtendedPictographic}, // E3.0 [5] (๐Ÿฅ‡..๐Ÿฅ‹) 1st place medal..martial arts uniform + {0x1F94C, 0x1F94C, prExtendedPictographic}, // E5.0 [1] (๐ŸฅŒ) curling stone + {0x1F94D, 0x1F94F, prExtendedPictographic}, // E11.0 [3] (๐Ÿฅ..๐Ÿฅ) lacrosse..flying disc + {0x1F950, 0x1F95E, prExtendedPictographic}, // E3.0 [15] (๐Ÿฅ..๐Ÿฅž) croissant..pancakes + {0x1F95F, 0x1F96B, prExtendedPictographic}, // E5.0 [13] (๐ŸฅŸ..๐Ÿฅซ) dumpling..canned food + {0x1F96C, 0x1F970, prExtendedPictographic}, // E11.0 [5] (๐Ÿฅฌ..๐Ÿฅฐ) leafy green..smiling face with hearts + {0x1F971, 0x1F971, prExtendedPictographic}, // E12.0 [1] (๐Ÿฅฑ) yawning face + {0x1F972, 0x1F972, prExtendedPictographic}, // E13.0 [1] (๐Ÿฅฒ) smiling face with tear + {0x1F973, 0x1F976, prExtendedPictographic}, // E11.0 [4] (๐Ÿฅณ..๐Ÿฅถ) partying face..cold face + {0x1F977, 0x1F978, prExtendedPictographic}, // E13.0 [2] (๐Ÿฅท..๐Ÿฅธ) ninja..disguised face + {0x1F979, 0x1F979, prExtendedPictographic}, // E14.0 [1] (๐Ÿฅน) face holding back tears + {0x1F97A, 0x1F97A, prExtendedPictographic}, // E11.0 [1] (๐Ÿฅบ) pleading face + {0x1F97B, 0x1F97B, prExtendedPictographic}, // E12.0 [1] (๐Ÿฅป) sari + {0x1F97C, 0x1F97F, prExtendedPictographic}, // E11.0 [4] (๐Ÿฅผ..๐Ÿฅฟ) lab coat..flat shoe + {0x1F980, 0x1F984, prExtendedPictographic}, // E1.0 [5] (๐Ÿฆ€..๐Ÿฆ„) crab..unicorn + {0x1F985, 0x1F991, prExtendedPictographic}, // E3.0 [13] (๐Ÿฆ…..๐Ÿฆ‘) eagle..squid + {0x1F992, 0x1F997, prExtendedPictographic}, // E5.0 [6] (๐Ÿฆ’..๐Ÿฆ—) giraffe..cricket + {0x1F998, 0x1F9A2, prExtendedPictographic}, // E11.0 [11] (๐Ÿฆ˜..๐Ÿฆข) kangaroo..swan + {0x1F9A3, 0x1F9A4, prExtendedPictographic}, // E13.0 [2] (๐Ÿฆฃ..๐Ÿฆค) mammoth..dodo + {0x1F9A5, 0x1F9AA, prExtendedPictographic}, // E12.0 [6] (๐Ÿฆฅ..๐Ÿฆช) sloth..oyster + {0x1F9AB, 0x1F9AD, prExtendedPictographic}, // E13.0 [3] (๐Ÿฆซ..๐Ÿฆญ) beaver..seal + {0x1F9AE, 0x1F9AF, prExtendedPictographic}, // E12.0 [2] (๐Ÿฆฎ..๐Ÿฆฏ) guide dog..white cane + {0x1F9B0, 0x1F9B9, prExtendedPictographic}, // E11.0 [10] (๐Ÿฆฐ..๐Ÿฆน) red hair..supervillain + {0x1F9BA, 0x1F9BF, prExtendedPictographic}, // E12.0 [6] (๐Ÿฆบ..๐Ÿฆฟ) safety vest..mechanical leg + {0x1F9C0, 0x1F9C0, prExtendedPictographic}, // E1.0 [1] (๐Ÿง€) cheese wedge + {0x1F9C1, 0x1F9C2, prExtendedPictographic}, // E11.0 [2] (๐Ÿง..๐Ÿง‚) cupcake..salt + {0x1F9C3, 0x1F9CA, prExtendedPictographic}, // E12.0 [8] (๐Ÿงƒ..๐ŸงŠ) beverage box..ice + {0x1F9CB, 0x1F9CB, prExtendedPictographic}, // E13.0 [1] (๐Ÿง‹) bubble tea + {0x1F9CC, 0x1F9CC, prExtendedPictographic}, // E14.0 [1] (๐ŸงŒ) troll + {0x1F9CD, 0x1F9CF, prExtendedPictographic}, // E12.0 [3] (๐Ÿง..๐Ÿง) person standing..deaf person + {0x1F9D0, 0x1F9E6, prExtendedPictographic}, // E5.0 [23] (๐Ÿง..๐Ÿงฆ) face with monocle..socks + {0x1F9E7, 0x1F9FF, prExtendedPictographic}, // E11.0 [25] (๐Ÿงง..๐Ÿงฟ) red envelope..nazar amulet + {0x1FA00, 0x1FA6F, prExtendedPictographic}, // E0.0 [112] (๐Ÿจ€..๐Ÿฉฏ) NEUTRAL CHESS KING.. + {0x1FA70, 0x1FA73, prExtendedPictographic}, // E12.0 [4] (๐Ÿฉฐ..๐Ÿฉณ) ballet shoes..shorts + {0x1FA74, 0x1FA74, prExtendedPictographic}, // E13.0 [1] (๐Ÿฉด) thong sandal + {0x1FA75, 0x1FA77, prExtendedPictographic}, // E15.0 [3] (๐Ÿฉต..๐Ÿฉท) light blue heart..pink heart + {0x1FA78, 0x1FA7A, prExtendedPictographic}, // E12.0 [3] (๐Ÿฉธ..๐Ÿฉบ) drop of blood..stethoscope + {0x1FA7B, 0x1FA7C, prExtendedPictographic}, // E14.0 [2] (๐Ÿฉป..๐Ÿฉผ) x-ray..crutch + {0x1FA7D, 0x1FA7F, prExtendedPictographic}, // E0.0 [3] (๐Ÿฉฝ..๐Ÿฉฟ) .. + {0x1FA80, 0x1FA82, prExtendedPictographic}, // E12.0 [3] (๐Ÿช€..๐Ÿช‚) yo-yo..parachute + {0x1FA83, 0x1FA86, prExtendedPictographic}, // E13.0 [4] (๐Ÿชƒ..๐Ÿช†) boomerang..nesting dolls + {0x1FA87, 0x1FA88, prExtendedPictographic}, // E15.0 [2] (๐Ÿช‡..๐Ÿชˆ) maracas..flute + {0x1FA89, 0x1FA8F, prExtendedPictographic}, // E0.0 [7] (๐Ÿช‰..๐Ÿช) .. + {0x1FA90, 0x1FA95, prExtendedPictographic}, // E12.0 [6] (๐Ÿช..๐Ÿช•) ringed planet..banjo + {0x1FA96, 0x1FAA8, prExtendedPictographic}, // E13.0 [19] (๐Ÿช–..๐Ÿชจ) military helmet..rock + {0x1FAA9, 0x1FAAC, prExtendedPictographic}, // E14.0 [4] (๐Ÿชฉ..๐Ÿชฌ) mirror ball..hamsa + {0x1FAAD, 0x1FAAF, prExtendedPictographic}, // E15.0 [3] (๐Ÿชญ..๐Ÿชฏ) folding hand fan..khanda + {0x1FAB0, 0x1FAB6, prExtendedPictographic}, // E13.0 [7] (๐Ÿชฐ..๐Ÿชถ) fly..feather + {0x1FAB7, 0x1FABA, prExtendedPictographic}, // E14.0 [4] (๐Ÿชท..๐Ÿชบ) lotus..nest with eggs + {0x1FABB, 0x1FABD, prExtendedPictographic}, // E15.0 [3] (๐Ÿชป..๐Ÿชฝ) hyacinth..wing + {0x1FABE, 0x1FABE, prExtendedPictographic}, // E0.0 [1] (๐Ÿชพ) + {0x1FABF, 0x1FABF, prExtendedPictographic}, // E15.0 [1] (๐Ÿชฟ) goose + {0x1FAC0, 0x1FAC2, prExtendedPictographic}, // E13.0 [3] (๐Ÿซ€..๐Ÿซ‚) anatomical heart..people hugging + {0x1FAC3, 0x1FAC5, prExtendedPictographic}, // E14.0 [3] (๐Ÿซƒ..๐Ÿซ…) pregnant man..person with crown + {0x1FAC6, 0x1FACD, prExtendedPictographic}, // E0.0 [8] (๐Ÿซ†..๐Ÿซ) .. + {0x1FACE, 0x1FACF, prExtendedPictographic}, // E15.0 [2] (๐ŸซŽ..๐Ÿซ) moose..donkey + {0x1FAD0, 0x1FAD6, prExtendedPictographic}, // E13.0 [7] (๐Ÿซ..๐Ÿซ–) blueberries..teapot + {0x1FAD7, 0x1FAD9, prExtendedPictographic}, // E14.0 [3] (๐Ÿซ—..๐Ÿซ™) pouring liquid..jar + {0x1FADA, 0x1FADB, prExtendedPictographic}, // E15.0 [2] (๐Ÿซš..๐Ÿซ›) ginger root..pea pod + {0x1FADC, 0x1FADF, prExtendedPictographic}, // E0.0 [4] (๐Ÿซœ..๐ŸซŸ) .. + {0x1FAE0, 0x1FAE7, prExtendedPictographic}, // E14.0 [8] (๐Ÿซ ..๐Ÿซง) melting face..bubbles + {0x1FAE8, 0x1FAE8, prExtendedPictographic}, // E15.0 [1] (๐Ÿซจ) shaking face + {0x1FAE9, 0x1FAEF, prExtendedPictographic}, // E0.0 [7] (๐Ÿซฉ..๐Ÿซฏ) .. + {0x1FAF0, 0x1FAF6, prExtendedPictographic}, // E14.0 [7] (๐Ÿซฐ..๐Ÿซถ) hand with index finger and thumb crossed..heart hands + {0x1FAF7, 0x1FAF8, prExtendedPictographic}, // E15.0 [2] (๐Ÿซท..๐Ÿซธ) leftwards pushing hand..rightwards pushing hand + {0x1FAF9, 0x1FAFF, prExtendedPictographic}, // E0.0 [7] (๐Ÿซน..๐Ÿซฟ) .. + {0x1FBF0, 0x1FBF9, prNumeric}, // Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE + {0x1FC00, 0x1FFFD, prExtendedPictographic}, // E0.0[1022] (๐Ÿฐ€..๐Ÿฟฝ) .. + {0xE0001, 0xE0001, prFormat}, // Cf LANGUAGE TAG + {0xE0020, 0xE007F, prExtend}, // Cf [96] TAG SPACE..CANCEL TAG + {0xE0100, 0xE01EF, prExtend}, // Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 +} diff --git a/vendor/github.com/rivo/uniseg/wordrules.go b/vendor/github.com/rivo/uniseg/wordrules.go new file mode 100644 index 0000000000..57a8c68311 --- /dev/null +++ b/vendor/github.com/rivo/uniseg/wordrules.go @@ -0,0 +1,282 @@ +package uniseg + +import "unicode/utf8" + +// The states of the word break parser. +const ( + wbAny = iota + wbCR + wbLF + wbNewline + wbWSegSpace + wbHebrewLetter + wbALetter + wbWB7 + wbWB7c + wbNumeric + wbWB11 + wbKatakana + wbExtendNumLet + wbOddRI + wbEvenRI + wbZWJBit = 16 // This bit is set for any states followed by at least one zero-width joiner (see WB4 and WB3c). +) + +// wbTransitions implements the word break parser's state transitions. It's +// anologous to [grTransitions], see comments there for details. +// +// Unicode version 15.0.0. +func wbTransitions(state, prop int) (newState int, wordBreak bool, rule int) { + switch uint64(state) | uint64(prop)<<32 { + // WB3b. + case wbAny | prNewline<<32: + return wbNewline, true, 32 + case wbAny | prCR<<32: + return wbCR, true, 32 + case wbAny | prLF<<32: + return wbLF, true, 32 + + // WB3a. + case wbNewline | prAny<<32: + return wbAny, true, 31 + case wbCR | prAny<<32: + return wbAny, true, 31 + case wbLF | prAny<<32: + return wbAny, true, 31 + + // WB3. + case wbCR | prLF<<32: + return wbLF, false, 30 + + // WB3d. + case wbAny | prWSegSpace<<32: + return wbWSegSpace, true, 9990 + case wbWSegSpace | prWSegSpace<<32: + return wbWSegSpace, false, 34 + + // WB5. + case wbAny | prALetter<<32: + return wbALetter, true, 9990 + case wbAny | prHebrewLetter<<32: + return wbHebrewLetter, true, 9990 + case wbALetter | prALetter<<32: + return wbALetter, false, 50 + case wbALetter | prHebrewLetter<<32: + return wbHebrewLetter, false, 50 + case wbHebrewLetter | prALetter<<32: + return wbALetter, false, 50 + case wbHebrewLetter | prHebrewLetter<<32: + return wbHebrewLetter, false, 50 + + // WB7. Transitions to wbWB7 handled by transitionWordBreakState(). + case wbWB7 | prALetter<<32: + return wbALetter, false, 70 + case wbWB7 | prHebrewLetter<<32: + return wbHebrewLetter, false, 70 + + // WB7a. + case wbHebrewLetter | prSingleQuote<<32: + return wbAny, false, 71 + + // WB7c. Transitions to wbWB7c handled by transitionWordBreakState(). + case wbWB7c | prHebrewLetter<<32: + return wbHebrewLetter, false, 73 + + // WB8. + case wbAny | prNumeric<<32: + return wbNumeric, true, 9990 + case wbNumeric | prNumeric<<32: + return wbNumeric, false, 80 + + // WB9. + case wbALetter | prNumeric<<32: + return wbNumeric, false, 90 + case wbHebrewLetter | prNumeric<<32: + return wbNumeric, false, 90 + + // WB10. + case wbNumeric | prALetter<<32: + return wbALetter, false, 100 + case wbNumeric | prHebrewLetter<<32: + return wbHebrewLetter, false, 100 + + // WB11. Transitions to wbWB11 handled by transitionWordBreakState(). + case wbWB11 | prNumeric<<32: + return wbNumeric, false, 110 + + // WB13. + case wbAny | prKatakana<<32: + return wbKatakana, true, 9990 + case wbKatakana | prKatakana<<32: + return wbKatakana, false, 130 + + // WB13a. + case wbAny | prExtendNumLet<<32: + return wbExtendNumLet, true, 9990 + case wbALetter | prExtendNumLet<<32: + return wbExtendNumLet, false, 131 + case wbHebrewLetter | prExtendNumLet<<32: + return wbExtendNumLet, false, 131 + case wbNumeric | prExtendNumLet<<32: + return wbExtendNumLet, false, 131 + case wbKatakana | prExtendNumLet<<32: + return wbExtendNumLet, false, 131 + case wbExtendNumLet | prExtendNumLet<<32: + return wbExtendNumLet, false, 131 + + // WB13b. + case wbExtendNumLet | prALetter<<32: + return wbALetter, false, 132 + case wbExtendNumLet | prHebrewLetter<<32: + return wbHebrewLetter, false, 132 + case wbExtendNumLet | prNumeric<<32: + return wbNumeric, false, 132 + case wbExtendNumLet | prKatakana<<32: + return wbKatakana, false, 132 + + default: + return -1, false, -1 + } +} + +// transitionWordBreakState determines the new state of the word break parser +// given the current state and the next code point. It also returns whether a +// word boundary was detected. If more than one code point is needed to +// determine the new state, the byte slice or the string starting after rune "r" +// can be used (whichever is not nil or empty) for further lookups. +func transitionWordBreakState(state int, r rune, b []byte, str string) (newState int, wordBreak bool) { + // Determine the property of the next character. + nextProperty := property(workBreakCodePoints, r) + + // "Replacing Ignore Rules". + if nextProperty == prZWJ { + // WB4 (for zero-width joiners). + if state == wbNewline || state == wbCR || state == wbLF { + return wbAny | wbZWJBit, true // Make sure we don't apply WB4 to WB3a. + } + if state < 0 { + return wbAny | wbZWJBit, false + } + return state | wbZWJBit, false + } else if nextProperty == prExtend || nextProperty == prFormat { + // WB4 (for Extend and Format). + if state == wbNewline || state == wbCR || state == wbLF { + return wbAny, true // Make sure we don't apply WB4 to WB3a. + } + if state == wbWSegSpace || state == wbAny|wbZWJBit { + return wbAny, false // We don't break but this is also not WB3d or WB3c. + } + if state < 0 { + return wbAny, false + } + return state, false + } else if nextProperty == prExtendedPictographic && state >= 0 && state&wbZWJBit != 0 { + // WB3c. + return wbAny, false + } + if state >= 0 { + state = state &^ wbZWJBit + } + + // Find the applicable transition in the table. + var rule int + newState, wordBreak, rule = wbTransitions(state, nextProperty) + if newState < 0 { + // No specific transition found. Try the less specific ones. + anyPropState, anyPropWordBreak, anyPropRule := wbTransitions(state, prAny) + anyStateState, anyStateWordBreak, anyStateRule := wbTransitions(wbAny, nextProperty) + if anyPropState >= 0 && anyStateState >= 0 { + // Both apply. We'll use a mix (see comments for grTransitions). + newState, wordBreak, rule = anyStateState, anyStateWordBreak, anyStateRule + if anyPropRule < anyStateRule { + wordBreak, rule = anyPropWordBreak, anyPropRule + } + } else if anyPropState >= 0 { + // We only have a specific state. + newState, wordBreak, rule = anyPropState, anyPropWordBreak, anyPropRule + // This branch will probably never be reached because okAnyState will + // always be true given the current transition map. But we keep it here + // for future modifications to the transition map where this may not be + // true anymore. + } else if anyStateState >= 0 { + // We only have a specific property. + newState, wordBreak, rule = anyStateState, anyStateWordBreak, anyStateRule + } else { + // No known transition. WB999: Any รท Any. + newState, wordBreak, rule = wbAny, true, 9990 + } + } + + // For those rules that need to look up runes further in the string, we + // determine the property after nextProperty, skipping over Format, Extend, + // and ZWJ (according to WB4). It's -1 if not needed, if such a rune cannot + // be determined (because the text ends or the rune is faulty). + farProperty := -1 + if rule > 60 && + (state == wbALetter || state == wbHebrewLetter || state == wbNumeric) && + (nextProperty == prMidLetter || nextProperty == prMidNumLet || nextProperty == prSingleQuote || // WB6. + nextProperty == prDoubleQuote || // WB7b. + nextProperty == prMidNum) { // WB12. + for { + var ( + r rune + length int + ) + if b != nil { // Byte slice version. + r, length = utf8.DecodeRune(b) + b = b[length:] + } else { // String version. + r, length = utf8.DecodeRuneInString(str) + str = str[length:] + } + if r == utf8.RuneError { + break + } + prop := property(workBreakCodePoints, r) + if prop == prExtend || prop == prFormat || prop == prZWJ { + continue + } + farProperty = prop + break + } + } + + // WB6. + if rule > 60 && + (state == wbALetter || state == wbHebrewLetter) && + (nextProperty == prMidLetter || nextProperty == prMidNumLet || nextProperty == prSingleQuote) && + (farProperty == prALetter || farProperty == prHebrewLetter) { + return wbWB7, false + } + + // WB7b. + if rule > 72 && + state == wbHebrewLetter && + nextProperty == prDoubleQuote && + farProperty == prHebrewLetter { + return wbWB7c, false + } + + // WB12. + if rule > 120 && + state == wbNumeric && + (nextProperty == prMidNum || nextProperty == prMidNumLet || nextProperty == prSingleQuote) && + farProperty == prNumeric { + return wbWB11, false + } + + // WB15 and WB16. + if newState == wbAny && nextProperty == prRegionalIndicator { + if state != wbOddRI && state != wbEvenRI { // Includes state == -1. + // Transition into the first RI. + return wbOddRI, true + } + if state == wbOddRI { + // Don't break pairs of Regional Indicators. + return wbEvenRI, false + } + return wbOddRI, true // We can break after a pair. + } + + return +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c2ae14befe..cc1b15850d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -136,6 +136,10 @@ github.com/alicebob/miniredis/v2/metro github.com/alicebob/miniredis/v2/proto github.com/alicebob/miniredis/v2/server github.com/alicebob/miniredis/v2/size +# github.com/andybalholm/brotli v1.1.0 +## explicit; go 1.13 +github.com/andybalholm/brotli +github.com/andybalholm/brotli/matchfinder # github.com/armon/go-metrics v0.4.1 ## explicit; go 1.12 github.com/armon/go-metrics @@ -273,6 +277,9 @@ github.com/aws/smithy-go/rand github.com/aws/smithy-go/time github.com/aws/smithy-go/transport/http github.com/aws/smithy-go/transport/http/internal/io +# github.com/axiomhq/hyperloglog v0.2.5 +## explicit; go 1.23 +github.com/axiomhq/hyperloglog # github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 ## explicit; go 1.20 github.com/bboreham/go-loser @@ -621,6 +628,9 @@ github.com/json-iterator/go # github.com/julienschmidt/httprouter v1.3.0 ## explicit; go 1.7 github.com/julienschmidt/httprouter +# github.com/kamstrup/intmap v0.5.1 +## explicit; go 1.23 +github.com/kamstrup/intmap # github.com/klauspost/compress v1.17.11 ## explicit; go 1.21 github.com/klauspost/compress @@ -668,6 +678,9 @@ github.com/mattn/go-colorable # github.com/mattn/go-isatty v0.0.20 ## explicit; go 1.15 github.com/mattn/go-isatty +# github.com/mattn/go-runewidth v0.0.15 +## explicit; go 1.9 +github.com/mattn/go-runewidth # github.com/matttproud/golang_protobuf_extensions v1.0.4 ## explicit; go 1.9 github.com/matttproud/golang_protobuf_extensions/pbutil @@ -733,6 +746,9 @@ github.com/oklog/run # github.com/oklog/ulid v1.3.1 ## explicit github.com/oklog/ulid +# github.com/olekukonko/tablewriter v0.0.5 +## explicit; go 1.12 +github.com/olekukonko/tablewriter # github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.116.0 ## explicit; go 1.22.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics/identity @@ -763,6 +779,42 @@ github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log github.com/opentracing/opentracing-go/mocktracer +# github.com/parquet-go/parquet-go v0.25.0 => github.com/alanprot/parquet-go v0.25.0-rc.1 +## explicit; go 1.22 +github.com/parquet-go/parquet-go +github.com/parquet-go/parquet-go/bloom +github.com/parquet-go/parquet-go/bloom/xxhash +github.com/parquet-go/parquet-go/compress +github.com/parquet-go/parquet-go/compress/brotli +github.com/parquet-go/parquet-go/compress/gzip +github.com/parquet-go/parquet-go/compress/lz4 +github.com/parquet-go/parquet-go/compress/snappy +github.com/parquet-go/parquet-go/compress/uncompressed +github.com/parquet-go/parquet-go/compress/zstd +github.com/parquet-go/parquet-go/deprecated +github.com/parquet-go/parquet-go/encoding +github.com/parquet-go/parquet-go/encoding/bitpacked +github.com/parquet-go/parquet-go/encoding/bytestreamsplit +github.com/parquet-go/parquet-go/encoding/delta +github.com/parquet-go/parquet-go/encoding/plain +github.com/parquet-go/parquet-go/encoding/rle +github.com/parquet-go/parquet-go/encoding/thrift +github.com/parquet-go/parquet-go/format +github.com/parquet-go/parquet-go/hashprobe +github.com/parquet-go/parquet-go/hashprobe/aeshash +github.com/parquet-go/parquet-go/hashprobe/wyhash +github.com/parquet-go/parquet-go/internal/bitpack +github.com/parquet-go/parquet-go/internal/bytealg +github.com/parquet-go/parquet-go/internal/debug +github.com/parquet-go/parquet-go/internal/unsafecast +github.com/parquet-go/parquet-go/sparse +# github.com/pierrec/lz4/v4 v4.1.21 +## explicit; go 1.14 +github.com/pierrec/lz4/v4 +github.com/pierrec/lz4/v4/internal/lz4block +github.com/pierrec/lz4/v4/internal/lz4errors +github.com/pierrec/lz4/v4/internal/lz4stream +github.com/pierrec/lz4/v4/internal/xxh32 # github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c ## explicit; go 1.14 github.com/pkg/browser @@ -936,6 +988,9 @@ github.com/prometheus/sigv4 github.com/redis/rueidis github.com/redis/rueidis/internal/cmds github.com/redis/rueidis/internal/util +# github.com/rivo/uniseg v0.4.7 +## explicit; go 1.18 +github.com/rivo/uniseg # github.com/rs/cors v1.11.1 ## explicit; go 1.13 github.com/rs/cors @@ -1657,3 +1712,4 @@ k8s.io/utils/clock # github.com/google/gnostic => github.com/googleapis/gnostic v0.6.9 # gopkg.in/alecthomas/kingpin.v2 => github.com/alecthomas/kingpin v1.3.8-0.20210301060133-17f40c25f497 # google.golang.org/grpc => google.golang.org/grpc v1.65.0 +# github.com/parquet-go/parquet-go => github.com/alanprot/parquet-go v0.25.0-rc.1