Install this package:
emerge -a dev-python/tokenizersIf the package is masked, you can unmask it using the autounmask tool or standard emerge options:
autounmask dev-python/tokenizersOr alternatively:
emerge --autounmask-write -a dev-python/tokenizersNo metadata available.
| Type | File | Size | Versions |
|---|---|---|---|
| DIST | tokenizers-0.22.2.tar.gz | 372115 bytes | 0.22.2 |
| DIST | tokenizers-0.23.1.tar.gz | 365748 bytes | 0.23.1 |
| Type | File | Size |
|---|---|---|
| DIST | ahash-0.8.12.crate | 43413 bytes |
| DIST | aho-corasick-1.1.4.crate | 184015 bytes |
| DIST | anstream-0.6.21.crate | 29516 bytes |
| DIST | anstream-1.0.0.crate | 28916 bytes |
| DIST | anstyle-1.0.13.crate | 17651 bytes |
| DIST | anstyle-1.0.14.crate | 17787 bytes |
| DIST | anstyle-parse-0.2.7.crate | 21707 bytes |
| DIST | anstyle-parse-1.0.0.crate | 22042 bytes |
| DIST | anstyle-query-1.1.5.crate | 10264 bytes |
| DIST | anstyle-wincon-3.0.11.crate | 12638 bytes |
| DIST | anyhow-1.0.102.crate | 48658 bytes |
| DIST | autocfg-1.5.0.crate | 18729 bytes |
| DIST | base64-0.13.1.crate | 61002 bytes |
| DIST | bitflags-2.10.0.crate | 48427 bytes |
| DIST | bitflags-2.11.1.crate | 49082 bytes |
| DIST | bumpalo-3.19.0.crate | 96414 bytes |
| DIST | bumpalo-3.20.2.crate | 97696 bytes |
| DIST | castaway-0.2.4.crate | 12546 bytes |
| DIST | cc-1.2.48.crate | 92458 bytes |
| DIST | cc-1.2.61.crate | 97163 bytes |
| DIST | cfg-if-1.0.4.crate | 9360 bytes |
| DIST | colorchoice-1.0.4.crate | 8196 bytes |
| DIST | colorchoice-1.0.5.crate | 8187 bytes |
| DIST | compact_str-0.9.0.crate | 72135 bytes |
| DIST | console-0.16.1.crate | 39129 bytes |
| DIST | console-0.16.3.crate | 34041 bytes |
| DIST | crossbeam-deque-0.8.6.crate | 22471 bytes |
| DIST | crossbeam-epoch-0.9.18.crate | 46875 bytes |
| DIST | crossbeam-utils-0.8.21.crate | 42691 bytes |
| DIST | daachorse-1.0.1.crate | 48583 bytes |
| DIST | darling-0.20.11.crate | 37614 bytes |
| DIST | darling_core-0.20.11.crate | 68006 bytes |
| DIST | darling_macro-0.20.11.crate | 2532 bytes |
| DIST | dary_heap-0.3.8.crate | 26544 bytes |
| DIST | dary_heap-0.3.9.crate | 27054 bytes |
| DIST | derive_builder-0.20.2.crate | 36661 bytes |
| DIST | derive_builder_core-0.20.2.crate | 31397 bytes |
| DIST | derive_builder_macro-0.20.2.crate | 6360 bytes |
| DIST | either-1.15.0.crate | 20114 bytes |
| DIST | encode_unicode-1.0.0.crate | 56986 bytes |
| DIST | env_filter-0.1.4.crate | 15281 bytes |
| DIST | env_filter-1.0.1.crate | 15045 bytes |
| DIST | env_logger-0.11.10.crate | 31613 bytes |
| DIST | env_logger-0.11.8.crate | 32538 bytes |
| DIST | equivalent-1.0.2.crate | 7419 bytes |
| DIST | errno-0.3.14.crate | 12002 bytes |
| DIST | esaxx-rs-0.1.10.crate | 175210 bytes |
| DIST | fastrand-2.3.0.crate | 15076 bytes |
| DIST | fastrand-2.4.1.crate | 20773 bytes |
| DIST | find-msvc-tools-0.1.5.crate | 30942 bytes |
| DIST | find-msvc-tools-0.1.9.crate | 31493 bytes |
| DIST | fnv-1.0.7.crate | 11266 bytes |
| DIST | foldhash-0.1.5.crate | 21901 bytes |
| DIST | futures-0.3.31.crate | 54953 bytes |
| DIST | futures-channel-0.3.31.crate | 31971 bytes |
| DIST | futures-channel-0.3.32.crate | 33213 bytes |
| DIST | futures-core-0.3.31.crate | 14318 bytes |
| DIST | futures-core-0.3.32.crate | 14684 bytes |
| DIST | futures-executor-0.3.31.crate | 17965 bytes |
| DIST | futures-io-0.3.31.crate | 9047 bytes |
| DIST | futures-macro-0.3.31.crate | 11341 bytes |
| DIST | futures-macro-0.3.32.crate | 11929 bytes |
| DIST | futures-sink-0.3.31.crate | 7958 bytes |
| DIST | futures-task-0.3.31.crate | 11217 bytes |
| DIST | futures-task-0.3.32.crate | 11377 bytes |
| DIST | futures-util-0.3.31.crate | 162124 bytes |
| DIST | futures-util-0.3.32.crate | 168797 bytes |
| DIST | getrandom-0.3.4.crate | 50932 bytes |
| DIST | getrandom-0.4.2.crate | 53905 bytes |
| DIST | hashbrown-0.15.5.crate | 140908 bytes |
| DIST | hashbrown-0.17.0.crate | 154597 bytes |
| DIST | heck-0.5.0.crate | 11517 bytes |
| DIST | id-arena-2.3.0.crate | 12883 bytes |
| DIST | ident_case-1.0.1.crate | 3492 bytes |
| DIST | indexmap-2.14.0.crate | 104085 bytes |
| DIST | indicatif-0.18.3.crate | 66504 bytes |
| DIST | indicatif-0.18.4.crate | 68731 bytes |
| DIST | indoc-2.0.7.crate | 17184 bytes |
| DIST | is_terminal_polyfill-1.70.2.crate | 7548 bytes |
| DIST | itertools-0.14.0.crate | 152715 bytes |
| DIST | itoa-1.0.15.crate | 11231 bytes |
| DIST | itoa-1.0.18.crate | 15935 bytes |
| DIST | jiff-0.2.16.crate | 731814 bytes |
| DIST | jiff-0.2.24.crate | 760237 bytes |
| DIST | jiff-static-0.2.16.crate | 77672 bytes |
| DIST | jiff-static-0.2.24.crate | 83747 bytes |
| DIST | js-sys-0.3.83.crate | 56425 bytes |
| DIST | js-sys-0.3.95.crate | 104031 bytes |
| DIST | leb128fmt-0.1.0.crate | 14138 bytes |
| DIST | libc-0.2.177.crate | 792045 bytes |
| DIST | libc-0.2.186.crate | 821883 bytes |
| DIST | linux-raw-sys-0.11.0.crate | 2659624 bytes |
| DIST | linux-raw-sys-0.12.1.crate | 3006116 bytes |
| DIST | log-0.4.28.crate | 51131 bytes |
| DIST | log-0.4.29.crate | 51515 bytes |
| DIST | macro_rules_attribute-0.2.2.crate | 15330 bytes |
| DIST | macro_rules_attribute-proc_macro-0.2.2.crate | 8271 bytes |
| DIST | matrixmultiply-0.3.10.crate | 58170 bytes |
| DIST | memchr-2.7.6.crate | 97616 bytes |
| DIST | memchr-2.8.0.crate | 97780 bytes |
| DIST | memoffset-0.9.1.crate | 9032 bytes |
| DIST | minimal-lexical-0.2.1.crate | 94841 bytes |
| DIST | mio-1.1.0.crate | 105232 bytes |
| DIST | mio-1.2.0.crate | 106403 bytes |
| DIST | monostate-0.1.18.crate | 15992 bytes |
| DIST | monostate-impl-0.1.18.crate | 7862 bytes |
| DIST | ndarray-0.16.1.crate | 291026 bytes |
| DIST | ndarray-0.17.2.crate | 243631 bytes |
| DIST | nom-7.1.3.crate | 117570 bytes |
| DIST | num-complex-0.4.6.crate | 30352 bytes |
| DIST | num-integer-0.1.46.crate | 22331 bytes |
| DIST | num-traits-0.2.19.crate | 51631 bytes |
| DIST | numpy-0.26.0.crate | 82705 bytes |
| DIST | numpy-0.28.0.crate | 80455 bytes |
| DIST | once_cell-1.21.3.crate | 34534 bytes |
| DIST | once_cell-1.21.4.crate | 35010 bytes |
| DIST | once_cell_polyfill-1.70.2.crate | 7448 bytes |
| DIST | onig-6.5.1.crate | 32394 bytes |
| DIST | onig-6.5.3.crate | 32075 bytes |
| DIST | onig_sys-69.9.1.crate | 656378 bytes |
| DIST | onig_sys-69.9.3.crate | 657340 bytes |
| DIST | paste-1.0.15.crate | 18374 bytes |
| DIST | pin-project-lite-0.2.16.crate | 30504 bytes |
| DIST | pin-project-lite-0.2.17.crate | 31034 bytes |
| DIST | pin-utils-0.1.0.crate | 7580 bytes |
| DIST | pkg-config-0.3.32.crate | 21370 bytes |
| DIST | pkg-config-0.3.33.crate | 21590 bytes |
| DIST | portable-atomic-1.11.1.crate | 185506 bytes |
| DIST | portable-atomic-1.13.1.crate | 197001 bytes |
| DIST | portable-atomic-util-0.2.4.crate | 47043 bytes |
| DIST | portable-atomic-util-0.2.7.crate | 53577 bytes |
| DIST | ppv-lite86-0.2.21.crate | 22522 bytes |
| DIST | prettyplease-0.2.37.crate | 72033 bytes |
| DIST | proc-macro2-1.0.103.crate | 60024 bytes |
| DIST | proc-macro2-1.0.106.crate | 59765 bytes |
| DIST | pyo3-0.26.0.crate | 1151579 bytes |
| DIST | pyo3-0.28.2.crate | 1202843 bytes |
| DIST | pyo3-async-runtimes-0.26.0.crate | 52499 bytes |
| DIST | pyo3-async-runtimes-0.28.0.crate | 51548 bytes |
| DIST | pyo3-build-config-0.26.0.crate | 34309 bytes |
| DIST | pyo3-build-config-0.28.2.crate | 35958 bytes |
| DIST | pyo3-ffi-0.26.0.crate | 78247 bytes |
| DIST | pyo3-ffi-0.28.2.crate | 80837 bytes |
| DIST | pyo3-macros-0.26.0.crate | 8906 bytes |
| DIST | pyo3-macros-0.28.2.crate | 8960 bytes |
| DIST | pyo3-macros-backend-0.26.0.crate | 81809 bytes |
| DIST | pyo3-macros-backend-0.28.2.crate | 87939 bytes |
| DIST | quote-1.0.42.crate | 31504 bytes |
| DIST | quote-1.0.45.crate | 31564 bytes |
| DIST | r-efi-5.3.0.crate | 64532 bytes |
| DIST | r-efi-6.0.0.crate | 65303 bytes |
| DIST | rand-0.9.2.crate | 99930 bytes |
| DIST | rand-0.9.4.crate | 99786 bytes |
| DIST | rand_chacha-0.9.0.crate | 18258 bytes |
| DIST | rand_core-0.9.3.crate | 24543 bytes |
| DIST | rand_core-0.9.5.crate | 24129 bytes |
| DIST | rawpointer-0.2.1.crate | 7490 bytes |
| DIST | rayon-1.11.0.crate | 182470 bytes |
| DIST | rayon-1.12.0.crate | 183391 bytes |
| DIST | rayon-cond-0.4.0.crate | 10659 bytes |
| DIST | rayon-core-1.13.0.crate | 73151 bytes |
| DIST | regex-1.12.2.crate | 163843 bytes |
| DIST | regex-1.12.3.crate | 153392 bytes |
| DIST | regex-automata-0.4.13.crate | 625250 bytes |
| DIST | regex-automata-0.4.14.crate | 618012 bytes |
| DIST | regex-syntax-0.8.10.crate | 358781 bytes |
| DIST | regex-syntax-0.8.8.crate | 359141 bytes |
| DIST | rustc-hash-2.1.1.crate | 14154 bytes |
| DIST | rustc-hash-2.1.2.crate | 14754 bytes |
| DIST | rustix-1.1.2.crate | 422717 bytes |
| DIST | rustix-1.1.4.crate | 425241 bytes |
| DIST | rustversion-1.0.22.crate | 21096 bytes |
| DIST | ryu-1.0.20.crate | 48738 bytes |
| DIST | ryu-1.0.23.crate | 52823 bytes |
| DIST | semver-1.0.28.crate | 33064 bytes |
| DIST | serde-1.0.228.crate | 83652 bytes |
| DIST | serde_core-1.0.228.crate | 63111 bytes |
| DIST | serde_derive-1.0.228.crate | 59605 bytes |
| DIST | serde_json-1.0.145.crate | 155748 bytes |
| DIST | serde_json-1.0.149.crate | 155994 bytes |
| DIST | shlex-1.3.0.crate | 18713 bytes |
| DIST | signal-hook-registry-1.4.7.crate | 19491 bytes |
| DIST | signal-hook-registry-1.4.8.crate | 21668 bytes |
| DIST | slab-0.4.11.crate | 18549 bytes |
| DIST | slab-0.4.12.crate | 19080 bytes |
| DIST | smallvec-1.15.1.crate | 38116 bytes |
| DIST | spm_precompiled-0.1.4.crate | 557527 bytes |
| DIST | static_assertions-1.1.0.crate | 18480 bytes |
| DIST | strsim-0.11.1.crate | 14266 bytes |
| DIST | syn-2.0.111.crate | 302117 bytes |
| DIST | syn-2.0.117.crate | 306011 bytes |
| DIST | target-lexicon-0.13.3.crate | 28498 bytes |
| DIST | target-lexicon-0.13.5.crate | 26928 bytes |
| DIST | tempfile-3.23.0.crate | 43063 bytes |
| DIST | tempfile-3.27.0.crate | 44780 bytes |
| DIST | thiserror-2.0.17.crate | 28857 bytes |
| DIST | thiserror-2.0.18.crate | 28875 bytes |
| DIST | thiserror-impl-2.0.17.crate | 21344 bytes |
| DIST | thiserror-impl-2.0.18.crate | 21413 bytes |
| DIST | tokio-1.48.0.crate | 843434 bytes |
| DIST | tokio-1.52.1.crate | 900270 bytes |
| DIST | tokio-macros-2.6.0.crate | 16505 bytes |
| DIST | tokio-macros-2.7.0.crate | 15859 bytes |
| DIST | unicode-ident-1.0.22.crate | 47919 bytes |
| DIST | unicode-ident-1.0.24.crate | 49298 bytes |
| DIST | unicode-normalization-alignments-0.1.12.crate | 91546 bytes |
| DIST | unicode-segmentation-1.12.0.crate | 106323 bytes |
| DIST | unicode-segmentation-1.13.2.crate | 111829 bytes |
| DIST | unicode-width-0.2.2.crate | 282768 bytes |
| DIST | unicode-xid-0.2.6.crate | 15744 bytes |
| DIST | unicode_categories-0.1.1.crate | 87298 bytes |
| DIST | unindent-0.2.4.crate | 7422 bytes |
| DIST | unit-prefix-0.5.2.crate | 8041 bytes |
| DIST | utf8parse-0.2.2.crate | 13499 bytes |
| DIST | version_check-0.9.5.crate | 15554 bytes |
| DIST | wasi-0.11.1+wasi-snapshot-preview1.crate | 28477 bytes |
| DIST | wasip2-1.0.1+wasi-0.2.4.crate | 132087 bytes |
| DIST | wasip2-1.0.3+wasi-0.2.9.crate | 134512 bytes |
| DIST | wasip3-0.4.0+wasi-0.3.0-rc-2026-01-06.crate | 123354 bytes |
| DIST | wasm-bindgen-0.2.106.crate | 48417 bytes |
| DIST | wasm-bindgen-0.2.118.crate | 61149 bytes |
| DIST | wasm-bindgen-macro-0.2.106.crate | 9266 bytes |
| DIST | wasm-bindgen-macro-0.2.118.crate | 9288 bytes |
| DIST | wasm-bindgen-macro-support-0.2.106.crate | 49711 bytes |
| DIST | wasm-bindgen-macro-support-0.2.118.crate | 64823 bytes |
| DIST | wasm-bindgen-shared-0.2.106.crate | 10115 bytes |
| DIST | wasm-bindgen-shared-0.2.118.crate | 10565 bytes |
| DIST | wasm-encoder-0.244.0.crate | 85804 bytes |
| DIST | wasm-metadata-0.244.0.crate | 21500 bytes |
| DIST | wasmparser-0.244.0.crate | 265607 bytes |
| DIST | web-time-1.1.0.crate | 18026 bytes |
| DIST | windows-link-0.2.1.crate | 6133 bytes |
| DIST | windows-sys-0.61.2.crate | 2517186 bytes |
| DIST | wit-bindgen-0.46.0.crate | 60508 bytes |
| DIST | wit-bindgen-0.51.0.crate | 68035 bytes |
| DIST | wit-bindgen-0.57.1.crate | 71227 bytes |
| DIST | wit-bindgen-core-0.51.0.crate | 35048 bytes |
| DIST | wit-bindgen-rust-0.51.0.crate | 57051 bytes |
| DIST | wit-bindgen-rust-macro-0.51.0.crate | 14283 bytes |
| DIST | wit-component-0.244.0.crate | 256882 bytes |
| DIST | wit-parser-0.244.0.crate | 159371 bytes |
| DIST | zerocopy-0.8.31.crate | 257633 bytes |
| DIST | zerocopy-0.8.48.crate | 287554 bytes |
| DIST | zerocopy-derive-0.8.31.crate | 90835 bytes |
| DIST | zerocopy-derive-0.8.48.crate | 97646 bytes |
| DIST | zmij-1.0.21.crate | 26665 bytes |