Gentoo Packages

sci-ml/tokenizers (stuff)

Search

Install

Install this package:

emerge -a sci-ml/tokenizers

Package Information

Description:
Implementation of today's most used tokenizers
Homepage:
https://github.com/huggingface/tokenizers
License:
Apache-2.0 Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD BSD-2 CDLA-Permissive-2.0 ISC MIT MPL-2.0 Unicode-3.0 ZLIB

Versions

Version EAPI Keywords Slot
0.23.2 8 ~amd64 ~arm64 0
0.23.1 8 ~amd64 ~arm64 0
0.22.2 8 ~amd64 ~arm64 0
0.22.1 8 ~amd64 ~arm64 0

Metadata

Maintainers

Upstream

Raw Metadata XML 
<pkgmetadata>
	<maintainer type="person">
		<email>tupone@gentoo.org</email>
		<name>Tupone Alfredo</name>
	</maintainer>
	<upstream>
		<remote-id type="github">huggingface/tokenizers</remote-id>
	</upstream>
</pkgmetadata>

Lint Warnings

Files

Manifest

Type File Size Versions
DIST tokenizers-0.22.0-crates.tar.xz 20740996 bytes 0.22.1
DIST tokenizers-0.22.1.gh.tar.gz 1563505 bytes 0.22.1
DIST tokenizers-0.22.2-crates.tar.xz 21576112 bytes 0.22.2
DIST tokenizers-0.22.2.gh.tar.gz 1568453 bytes 0.22.2
DIST tokenizers-0.23.1-crates.tar.xz 21626500 bytes 0.23.1
DIST tokenizers-0.23.1.gh.tar.gz 1613786 bytes 0.23.1
DIST tokenizers-0.23.2.gh.tar.gz 1475786 bytes 0.23.2
DIST tokenizers-python-0.22.1-crates.tar.xz 14184544 bytes 0.22.1
DIST tokenizers-python-0.22.2-crates.tar.xz 9862444 bytes 0.22.2
DIST tokenizers-python-0.23.1-crates.tar.xz 10988872 bytes 0.23.1
Unmatched Entries
Type File Size
DIST adler2-2.0.1.crate 13366 bytes
DIST ahash-0.8.12.crate 43413 bytes
DIST aho-corasick-1.1.5.crate 184315 bytes
DIST anes-0.1.6.crate 23857 bytes
DIST anstream-1.0.0.crate 28916 bytes
DIST anstyle-1.0.14.crate 17787 bytes
DIST anstyle-parse-1.0.0.crate 22042 bytes
DIST anstyle-query-1.1.5.crate 10264 bytes
DIST anstyle-wincon-3.0.11.crate 12638 bytes
DIST assert_approx_eq-1.1.0.crate 7512 bytes
DIST atomic-waker-1.1.2.crate 12422 bytes
DIST autocfg-1.5.1.crate 18911 bytes
DIST base64-0.13.1.crate 61002 bytes
DIST base64-0.22.1.crate 81597 bytes
DIST bit-set-0.8.0.crate 16289 bytes
DIST bit-vec-0.8.0.crate 24132 bytes
DIST bitflags-1.3.2.crate 23021 bytes
DIST bitflags-2.13.1.crate 51395 bytes
DIST bumpalo-3.20.3.crate 98671 bytes
DIST byteorder-1.5.0.crate 23288 bytes
DIST bytes-1.12.1.crate 75667 bytes
DIST cast-0.3.0.crate 11452 bytes
DIST castaway-0.2.4.crate 12546 bytes
DIST cc-1.4.4.crate 100602 bytes
DIST cc-1.4.5.crate 101464 bytes
DIST cfg-if-1.0.4.crate 9360 bytes
DIST cfg_aliases-0.2.2.crate 6527 bytes
DIST chacha20-0.10.2.crate 38197 bytes
DIST ciborium-0.2.2.crate 35611 bytes
DIST ciborium-io-0.2.2.crate 6697 bytes
DIST ciborium-ll-0.2.2.crate 14695 bytes
DIST clap-4.6.6.crate 64815 bytes
DIST clap_builder-4.6.6.crate 171756 bytes
DIST clap_lex-1.1.0.crate 13470 bytes
DIST colorchoice-1.0.5.crate 8187 bytes
DIST compact_str-0.9.1.crate 88299 bytes
DIST console-0.15.11.crate 37822 bytes
DIST console-0.16.4.crate 34353 bytes
DIST cpufeatures-0.3.1.crate 14272 bytes
DIST crc32fast-1.5.1.crate 47893 bytes
DIST criterion-0.6.0.crate 119470 bytes
DIST criterion-plot-0.5.0.crate 22706 bytes
DIST crossbeam-deque-0.8.7.crate 24141 bytes
DIST crossbeam-deque-0.8.8.crate 24634 bytes
DIST crossbeam-epoch-0.9.20.crate 47545 bytes
DIST crossbeam-epoch-0.9.21.crate 47936 bytes
DIST crossbeam-utils-0.8.22.crate 46874 bytes
DIST crossbeam-utils-0.8.23.crate 47174 bytes
DIST crunchy-0.2.4.crate 3887 bytes
DIST daachorse-3.0.3.crate 94191 bytes
DIST darling-0.20.11.crate 37614 bytes
DIST darling_core-0.20.11.crate 68006 bytes
DIST darling_macro-0.20.11.crate 2532 bytes
DIST dary_heap-0.3.9.crate 27054 bytes
DIST defmt-1.1.1.crate 34207 bytes
DIST defmt-macros-1.1.1.crate 27937 bytes
DIST defmt-parser-1.0.0.crate 10881 bytes
DIST derive_builder-0.20.2.crate 36661 bytes
DIST derive_builder_core-0.20.2.crate 31397 bytes
DIST derive_builder_macro-0.20.2.crate 6360 bytes
DIST dirs-6.0.0.crate 14190 bytes
DIST dirs-sys-0.5.0.crate 10157 bytes
DIST displaydoc-0.2.7.crate 21343 bytes
DIST either-1.18.0.crate 22357 bytes
DIST encode_unicode-1.0.0.crate 56986 bytes
DIST env_filter-2.0.0.crate 15184 bytes
DIST env_logger-0.11.11.crate 31884 bytes
DIST errno-0.3.14.crate 12002 bytes
DIST esaxx-rs-0.1.10.crate 175210 bytes
DIST fancy-regex-0.17.0.crate 109054 bytes
DIST fastrand-2.5.0.crate 20782 bytes
DIST find-msvc-tools-0.1.11.crate 31517 bytes
DIST find-msvc-tools-0.1.12.crate 31570 bytes
DIST flate2-1.1.10.crate 80244 bytes
DIST fnv-1.0.7.crate 11266 bytes
DIST form_urlencoded-1.2.2.crate 9347 bytes
DIST futures-channel-0.3.34.crate 33165 bytes
DIST futures-core-0.3.34.crate 14692 bytes
DIST futures-io-0.3.34.crate 9189 bytes
DIST futures-macro-0.3.34.crate 11926 bytes
DIST futures-sink-0.3.34.crate 8107 bytes
DIST futures-task-0.3.34.crate 11376 bytes
DIST futures-util-0.3.34.crate 169782 bytes
DIST getrandom-0.2.17.crate 40780 bytes
DIST getrandom-0.3.4.crate 50932 bytes
DIST getrandom-0.4.3.crate 52437 bytes
DIST half-2.7.1.crate 61040 bytes
DIST heck-0.5.0.crate 11517 bytes
DIST hf-hub-0.4.3.crate 46526 bytes
DIST http-1.5.0.crate 111161 bytes
DIST http-body-1.1.0.crate 7887 bytes
DIST http-body-util-0.1.5.crate 18574 bytes
DIST httparse-1.10.1.crate 45190 bytes
DIST hyper-1.11.1.crate 170053 bytes
DIST hyper-rustls-0.27.9.crate 22935 bytes
DIST hyper-util-0.1.20.crate 112807 bytes
DIST icu_collections-2.3.0.crate 87970 bytes
DIST icu_locale_core-2.3.0.crate 80250 bytes
DIST icu_normalizer-2.3.0.crate 530595 bytes
DIST icu_normalizer_data-2.3.0.crate 67215 bytes
DIST icu_properties-2.3.0.crate 102637 bytes
DIST icu_properties_data-2.3.0.crate 169152 bytes
DIST icu_provider-2.3.1.crate 52447 bytes
DIST ident_case-1.0.1.crate 3492 bytes
DIST idna-1.1.0.crate 148747 bytes
DIST idna_adapter-1.2.2.crate 10444 bytes
DIST indicatif-0.17.11.crate 66577 bytes
DIST indicatif-0.18.6.crate 70598 bytes
DIST ipnet-2.12.2.crate 30804 bytes
DIST is_terminal_polyfill-1.70.2.crate 7548 bytes
DIST itertools-0.10.5.crate 115354 bytes
DIST itertools-0.13.0.crate 146261 bytes
DIST itertools-0.14.0.crate 152715 bytes
DIST itoa-1.0.18.crate 15935 bytes
DIST jiff-0.2.35.crate 676554 bytes
DIST jiff-core-0.1.0.crate 127943 bytes
DIST jiff-static-0.2.35.crate 7996 bytes
DIST js-sys-0.3.104.crate 108816 bytes
DIST js-sys-0.3.105.crate 113002 bytes
DIST lazy_static-1.5.0.crate 14025 bytes
DIST libc-0.2.189.crate 851502 bytes
DIST libredox-0.1.23.crate 10541 bytes
DIST linux-raw-sys-0.12.1.crate 3006116 bytes
DIST litemap-0.8.3.crate 35045 bytes
DIST log-0.4.34.crate 53395 bytes
DIST lru-slab-0.1.2.crate 9090 bytes
DIST macro_rules_attribute-0.2.3.crate 15329 bytes
DIST macro_rules_attribute-proc_macro-0.2.3.crate 8272 bytes
DIST matrixmultiply-0.3.11.crate 51167 bytes
DIST memchr-2.8.3.crate 99165 bytes
DIST minimal-lexical-0.2.1.crate 94841 bytes
DIST miniz_oxide-0.9.1.crate 70519 bytes
DIST mio-1.2.3.crate 110455 bytes
DIST monostate-0.1.18.crate 15992 bytes
DIST monostate-impl-0.1.18.crate 7862 bytes
DIST ndarray-0.16.1.crate 291026 bytes
DIST ndarray-0.17.2.crate 243631 bytes
DIST nom-7.1.3.crate 117570 bytes
DIST nu-ansi-term-0.50.3.crate 29597 bytes
DIST num-complex-0.4.6.crate 30352 bytes
DIST num-integer-0.1.47.crate 23502 bytes
DIST num-traits-0.2.19.crate 51631 bytes
DIST number_prefix-0.4.0.crate 6922 bytes
DIST numpy-0.29.0.crate 86464 bytes
DIST once_cell-1.21.4.crate 35010 bytes
DIST once_cell_polyfill-1.70.2.crate 7448 bytes
DIST onig-6.5.3.crate 32075 bytes
DIST onig_sys-69.9.3.crate 657340 bytes
DIST oorandom-11.1.5.crate 23750 bytes
DIST option-ext-0.2.0.crate 7345 bytes
DIST paste-1.0.15.crate 18374 bytes
DIST pastey-0.2.3.crate 20361 bytes
DIST percent-encoding-2.3.2.crate 11583 bytes
DIST pin-project-lite-0.2.17.crate 31034 bytes
DIST pkg-config-0.3.34.crate 21370 bytes
DIST plotters-0.3.7.crate 149031 bytes
DIST plotters-backend-0.3.7.crate 13709 bytes
DIST plotters-svg-0.3.7.crate 6715 bytes
DIST portable-atomic-1.15.0.crate 202705 bytes
DIST portable-atomic-util-0.2.7.crate 53577 bytes
DIST potential_utf-0.1.6.crate 10420 bytes
DIST ppv-lite86-0.2.21.crate 22522 bytes
DIST proc-macro2-1.0.107.crate 59588 bytes
DIST pyo3-0.29.2.crate 1255741 bytes
DIST pyo3-async-runtimes-0.29.0.crate 51487 bytes
DIST pyo3-build-config-0.29.2.crate 41104 bytes
DIST pyo3-ffi-0.29.2.crate 90029 bytes
DIST pyo3-macros-0.29.2.crate 9044 bytes
DIST pyo3-macros-backend-0.29.2.crate 93749 bytes
DIST quinn-0.11.11.crate 85390 bytes
DIST quinn-proto-0.11.17.crate 246202 bytes
DIST quinn-udp-0.5.15.crate 32501 bytes
DIST quote-1.0.47.crate 31622 bytes
DIST r-efi-5.3.0.crate 64532 bytes
DIST r-efi-6.0.0.crate 65303 bytes
DIST rand-0.10.2.crate 104493 bytes
DIST rand-0.9.5.crate 100216 bytes
DIST rand_chacha-0.9.0.crate 18258 bytes
DIST rand_core-0.10.1.crate 22833 bytes
DIST rand_core-0.9.5.crate 24129 bytes
DIST rand_pcg-0.10.2.crate 15439 bytes
DIST rawpointer-0.2.1.crate 7490 bytes
DIST rayon-1.12.0.crate 183391 bytes
DIST rayon-cond-0.4.0.crate 10659 bytes
DIST rayon-core-1.13.0.crate 73151 bytes
DIST redox_users-0.5.2.crate 17280 bytes
DIST regex-1.13.1.crate 157118 bytes
DIST regex-automata-0.4.18.crate 628707 bytes
DIST regex-syntax-0.8.11.crate 359055 bytes
DIST reqwest-0.12.28.crate 157031 bytes
DIST ring-0.17.14.crate 1502610 bytes
DIST rustc-hash-2.1.3.crate 14811 bytes
DIST rustix-1.1.4.crate 425241 bytes
DIST rustls-0.23.44.crate 377662 bytes
DIST rustls-pki-types-1.15.1.crate 39322 bytes
DIST rustls-webpki-0.103.15.crate 87465 bytes
DIST rustversion-1.0.23.crate 21013 bytes
DIST ryu-1.0.23.crate 52823 bytes
DIST same-file-1.0.6.crate 10183 bytes
DIST serde-1.0.229.crate 83669 bytes
DIST serde_core-1.0.229.crate 63100 bytes
DIST serde_derive-1.0.229.crate 59864 bytes
DIST serde_json-1.0.151.crate 156556 bytes
DIST serde_urlencoded-0.7.1.crate 12822 bytes
DIST sharded-slab-0.1.7.crate 58227 bytes
DIST shlex-2.0.1.crate 19332 bytes
DIST signal-hook-registry-1.4.8.crate 21668 bytes
DIST simd-adler32-0.3.10.crate 18760 bytes
DIST slab-0.4.12.crate 19080 bytes
DIST smallvec-1.16.0.crate 32647 bytes
DIST socket2-0.6.5.crate 60031 bytes
DIST socks-0.3.4.crate 13172 bytes
DIST spm_precompiled-0.1.4.crate 557527 bytes
DIST stable_deref_trait-1.2.1.crate 8186 bytes
DIST static_assertions-1.1.0.crate 18480 bytes
DIST strsim-0.11.1.crate 14266 bytes
DIST subtle-2.6.1.crate 14562 bytes
DIST syn-2.0.119.crate 307407 bytes
DIST syn-3.0.4.crate 313047 bytes
DIST syn-3.0.5.crate 313222 bytes
DIST sync_wrapper-1.0.2.crate 6958 bytes
DIST synstructure-0.13.2.crate 18950 bytes
DIST target-lexicon-0.13.5.crate 26928 bytes
DIST tempfile-3.27.0.crate 44780 bytes
DIST thiserror-2.0.20.crate 28969 bytes
DIST thiserror-impl-2.0.20.crate 21436 bytes
DIST thread_local-1.1.10.crate 16361 bytes
DIST tinystr-0.8.4.crate 25003 bytes
DIST tinytemplate-1.2.1.crate 26490 bytes
DIST tinyvec-1.13.2.crate 60587 bytes
DIST tinyvec_macros-0.1.1.crate 5865 bytes
DIST tokio-1.53.1.crate 920101 bytes
DIST tokio-macros-2.7.2.crate 16081 bytes
DIST tokio-rustls-0.26.5.crate 29077 bytes
DIST tokio-util-0.7.19.crate 147245 bytes
DIST tower-0.5.3.crate 110488 bytes
DIST tower-http-0.6.11.crate 156773 bytes
DIST tower-layer-0.3.3.crate 6180 bytes
DIST tower-service-0.3.3.crate 6950 bytes
DIST tracing-0.1.44.crate 463135 bytes
DIST tracing-attributes-0.1.31.crate 39733 bytes
DIST tracing-core-0.1.36.crate 63967 bytes
DIST tracing-log-0.2.0.crate 17561 bytes
DIST tracing-subscriber-0.3.23.crate 213773 bytes
DIST try-lock-0.2.5.crate 4314 bytes
DIST unicode-ident-1.0.24.crate 49298 bytes
DIST unicode-normalization-alignments-0.1.12.crate 91546 bytes
DIST unicode-segmentation-1.13.3.crate 112325 bytes
DIST unicode-width-0.2.2.crate 282768 bytes
DIST unicode_categories-0.1.1.crate 87298 bytes
DIST unit-prefix-0.5.2.crate 8041 bytes
DIST untrusted-0.9.0.crate 14447 bytes
DIST ureq-2.12.1.crate 115366 bytes
DIST url-2.5.8.crate 86512 bytes
DIST utf8_iter-1.0.4.crate 10437 bytes
DIST utf8parse-0.2.2.crate 13499 bytes
DIST valuable-0.1.1.crate 28679 bytes
DIST version_check-0.9.5.crate 15554 bytes
DIST walkdir-2.5.0.crate 23951 bytes
DIST want-0.3.1.crate 6398 bytes
DIST wasi-0.11.1+wasi-snapshot-preview1.crate 28477 bytes
DIST wasip2-1.0.4+wasi-0.2.12.crate 135311 bytes
DIST wasm-bindgen-0.2.127.crate 68138 bytes
DIST wasm-bindgen-0.2.128.crate 70345 bytes
DIST wasm-bindgen-futures-0.4.78.crate 8291 bytes
DIST wasm-bindgen-macro-0.2.127.crate 9372 bytes
DIST wasm-bindgen-macro-0.2.128.crate 9595 bytes
DIST wasm-bindgen-macro-support-0.2.127.crate 85236 bytes
DIST wasm-bindgen-macro-support-0.2.128.crate 118512 bytes
DIST wasm-bindgen-shared-0.2.127.crate 11291 bytes
DIST wasm-bindgen-shared-0.2.128.crate 12994 bytes
DIST wasm-streams-0.4.2.crate 36773 bytes
DIST web-sys-0.3.105.crate 682289 bytes
DIST web-time-1.1.0.crate 18026 bytes
DIST webpki-roots-0.26.11.crate 15557 bytes
DIST webpki-roots-1.0.9.crate 217462 bytes
DIST winapi-0.3.9.crate 1200382 bytes
DIST winapi-i686-pc-windows-gnu-0.4.0.crate 2918815 bytes
DIST winapi-util-0.1.11.crate 13368 bytes
DIST winapi-x86_64-pc-windows-gnu-0.4.0.crate 2947998 bytes
DIST windows-link-0.2.1.crate 6133 bytes
DIST windows-sys-0.52.0.crate 2576877 bytes
DIST windows-sys-0.59.0.crate 2387323 bytes
DIST windows-sys-0.60.2.crate 2518479 bytes
DIST windows-sys-0.61.2.crate 2517186 bytes
DIST windows-targets-0.52.6.crate 6403 bytes
DIST windows-targets-0.53.5.crate 7126 bytes
DIST windows_aarch64_gnullvm-0.52.6.crate 435718 bytes
DIST windows_aarch64_gnullvm-0.53.1.crate 787748 bytes
DIST windows_aarch64_msvc-0.52.6.crate 832615 bytes
DIST windows_aarch64_msvc-0.53.1.crate 838009 bytes
DIST windows_i686_gnu-0.52.6.crate 880402 bytes
DIST windows_i686_gnu-0.53.1.crate 939775 bytes
DIST windows_i686_gnullvm-0.52.6.crate 475940 bytes
DIST windows_i686_gnullvm-0.53.1.crate 857396 bytes
DIST windows_i686_msvc-0.52.6.crate 901163 bytes
DIST windows_i686_msvc-0.53.1.crate 907688 bytes
DIST windows_x86_64_gnu-0.52.6.crate 836363 bytes
DIST windows_x86_64_gnu-0.53.1.crate 903712 bytes
DIST windows_x86_64_gnullvm-0.52.6.crate 435707 bytes
DIST windows_x86_64_gnullvm-0.53.1.crate 787739 bytes
DIST windows_x86_64_msvc-0.52.6.crate 832564 bytes
DIST windows_x86_64_msvc-0.53.1.crate 837950 bytes
DIST wit-bindgen-0.57.1.crate 71227 bytes
DIST writeable-0.6.4.crate 31222 bytes
DIST yoke-0.8.3.crate 33132 bytes
DIST yoke-derive-0.8.2.crate 15077 bytes
DIST zerocopy-0.8.56.crate 284907 bytes
DIST zerocopy-0.8.57.crate 285690 bytes
DIST zerocopy-derive-0.8.56.crate 105007 bytes
DIST zerocopy-derive-0.8.57.crate 106175 bytes
DIST zerofrom-0.1.8.crate 6156 bytes
DIST zerofrom-derive-0.1.7.crate 8853 bytes
DIST zeroize-1.9.0.crate 24480 bytes
DIST zerotrie-0.2.5.crate 84204 bytes
DIST zerovec-0.11.8.crate 130820 bytes
DIST zerovec-derive-0.11.6.crate 23102 bytes
DIST zlib-rs-0.6.7.crate 215740 bytes
DIST zmij-1.0.23.crate 28612 bytes