Gentoo Packages

# Copyright 2023-2026 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

# Generated with pycargoebuild 0.15.0 from both crate graphs. The core graph
# needs a generated lockfile; union both CRATES sets when bumping. ::gentoo's
# prebundled archives are unavailable for 0.23.2. verified 2026-09-09

EAPI=8

DISTUTILS_USE_PEP517=maturin
PYTHON_COMPAT=( python3_{12..14} )
DISTUTILS_EXT=1
DISTUTILS_SINGLE_IMPL=1
RUST_MIN_VER="1.87.0"

CRATES="
	adler2@2.0.1
	ahash@0.8.12
	aho-corasick@1.1.5
	anes@0.1.6
	anstream@1.0.0
	anstyle-parse@1.0.0
	anstyle-query@1.1.5
	anstyle-wincon@3.0.11
	anstyle@1.0.14
	assert_approx_eq@1.1.0
	atomic-waker@1.1.2
	autocfg@1.5.1
	base64@0.13.1
	base64@0.22.1
	bit-set@0.8.0
	bit-vec@0.8.0
	bitflags@1.3.2
	bitflags@2.13.1
	bumpalo@3.20.3
	byteorder@1.5.0
	bytes@1.12.1
	cast@0.3.0
	castaway@0.2.4
	cc@1.4.4
	cc@1.4.5
	cfg-if@1.0.4
	cfg_aliases@0.2.2
	chacha20@0.10.2
	ciborium-io@0.2.2
	ciborium-ll@0.2.2
	ciborium@0.2.2
	clap@4.6.6
	clap_builder@4.6.6
	clap_lex@1.1.0
	colorchoice@1.0.5
	compact_str@0.9.1
	console@0.15.11
	console@0.16.4
	cpufeatures@0.3.1
	crc32fast@1.5.1
	criterion-plot@0.5.0
	criterion@0.6.0
	crossbeam-deque@0.8.7
	crossbeam-deque@0.8.8
	crossbeam-epoch@0.9.20
	crossbeam-epoch@0.9.21
	crossbeam-utils@0.8.22
	crossbeam-utils@0.8.23
	crunchy@0.2.4
	daachorse@3.0.3
	darling@0.20.11
	darling_core@0.20.11
	darling_macro@0.20.11
	dary_heap@0.3.9
	defmt-macros@1.1.1
	defmt-parser@1.0.0
	defmt@1.1.1
	derive_builder@0.20.2
	derive_builder_core@0.20.2
	derive_builder_macro@0.20.2
	dirs-sys@0.5.0
	dirs@6.0.0
	displaydoc@0.2.7
	either@1.18.0
	encode_unicode@1.0.0
	env_filter@2.0.0
	env_logger@0.11.11
	errno@0.3.14
	esaxx-rs@0.1.10
	fancy-regex@0.17.0
	fastrand@2.5.0
	find-msvc-tools@0.1.11
	find-msvc-tools@0.1.12
	flate2@1.1.10
	fnv@1.0.7
	form_urlencoded@1.2.2
	futures-channel@0.3.34
	futures-core@0.3.34
	futures-io@0.3.34
	futures-macro@0.3.34
	futures-sink@0.3.34
	futures-task@0.3.34
	futures-util@0.3.34
	getrandom@0.2.17
	getrandom@0.3.4
	getrandom@0.4.3
	half@2.7.1
	heck@0.5.0
	hf-hub@0.4.3
	http-body-util@0.1.5
	http-body@1.1.0
	http@1.5.0
	httparse@1.10.1
	hyper-rustls@0.27.9
	hyper-util@0.1.20
	hyper@1.11.1
	icu_collections@2.3.0
	icu_locale_core@2.3.0
	icu_normalizer@2.3.0
	icu_normalizer_data@2.3.0
	icu_properties@2.3.0
	icu_properties_data@2.3.0
	icu_provider@2.3.1
	ident_case@1.0.1
	idna@1.1.0
	idna_adapter@1.2.2
	indicatif@0.17.11
	indicatif@0.18.6
	ipnet@2.12.2
	is_terminal_polyfill@1.70.2
	itertools@0.10.5
	itertools@0.13.0
	itertools@0.14.0
	itoa@1.0.18
	jiff-core@0.1.0
	jiff-static@0.2.35
	jiff@0.2.35
	js-sys@0.3.104
	js-sys@0.3.105
	lazy_static@1.5.0
	libc@0.2.189
	libredox@0.1.23
	linux-raw-sys@0.12.1
	litemap@0.8.3
	log@0.4.34
	lru-slab@0.1.2
	macro_rules_attribute-proc_macro@0.2.3
	macro_rules_attribute@0.2.3
	matrixmultiply@0.3.11
	memchr@2.8.3
	minimal-lexical@0.2.1
	miniz_oxide@0.9.1
	mio@1.2.3
	monostate-impl@0.1.18
	monostate@0.1.18
	ndarray@0.16.1
	ndarray@0.17.2
	nom@7.1.3
	nu-ansi-term@0.50.3
	num-complex@0.4.6
	num-integer@0.1.47
	num-traits@0.2.19
	number_prefix@0.4.0
	numpy@0.29.0
	once_cell@1.21.4
	once_cell_polyfill@1.70.2
	onig@6.5.3
	onig_sys@69.9.3
	oorandom@11.1.5
	option-ext@0.2.0
	paste@1.0.15
	pastey@0.2.3
	percent-encoding@2.3.2
	pin-project-lite@0.2.17
	pkg-config@0.3.34
	plotters-backend@0.3.7
	plotters-svg@0.3.7
	plotters@0.3.7
	portable-atomic-util@0.2.7
	portable-atomic@1.15.0
	potential_utf@0.1.6
	ppv-lite86@0.2.21
	proc-macro2@1.0.107
	pyo3-async-runtimes@0.29.0
	pyo3-build-config@0.29.2
	pyo3-ffi@0.29.2
	pyo3-macros-backend@0.29.2
	pyo3-macros@0.29.2
	pyo3@0.29.2
	quinn-proto@0.11.17
	quinn-udp@0.5.15
	quinn@0.11.11
	quote@1.0.47
	r-efi@5.3.0
	r-efi@6.0.0
	rand@0.10.2
	rand@0.9.5
	rand_chacha@0.9.0
	rand_core@0.10.1
	rand_core@0.9.5
	rand_pcg@0.10.2
	rawpointer@0.2.1
	rayon-cond@0.4.0
	rayon-core@1.13.0
	rayon@1.12.0
	redox_users@0.5.2
	regex-automata@0.4.18
	regex-syntax@0.8.11
	regex@1.13.1
	reqwest@0.12.28
	ring@0.17.14
	rustc-hash@2.1.3
	rustix@1.1.4
	rustls-pki-types@1.15.1
	rustls-webpki@0.103.15
	rustls@0.23.44
	rustversion@1.0.23
	ryu@1.0.23
	same-file@1.0.6
	serde@1.0.229
	serde_core@1.0.229
	serde_derive@1.0.229
	serde_json@1.0.151
	serde_urlencoded@0.7.1
	sharded-slab@0.1.7
	shlex@2.0.1
	signal-hook-registry@1.4.8
	simd-adler32@0.3.10
	slab@0.4.12
	smallvec@1.16.0
	socket2@0.6.5
	socks@0.3.4
	spm_precompiled@0.1.4
	stable_deref_trait@1.2.1
	static_assertions@1.1.0
	strsim@0.11.1
	subtle@2.6.1
	syn@2.0.119
	syn@3.0.4
	syn@3.0.5
	sync_wrapper@1.0.2
	synstructure@0.13.2
	target-lexicon@0.13.5
	tempfile@3.27.0
	thiserror-impl@2.0.20
	thiserror@2.0.20
	thread_local@1.1.10
	tinystr@0.8.4
	tinytemplate@1.2.1
	tinyvec@1.13.2
	tinyvec_macros@0.1.1
	tokio-macros@2.7.2
	tokio-rustls@0.26.5
	tokio-util@0.7.19
	tokio@1.53.1
	tower-http@0.6.11
	tower-layer@0.3.3
	tower-service@0.3.3
	tower@0.5.3
	tracing-attributes@0.1.31
	tracing-core@0.1.36
	tracing-log@0.2.0
	tracing-subscriber@0.3.23
	tracing@0.1.44
	try-lock@0.2.5
	unicode-ident@1.0.24
	unicode-normalization-alignments@0.1.12
	unicode-segmentation@1.13.3
	unicode-width@0.2.2
	unicode_categories@0.1.1
	unit-prefix@0.5.2
	untrusted@0.9.0
	ureq@2.12.1
	url@2.5.8
	utf8_iter@1.0.4
	utf8parse@0.2.2
	valuable@0.1.1
	version_check@0.9.5
	walkdir@2.5.0
	want@0.3.1
	wasi@0.11.1+wasi-snapshot-preview1
	wasip2@1.0.4+wasi-0.2.12
	wasm-bindgen-futures@0.4.78
	wasm-bindgen-macro-support@0.2.127
	wasm-bindgen-macro-support@0.2.128
	wasm-bindgen-macro@0.2.127
	wasm-bindgen-macro@0.2.128
	wasm-bindgen-shared@0.2.127
	wasm-bindgen-shared@0.2.128
	wasm-bindgen@0.2.127
	wasm-bindgen@0.2.128
	wasm-streams@0.4.2
	web-sys@0.3.105
	web-time@1.1.0
	webpki-roots@0.26.11
	webpki-roots@1.0.9
	winapi-i686-pc-windows-gnu@0.4.0
	winapi-util@0.1.11
	winapi-x86_64-pc-windows-gnu@0.4.0
	winapi@0.3.9
	windows-link@0.2.1
	windows-sys@0.52.0
	windows-sys@0.59.0
	windows-sys@0.60.2
	windows-sys@0.61.2
	windows-targets@0.52.6
	windows-targets@0.53.5
	windows_aarch64_gnullvm@0.52.6
	windows_aarch64_gnullvm@0.53.1
	windows_aarch64_msvc@0.52.6
	windows_aarch64_msvc@0.53.1
	windows_i686_gnu@0.52.6
	windows_i686_gnu@0.53.1
	windows_i686_gnullvm@0.52.6
	windows_i686_gnullvm@0.53.1
	windows_i686_msvc@0.52.6
	windows_i686_msvc@0.53.1
	windows_x86_64_gnu@0.52.6
	windows_x86_64_gnu@0.53.1
	windows_x86_64_gnullvm@0.52.6
	windows_x86_64_gnullvm@0.53.1
	windows_x86_64_msvc@0.52.6
	windows_x86_64_msvc@0.53.1
	wit-bindgen@0.57.1
	writeable@0.6.4
	yoke-derive@0.8.2
	yoke@0.8.3
	zerocopy-derive@0.8.56
	zerocopy-derive@0.8.57
	zerocopy@0.8.56
	zerocopy@0.8.57
	zerofrom-derive@0.1.7
	zerofrom@0.1.8
	zeroize@1.9.0
	zerotrie@0.2.5
	zerovec-derive@0.11.6
	zerovec@0.11.8
	zlib-rs@0.6.7
	zmij@1.0.23
"

inherit cargo distutils-r1

DESCRIPTION="Implementation of today's most used tokenizers"
HOMEPAGE="https://github.com/huggingface/tokenizers"
SRC_URI="
	https://github.com/huggingface/${PN}/archive/refs/tags/v${PV}.tar.gz
	-> ${P}.gh.tar.gz
	${CARGO_CRATE_URIS}
"

LICENSE="Apache-2.0"
# Dependent crate licenses
LICENSE+="
	Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD BSD-2 CDLA-Permissive-2.0 ISC MIT MPL-2.0 Unicode-3.0 ZLIB
"
SLOT="0"
KEYWORDS="~amd64 ~arm64"

RDEPEND="dev-libs/oniguruma"
BDEPEND="
	test? ( sci-ml/datasets[${PYTHON_SINGLE_USEDEP}] )
	$(python_gen_cond_dep '
		dev-python/setuptools-rust[${PYTHON_USEDEP}]
	')
"

EPYTEST_PLUGINS=( )
distutils_enable_tests pytest

QA_FLAGS_IGNORED=".*/site-packages/tokenizers/.*so"

src_unpack() {
	cargo_src_unpack
}

pkg_setup() {
	python-single-r1_pkg_setup
	rust_pkg_setup
}

src_prepare() {
	default
	cd bindings/python
	eapply "${FILESDIR}"/${PN}-0.21.2-test.patch
	distutils-r1_src_prepare
}

src_configure() {
	cd tokenizers
	cargo_src_configure
	cd ../bindings/python
	distutils-r1_src_configure
}

src_compile() {
	export RUSTONIG_SYSTEM_LIBONIG=1
	cd tokenizers
	cargo_src_compile
	cd ../bindings/python
	distutils-r1_src_compile
}

src_test() {
	cd bindings/python
	local -x EPYTEST_IGNORE=( benches/test_tiktoken.py )
	local -x EPYTEST_DESELECT=(
		tests/bindings/test_encoding.py::TestEncoding::test_char_to_token
		tests/bindings/test_encoding.py::TestEncoding::test_char_to_word
		tests/bindings/test_encoding.py::TestEncoding::test_invalid_truncate_direction
		tests/bindings/test_encoding.py::TestEncoding::test_n_sequences
		tests/bindings/test_encoding.py::TestEncoding::test_sequence_ids
		tests/bindings/test_encoding.py::TestEncoding::test_token_to_chars
		tests/bindings/test_encoding.py::TestEncoding::test_token_to_sequence
		tests/bindings/test_encoding.py::TestEncoding::test_token_to_word
		tests/bindings/test_encoding.py::TestEncoding::test_truncation
		tests/bindings/test_encoding.py::TestEncoding::test_word_to_chars
		tests/bindings/test_encoding.py::TestEncoding::test_word_to_tokens
		tests/bindings/test_models.py::TestWordLevel::test_instantiate
		tests/bindings/test_models.py::TestWordPiece::test_instantiate
		tests/bindings/test_processors.py::TestByteLevelProcessing::test_processing
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_async_methods_existence
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_basic_encoding
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_concurrency
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_decode
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_encode
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_error_handling
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_large_batch
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_numpy_inputs
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_various_input_formats
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_performance_comparison
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_with_special_tokens
		tests/bindings/test_tokenizer.py::TestAsyncTokenizer::test_with_truncation_padding
		tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_add_special_tokens
		tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_formats
		tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_special_tokens
		tests/bindings/test_tokenizer.py::TestTokenizer::test_decode_skip_special_tokens
		tests/bindings/test_tokenizer.py::TestTokenizer::test_decode_stream_fallback
		tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained
		tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained_revision
		tests/bindings/test_tokenizer.py::TestTokenizer::test_splitting
		tests/bindings/test_trainers.py::TestUnigram::test_continuing_prefix_trainer_mismatch
		tests/bindings/test_trainers.py::TestUnigram::test_train
		tests/bindings/test_trainers.py::TestUnigram::test_train_parallelism_with_custom_pretokenizer
		tests/documentation/test_pipeline.py::TestPipeline::test_bert_example
		tests/documentation/test_pipeline.py::TestPipeline::test_pipeline
		tests/documentation/test_quicktour.py::TestQuicktour::test_quicktour
		tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_datasets
		tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_gzip
		tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_add_prefix_space
		tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_basic_encode
		tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_lowerspace
		tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_multiprocessing_with_parallelism
		tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_basic_encode
		tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_multiprocessing_with_parallelism
		tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_basic_encode
		tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_decoding
		tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_lowercase
		tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_multiprocessing_with_parallelism
		tests/test_serialization.py::TestSerialization::test_full_serialization_albert
		tests/test_serialization.py::TestSerialization::test_str_big
	)
	distutils-r1_src_test
}

src_install() {
	cd tokenizers
	cd ../bindings/python
	distutils-r1_src_install
}