making version 0.21.2 of tokenizers locked in project deps.

alkavan · mariadb-RyanKillea · commit b10c9052bca8 · 2025-07-31T08:01:20.000-07:00
due to a new version of tokenizers==0.21.4 released hours ago, the installation fails because the pypi package didn't supply *any* of the platform binaries.

while the issue might be resolved soon, it's unknown when, and will probably will happen in the next release of tokenizers as well.

the issue might be not trivial for most people.
diff --git a/pyproject.toml b/pyproject.toml
@@ -12,5 +12,6 @@ dependencies = [
     "openai>=1.78.1",
     "python-dotenv>=1.1.0",
     "sentence-transformers>=4.1.0",
-    "asyncmy>=0.2.10"
+    "asyncmy>=0.2.10",
+    "tokenizers==0.21.2",
 ]

Original file line number	Diff line number	Diff line change
`@@ -12,5 +12,6 @@ dependencies = [`
`12`	`12`	`"openai>=1.78.1",`
`13`	`13`	`"python-dotenv>=1.1.0",`
`14`	`14`	`"sentence-transformers>=4.1.0",`
`15`		`- "asyncmy>=0.2.10"`
	`15`	`+ "asyncmy>=0.2.10",`
	`16`	`+ "tokenizers==0.21.2",`
`16`	`17`	`]`