test forcing tokenizers

2025-12-16 09:24:54 +08:00 · 2024-08-08 17:55:45 +02:00
1 changed files with 1 additions and 0 deletions
--- a/setup.py
+++ b/setup.py
@@ -135,6 +135,7 @@ _deps = [
    "transformers>=4.41.2",
    "urllib3<=2.0.0",
    "black",
+    "tokenizers==0.20.0rc1"
 ]

 # this is a lookup table with items like: