bump tokenicer to v0.0.3 (#1308)

* bump tokenicer to v0.0.3

* fix ruff sh will compile project

Author: CSY-ModelCloud

.github/workflows/code_quality.yml

@@ -9,16 +9,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.11]
-        os: [ubuntu-22.04]
+        python-version: [3.13]
+        os: [ubuntu-24.04]
     runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
       with:
         repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
         ref: ${{ github.event.pull_request.head.ref || github.ref }}
-    - uses: actions/setup-python@v5 
+
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'

format/format.sh

@@ -3,7 +3,7 @@
 cd "$(dirname "$0")" || exit
 
 # force ruff/isort to be same version as setup.py
-pip install -U gptqmodel["quality"]
+pip install -U ruff==0.9.6 isort==6.0.0
 
 ruff check ../gptqmodel/models ../gptqmodel/nn_modules ../gptqmodel/quantization ../gptqmodel/utils ../gptqmodel/__init__.py ../examples ../tests ../setup.py --fix --unsafe-fixes
 ruff_status=$?

requirements.txt

@@ -13,4 +13,4 @@ hf_transfer>=0.1.9
 huggingface_hub>=0.28.1
 lm-eval==0.4.7
 colorlog>=6.9.0
-tokenicer>=0.0.2
+tokenicer>=0.0.3

tests/test_transformers.py

@@ -15,15 +15,14 @@
 # limitations under the License.
 import os
 
-
 os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 import tempfile  # noqa: E402
 import unittest  # noqa: E402
 
 import transformers  # noqa: E402
+from gptqmodel.utils.torch import torch_empty_cache  # noqa: E402
 from packaging.version import Version  # noqa: E402
 from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig  # noqa: E402
-from gptqmodel.utils.torch import torch_empty_cache # noqa: E402
 
 
 class TestTransformersIntegration(unittest.TestCase):