[Windows] Fix build issues using Clang-CL on Windows, add CI

Author: GregoryComer

.github/workflows/pull.yml

@@ -34,3 +34,27 @@ jobs:
 
         # Run tests
         pytest
+ 
+  windows:
+    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+    with:
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        conda init powershell
+        powershell -Command "& {
+          Set-PSDebug -Trace 1
+          \$ErrorActionPreference = 'Stop'
+          \$PSNativeCommandUseErrorActionPreference = \$true
+          
+          cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test
+          cmake --build build/test -j9 --config Debug
+          pushd build/test && ctest && popd
+
+          # Install tokenizers
+          pip install . -v
+          pip install pytest blobfile transformers>=4.53.1
+
+          # Run tests
+          pytest
+        }"

.gitignore

@@ -34,3 +34,7 @@ pip-out/
 *~
 .~lock.*
 *.idea
+
+*.so
+*.dylib
+*.pyd

CMakeLists.txt

@@ -28,7 +28,9 @@ include(CMakePackageConfigHelpers)
 include(Utils.cmake)
 
 # Ignore weak attribute warning
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
+if(NOT MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
+endif()
 
 set(ABSL_ENABLE_INSTALL ON)
 set(ABSL_PROPAGATE_CXX_STD ON)

include/pytorch/tokenizers/tiktoken.h

@@ -23,6 +23,12 @@
 #include <pytorch/tokenizers/result.h>
 #include <pytorch/tokenizers/tokenizer.h>
 
+#ifdef _WIN32
+// ssize_t isn't available on Windows. Alias it to the Windows SSIZE_T value.
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#endif
+
 namespace tokenizers {
 
 static constexpr int32_t kSpecialTokensSize = 256;

setup.py

@@ -30,9 +30,6 @@ class CMakeBuild(build_ext):
     def build_extension(self, ext):  # noqa C901
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
 
-        # Ensure the extension goes into the pytorch_tokenizers package directory
-        extdir = os.path.join(extdir, "pytorch_tokenizers")
-
         # Required for auto-detection & inclusion of auxiliary "native" libs
         if not extdir.endswith(os.path.sep):
             extdir += os.path.sep
@@ -55,6 +52,10 @@ def build_extension(self, ext):  # noqa C901
         ]
         build_args = ["--target", "pytorch_tokenizers_cpp"]
 
+        # Use Clang for Windows builds.
+        if sys.platform == "win32":
+            cmake_args += ["-T ClangCL"]
+
         # Adding CMake arguments set as environment variable
         # (needed e.g. to build for ARM OSX on conda-forge)
         if "CMAKE_ARGS" in os.environ:
@@ -124,6 +125,15 @@ def build_extension(self, ext):  # noqa C901
             ["cmake", "--build", "."] + build_args, cwd=build_temp, check=True
         )
 
+        if sys.platform == "win32":
+            # Setuptools seems to look for the artifact in a different location on
+            # Windows. I'm not entirely sure why... Move the artifact up one level.
+            pyd_files = list(Path(extdir).glob("*.pyd"))
+            artifact_dst_dir = Path(extdir).parent
+            for f in pyd_files:
+                dst_path = artifact_dst_dir / os.path.basename(f)
+                os.replace(f, dst_path)
+
 
 setup(
     name="pytorch-tokenizers",
@@ -132,7 +142,7 @@ def build_extension(self, ext):  # noqa C901
     long_description_content_type="text/markdown",
     url="https://github.com/meta-pytorch/tokenizers",
     packages=find_packages(),
-    ext_modules=[CMakeExtension("pytorch_tokenizers_cpp")],
+    ext_modules=[CMakeExtension("pytorch_tokenizers.pytorch_tokenizers_cpp")],
     cmdclass={"build_ext": CMakeBuild},
     zip_safe=False,
     python_requires=">=3.10",

src/hf_tokenizer.cpp

@@ -34,14 +34,14 @@ Error HFTokenizer::load(const std::string& path) {
   std::string model_config_json = "";
   if (fs::is_directory(path)) {
     const fs::path root(path);
-    model_json = root / "tokenizer.json";
+    model_json = (root / "tokenizer.json").string();
     if (!fs::exists(model_json)) {
       TK_LOG(Info, "no tokenizer.json found in %s", path.c_str());
       return Error::LoadFailure;
     }
     const auto model_config_json_path = root / "tokenizer_config.json";
     if (fs::exists(model_config_json_path)) {
-      model_config_json = model_config_json_path;
+      model_config_json = model_config_json_path.string();
     }
   }