diff --git a/docs/user_guide/04_vectorizers.ipynb b/docs/user_guide/04_vectorizers.ipynb index d5870b88..3647a2fc 100644 --- a/docs/user_guide/04_vectorizers.ipynb +++ b/docs/user_guide/04_vectorizers.ipynb @@ -223,12 +223,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# You can also create many embeddings at once\n", - "embeddings = hf.embed_many(sentences, as_buffer=True, dtype=\"float32\")\n" + "embeddings = hf.embed_many(sentences, as_buffer=True)\n" ] }, { @@ -660,13 +660,13 @@ "metadata": {}, "source": [ "## Selecting your float data type\n", - "When embedding text as byte arrays RedisVL supports 4 different floating point data types, `float16`, `float32`, `float64` and `bfloat16`.\n", + "When embedding text as byte arrays RedisVL supports 4 different floating point data types, `float16`, `float32`, `float64` and `bfloat16`, and 2 integer types, `int8` and `uint8`.\n", "Your dtype set for your vectorizer must match what is defined in your search index. If one is not explicitly set the default is `float32`." ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -675,8 +675,9 @@ "# subsequent calls to embed('', as_buffer=True) and embed_many('', as_buffer=True) will now encode as float16\n", "float16_bytes = vectorizer.embed('test sentence', as_buffer=True)\n", "\n", - "# you can override this setting on each individual method call\n", - "float64_bytes = vectorizer.embed('test sentence', as_buffer=True, dtype=\"float64\")\n", + "# to generate embeddings with different dtype instantiate a new vectorizer\n", + "vectorizer_64 = HFTextVectorizer(dtype='float64')\n", + "float64_bytes = vectorizer_64.embed('test sentence', as_buffer=True)\n", "\n", "float16_bytes != float64_bytes" ] @@ -690,38 +691,6 @@ "# cleanup\n", "index.delete()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "dist = max(i for i in range(10))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dist" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -746,4 +715,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/redisvl/redis/utils.py b/redisvl/redis/utils.py index 209e3ccf..bce8bf4b 100644 --- a/redisvl/redis/utils.py +++ b/redisvl/redis/utils.py @@ -41,7 +41,7 @@ def array_to_buffer(array: List[float], dtype: str) -> bytes: raise ValueError( f"Invalid data type: {dtype}. Supported types are: {[t.lower() for t in VectorDataType]}" ) - return np.array(array).astype(dtype.lower()).tobytes() + return np.array(array, dtype=dtype.lower()).tobytes() def buffer_to_array(buffer: bytes, dtype: str) -> List[Any]: diff --git a/redisvl/schema/fields.py b/redisvl/schema/fields.py index c5e0eac4..2b5db557 100644 --- a/redisvl/schema/fields.py +++ b/redisvl/schema/fields.py @@ -30,6 +30,8 @@ class VectorDataType(str, Enum): FLOAT16 = "FLOAT16" FLOAT32 = "FLOAT32" FLOAT64 = "FLOAT64" + INT8 = "INT8" + UINT8 = "UINT8" class VectorIndexAlgorithm(str, Enum): diff --git a/redisvl/utils/vectorize/text/azureopenai.py b/redisvl/utils/vectorize/text/azureopenai.py index 5af97dfa..5a77f7dd 100644 --- a/redisvl/utils/vectorize/text/azureopenai.py +++ b/redisvl/utils/vectorize/text/azureopenai.py @@ -5,6 +5,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer # ignore that openai isn't imported @@ -167,6 +168,7 @@ def _set_model_dims(self, model) -> int: stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], @@ -213,6 +215,7 @@ def embed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed( self, text: str, @@ -251,6 +254,7 @@ def embed( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") async def aembed_many( self, texts: List[str], @@ -299,6 +303,7 @@ async def aembed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") async def aembed( self, text: str, diff --git a/redisvl/utils/vectorize/text/bedrock.py b/redisvl/utils/vectorize/text/bedrock.py index 091beadb..d64ca9ca 100644 --- a/redisvl/utils/vectorize/text/bedrock.py +++ b/redisvl/utils/vectorize/text/bedrock.py @@ -6,6 +6,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer @@ -118,6 +119,7 @@ def _set_model_dims(self, model: str) -> int: stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed( self, text: str, @@ -158,6 +160,7 @@ def embed( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], diff --git a/redisvl/utils/vectorize/text/cohere.py b/redisvl/utils/vectorize/text/cohere.py index c30863b7..859fd83e 100644 --- a/redisvl/utils/vectorize/text/cohere.py +++ b/redisvl/utils/vectorize/text/cohere.py @@ -5,6 +5,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer # ignore that cohere isn't imported @@ -111,6 +112,7 @@ def _set_model_dims(self, model) -> int: raise ValueError(f"Error setting embedding model dimensions: {str(e)}") return len(embedding) + @deprecated_argument("dtype") def embed( self, text: str, @@ -177,6 +179,7 @@ def embed( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], diff --git a/redisvl/utils/vectorize/text/custom.py b/redisvl/utils/vectorize/text/custom.py index 31172f02..59122435 100644 --- a/redisvl/utils/vectorize/text/custom.py +++ b/redisvl/utils/vectorize/text/custom.py @@ -2,6 +2,7 @@ from pydantic.v1 import PrivateAttr +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer @@ -155,6 +156,7 @@ def _validate_sync_callables(self) -> int: return dims + @deprecated_argument("dtype") def embed( self, text: str, @@ -191,6 +193,7 @@ def embed( return self._process_embedding(result, as_buffer, dtype) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], @@ -239,6 +242,7 @@ def embed_many( return embeddings @validate_async + @deprecated_argument("dtype") async def aembed( self, text: str, @@ -280,6 +284,7 @@ async def aembed( return self._process_embedding(result, as_buffer, dtype) @validate_async + @deprecated_argument("dtype") async def aembed_many( self, texts: List[str], diff --git a/redisvl/utils/vectorize/text/huggingface.py b/redisvl/utils/vectorize/text/huggingface.py index b2fbabc0..c387f04e 100644 --- a/redisvl/utils/vectorize/text/huggingface.py +++ b/redisvl/utils/vectorize/text/huggingface.py @@ -2,6 +2,7 @@ from pydantic.v1 import PrivateAttr +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer @@ -79,6 +80,7 @@ def _set_model_dims(self): raise ValueError(f"Error setting embedding model dimensions: {str(e)}") return len(embedding) + @deprecated_argument("dtype") def embed( self, text: str, @@ -112,6 +114,7 @@ def embed( embedding = self._client.encode([text], **kwargs)[0] return self._process_embedding(embedding.tolist(), as_buffer, dtype) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], diff --git a/redisvl/utils/vectorize/text/mistral.py b/redisvl/utils/vectorize/text/mistral.py index bb636b33..44f189d6 100644 --- a/redisvl/utils/vectorize/text/mistral.py +++ b/redisvl/utils/vectorize/text/mistral.py @@ -5,6 +5,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer # ignore that mistralai isn't imported @@ -116,6 +117,7 @@ def _set_model_dims(self, model) -> int: stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], @@ -162,6 +164,7 @@ def embed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed( self, text: str, @@ -200,6 +203,7 @@ def embed( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") async def aembed_many( self, texts: List[str], @@ -248,6 +252,7 @@ async def aembed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") async def aembed( self, text: str, diff --git a/redisvl/utils/vectorize/text/openai.py b/redisvl/utils/vectorize/text/openai.py index 7dd2cf93..9fa9cc18 100644 --- a/redisvl/utils/vectorize/text/openai.py +++ b/redisvl/utils/vectorize/text/openai.py @@ -5,6 +5,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer # ignore that openai isn't imported @@ -121,6 +122,7 @@ def _set_model_dims(self, model) -> int: stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], @@ -167,6 +169,7 @@ def embed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed( self, text: str, @@ -205,6 +208,7 @@ def embed( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") async def aembed_many( self, texts: List[str], @@ -253,6 +257,7 @@ async def aembed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") async def aembed( self, text: str, diff --git a/redisvl/utils/vectorize/text/vertexai.py b/redisvl/utils/vectorize/text/vertexai.py index 6ddc28b6..1be10c7a 100644 --- a/redisvl/utils/vectorize/text/vertexai.py +++ b/redisvl/utils/vectorize/text/vertexai.py @@ -5,6 +5,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer @@ -128,6 +129,7 @@ def _set_model_dims(self) -> int: stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], @@ -173,6 +175,7 @@ def embed_many( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed( self, text: str, diff --git a/redisvl/utils/vectorize/text/voyageai.py b/redisvl/utils/vectorize/text/voyageai.py index e4de9817..b1f4ac72 100644 --- a/redisvl/utils/vectorize/text/voyageai.py +++ b/redisvl/utils/vectorize/text/voyageai.py @@ -5,6 +5,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from tenacity.retry import retry_if_not_exception_type +from redisvl.utils.utils import deprecated_argument from redisvl.utils.vectorize.base import BaseVectorizer # ignore that voyageai isn't imported @@ -113,6 +114,7 @@ def _set_model_dims(self, model) -> int: raise ValueError(f"Error setting embedding model dimensions: {str(e)}") return len(embedding) + @deprecated_argument("dtype") def embed( self, text: str, @@ -158,6 +160,7 @@ def embed( stop=stop_after_attempt(6), retry=retry_if_not_exception_type(TypeError), ) + @deprecated_argument("dtype") def embed_many( self, texts: List[str], @@ -237,6 +240,7 @@ def embed_many( ] return embeddings + @deprecated_argument("dtype") async def aembed_many( self, texts: List[str], @@ -316,6 +320,7 @@ async def aembed_many( ] return embeddings + @deprecated_argument("dtype") async def aembed( self, text: str, diff --git a/schemas/semantic_router.yaml b/schemas/semantic_router.yaml index 7175efb7..25aa08e1 100644 --- a/schemas/semantic_router.yaml +++ b/schemas/semantic_router.yaml @@ -19,4 +19,5 @@ vectorizer: model: sentence-transformers/all-mpnet-base-v2 routing_config: max_k: 2 - aggregation_method: avg \ No newline at end of file + aggregation_method: avg + distance_threshold: 0.3 diff --git a/tests/integration/test_vectorizers.py b/tests/integration/test_vectorizers.py index 65a07f3b..e1de4a46 100644 --- a/tests/integration/test_vectorizers.py +++ b/tests/integration/test_vectorizers.py @@ -289,7 +289,7 @@ def test_default_dtype(vectorizer_): ) def test_other_dtypes(vectorizer_): # test initializing dtype in constructor - for dtype in ["float16", "float32", "float64", "bfloat16"]: + for dtype in ["float16", "float32", "float64", "bfloat16", "int8", "uint8"]: if issubclass(vectorizer_, CustomTextVectorizer): vectorizer = vectorizer_(embed=lambda x: [1.0, 2.0, 3.0], dtype=dtype) elif issubclass(vectorizer_, AzureOpenAITextVectorizer): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cd8a3c9f..7b538c31 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -141,6 +141,14 @@ def test_conversion_with_various_dtypes(dtype): assert array_to_buffer(array, dtype=dtype) == expected +@pytest.mark.parametrize("dtype", ["int8", "uint8"]) +def test_conversion_with_integer_dtypes(dtype): + """Test conversion of a list of floats to bytes with various dtypes""" + array = [0.0, 1.0, 2.2, 3.5] + expected = np.array(array, dtype=dtype).tobytes() + assert array_to_buffer(array, dtype=dtype) == expected + + def test_conversion_with_invalid_floats(): """Test conversion with invalid float values (numpy should handle them)""" array = [float("inf"), float("-inf"), float("nan")]