Skip to content

showcases additional cache functionality #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 2, 2024
Merged

showcases additional cache functionality #198

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3773c62
showcases additional cache functionality
justin-cechmanek Aug 2, 2024
1d4d08c
small tweak
tylerhutcherson Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 212 additions & 21 deletions docs/user_guide/llmcache_03.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,13 @@
"outputs": [],
"source": [
"import os\n",
"from openai import OpenAI\n",
"import getpass\n",
"import time\n",
"\n",
"import numpy as np\n",
"\n",
"from openai import OpenAI\n",
"\n",
"\n",
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"False\"\n",
"\n",
"api_key = os.getenv(\"OPENAI_API_KEY\") or getpass.getpass(\"Enter your OpenAI API key: \")\n",
Expand Down Expand Up @@ -77,7 +78,15 @@
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"22:11:38 redisvl.index.index INFO Index already exists, not overwriting.\n"
]
}
],
"source": [
"from redisvl.extensions.llmcache import SemanticCache\n",
"\n",
Expand Down Expand Up @@ -385,7 +394,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -409,14 +418,14 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Without caching, a call to openAI to answer this simple question took 0.9312698841094971 seconds.\n"
"Without caching, a call to openAI to answer this simple question took 0.9034533500671387 seconds.\n"
]
},
{
Expand All @@ -425,7 +434,7 @@
"'llmcache:67e0f6e28fe2a61c0022fd42bf734bb8ffe49d3e375fd69d692574295a20fc1a'"
]
},
"execution_count": 17,
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -452,8 +461,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Avg time taken with LLM cache enabled: 0.4896167993545532\n",
"Percentage of time saved: 47.42%\n"
"Avg time taken with LLM cache enabled: 0.09753389358520508\n",
"Percentage of time saved: 89.2%\n"
]
}
],
Expand Down Expand Up @@ -488,22 +497,22 @@
"├─────────────────────────────┼─────────────┤\n",
"│ num_docs │ 1 │\n",
"│ num_terms │ 19 │\n",
"│ max_doc_id │ 3 │\n",
"│ num_records │ 23 │\n",
"│ max_doc_id │ 6 │\n",
"│ num_records │ 53 │\n",
"│ percent_indexed │ 1 │\n",
"│ hash_indexing_failures │ 0 │\n",
"│ number_of_uses │ 19 │\n",
"│ bytes_per_record_avg │ 5.30435 │\n",
"│ number_of_uses │ 45 │\n",
"│ bytes_per_record_avg │ 45.0566 │\n",
"│ doc_table_size_mb │ 0.000134468 │\n",
"│ inverted_sz_mb │ 0.000116348 │\n",
"│ inverted_sz_mb │ 0.00227737 │\n",
"│ key_table_size_mb │ 2.76566e-05 │\n",
"│ offset_bits_per_record_avg │ 8 │\n",
"│ offset_vectors_sz_mb │ 2.09808e-05 │\n",
"│ offsets_per_term_avg │ 0.956522 │\n",
"│ records_per_doc_avg │ 23 │\n",
"│ offset_vectors_sz_mb │ 3.91006e-05 │\n",
"│ offsets_per_term_avg │ 0.773585 │\n",
"│ records_per_doc_avg │ 53 │\n",
"│ sortable_values_size_mb │ 0 │\n",
"│ total_indexing_time │ 1.211 │\n",
"│ total_inverted_index_blocks │ 19 │\n",
"│ total_indexing_time │ 19.454 │\n",
"│ total_inverted_index_blocks │ 21 │\n",
"│ vector_index_sz_mb │ 3.0161 │\n",
"╰─────────────────────────────┴─────────────╯\n"
]
Expand All @@ -516,13 +525,195 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"# Clear the cache AND delete the underlying index\n",
"llmcache.delete()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Cache Access Controls, Tags & Filters\n",
"When running complex workflows with similar applications, or handling multiple users it's important to keep data segregated. Building on top of RedisVL's support for complex and hybrid queries we can tag and filter cache entries using custom-defined `filterable_fields`.\n",
"\n",
"Let's store multiple users' data in our cache with similar prompts and ensure we return only the correct user information:"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'private_cache:5de9d651f802d9cc3f62b034ced3466bf886a542ce43fe1c2b4181726665bf9c'"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"private_cache = SemanticCache(\n",
" name=\"private_cache\",\n",
" filterable_fields=[{\"name\": \"user_id\", \"type\": \"tag\"}]\n",
")\n",
"\n",
"private_cache.store(\n",
" prompt=\"What is the phone number linked to my account?\",\n",
" response=\"The number on file is 123-555-0000\",\n",
" filters={\"user_id\": \"abc\"},\n",
")\n",
"\n",
"private_cache.store(\n",
" prompt=\"What's the phone number linked in my account?\",\n",
" response=\"The number on file is 123-555-1111\",\n",
" filters={\"user_id\": \"def\"},\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"found 1 entry \n",
"The number on file is 123-555-0000\n"
]
}
],
"source": [
"from redisvl.query.filter import Tag\n",
"\n",
"# define user id filter\n",
"user_id_filter = Tag(\"user_id\") == \"abc\"\n",
"\n",
"response = private_cache.check(\n",
" prompt=\"What is the phone number linked to my account?\",\n",
" filter_expression=user_id_filter,\n",
" num_results=2\n",
")\n",
"\n",
"print(f\"found {len(response)} entry \\n{response[0]['response']}\")"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"# Cleanup\n",
"private_cache.delete()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Multiple `filterable_fields` can be defined on a cache, and complex filter expressions can be constructed to filter on these fields, as well as the default fields already present."
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'account_data:d48ebb3a2efbdbc17930a8c7559c548a58b562b2572ef0be28f0bb4ece2382e1'"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"complex_cache = SemanticCache(\n",
" name='account_data',\n",
" filterable_fields=[\n",
" {\"name\": \"user_id\", \"type\": \"tag\"},\n",
" {\"name\": \"account_type\", \"type\": \"tag\"},\n",
" {\"name\": \"account_balance\", \"type\": \"numeric\"},\n",
" {\"name\": \"transaction_amount\", \"type\": \"numeric\"}\n",
" ]\n",
")\n",
"complex_cache.store(\n",
" prompt=\"what is my most recent checking account transaction under $100?\",\n",
" response=\"Your most recent transaction was for $75\",\n",
" filters={\"user_id\": \"abc\", \"account_type\": \"checking\", \"transaction_amount\": 75},\n",
")\n",
"complex_cache.store(\n",
" prompt=\"what is my most recent savings account transaction?\",\n",
" response=\"Your most recent deposit was for $300\",\n",
" filters={\"user_id\": \"abc\", \"account_type\": \"savings\", \"transaction_amount\": 300},\n",
")\n",
"complex_cache.store(\n",
" prompt=\"what is my most recent checking account transaction over $200?\",\n",
" response=\"Your most recent transaction was for $350\",\n",
" filters={\"user_id\": \"abc\", \"account_type\": \"checking\", \"transaction_amount\": 350},\n",
")\n",
"complex_cache.store(\n",
" prompt=\"what is my checking account balance?\",\n",
" response=\"Your current checking account is $1850\",\n",
" filters={\"user_id\": \"abc\", \"account_type\": \"checking\"},\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"found 1 entry\n",
"Your most recent transaction was for $350\n"
]
}
],
"source": [
"from redisvl.query.filter import Num\n",
"\n",
"value_filter = Num(\"transaction_amount\") > 100\n",
"account_filter = Tag(\"account_type\") == \"checking\"\n",
"complex_filter = value_filter & account_filter\n",
"\n",
"# check for checking account transactions over $100\n",
"complex_cache.set_threshold(0.3)\n",
"response = complex_cache.check(\n",
" prompt=\"what is my most recent checking account transaction?\",\n",
" filter_expression=complex_filter,\n",
" num_results=5\n",
")\n",
"print(f'found {len(response)} entry')\n",
"print(response[0][\"response\"])"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"# Cleanup\n",
"complex_cache.delete()"
]
}
],
"metadata": {
Expand All @@ -541,7 +732,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
"version": "3.12.2"
},
"orig_nbformat": 4
},
Expand Down
Loading