examples/whisper_example.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tutorial below is not working 100% as expected but is open for improvements and left as a starting point for the community to work on. This tutorial only provided if you have a big enough GPU to run the big model of whisper to avoid using the paid API.\n",
    "\n",
    "#### Possible improvements:\n",
    "* Add a function to split audio into chunks and execute the model on each chunk\n",
    "* Try the pipeline with bigger models, small and medium failed really hard in the use case we tested on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the stuff below to have your dependencies installed, of course this tutorial assumes you have knowledgegpt installed\n",
    "\n",
    "# %git clone https://huggingface.co/spaces/openai/whisper\n",
    "# %cd whisper\n",
    "# %pip install -r requirements.txt\n",
    "# %pip install gradio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import whisper\n",
    "model = whisper.load_model(\"medium\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "     \n",
    "def inference(audio):\n",
    "    audio = whisper.load_audio(audio)\n",
    "    audio = whisper.pad_or_trim(audio)\n",
    "    \n",
    "    mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
    "    \n",
    "    _, probs = model.detect_language(mel)\n",
    "    \n",
    "    options = whisper.DecodingOptions()\n",
    "    result = whisper.decode(model, mel, options)\n",
    "    \n",
    "    print(result.text)\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "So, Twitch is the world's largest livestream platform and owned by Amazon, and while originally it was almost entirely for gaming, it's now expanded to all sorts of shows and IRL content, with top streamers averaging tens of thousands of viewers and special events and tournaments getting into the hundreds of thousands. With so much content, there's pretty much something for everyone, whether you're a gamer, a musician, a hobbyist, or just a member of Twitch staff.\n"
     ]
    }
   ],
   "source": [
    "result = inference(\"audio.mp3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"So, Twitch is the world's largest livestream platform and owned by Amazon, and while originally it was almost entirely for gaming, it's now expanded to all sorts of shows and IRL content, with top streamers averaging tens of thousands of viewers and special events and tournaments getting into the hundreds of thousands. With so much content, there's pretty much something for everyone, whether you're a gamer, a musician, a hobbyist, or just a member of Twitch staff.\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cut the text into 1 sentence chunks\n",
    "sentences = result.text.split(\".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(sentences, columns = ['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>So, Twitch is the world's largest livestream p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>With so much content, there's pretty much som...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             content\n",
       "0  So, Twitch is the world's largest livestream p...\n",
       "1   With so much content, there's pretty much som...\n",
       "2                                                   "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from knowledgegpt.extractors.base_extractor import BaseExtractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "from example_config import SECRET_KEY\n",
    "openai.api_key = SECRET_KEY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing embeddings...\n",
      "model_lang en\n",
      "Selected 3 document sections:\n",
      "1\n",
      "0\n",
      "2\n",
      "Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don't know.\"\n",
      "\n",
      "Context:\n",
      "\n",
      "*  With so much content, there's pretty much something for everyone, whether you're a gamer, a musician, a hobbyist, or just a member of Twitch staff\n",
      "* So, Twitch is the world's largest livestream platform and owned by Amazon, and while originally it was almost entirely for gaming, it's now expanded to all sorts of shows and IRL content, with top streamers averaging tens of thousands of viewers and special events and tournaments getting into the hundreds of thousands\n",
      "* \n",
      "\n",
      " Q: What is Twitch?\n",
      " A:\n",
      "max_tokens_turbo:  3903\n",
      "all_done!\n"
     ]
    }
   ],
   "source": [
    "basic_extractor = BaseExtractor(dataframe=df, embedding_extractor=\"hf\", model_lang=\"en\", is_turbo=True)\n",
    "answer, prompt, messages = basic_extractor.extract(\"What is Twitch?\", max_tokens=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Twitch is the world's largest livestream platform owned by Amazon that originally focused on gaming but has now expanded to all sorts of shows and IRL content.\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "knowledgegpt-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}