Pytorch/[Kaggle] (Pytorch) Handwritten_Image_Classification (Grapheme language).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.metrics import recall_score\n",
    "import matplotlib.pyplot as plt\n",
    "from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
    "\n",
    "import joblib\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import random\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from albumentations import (\n",
    "    HorizontalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n",
    "    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n",
    "    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n",
    "    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Cutout, Rotate, Normalize\n",
    ")\n",
    "from albumentations.pytorch import ToTensor, ToTensorV2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 623\n",
    "def seed_everything(seed):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "seed_everything(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Read Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'C:\\\\Users\\\\bokhy\\\\Desktop\\\\kaggle\\\\bengaliai-cv19\\\\'\n",
    "df_train = pd.read_csv(os.path.join(path, 'train.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>grapheme_root</th>\n",
       "      <th>vowel_diacritic</th>\n",
       "      <th>consonant_diacritic</th>\n",
       "      <th>grapheme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Train_0</td>\n",
       "      <td>15</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>ক্ট্রো</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Train_1</td>\n",
       "      <td>159</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>হ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Train_2</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>খ্রী</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Train_3</td>\n",
       "      <td>53</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>র্টি</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Train_4</td>\n",
       "      <td>71</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>থ্রো</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  image_id  grapheme_root  vowel_diacritic  consonant_diacritic grapheme\n",
       "0  Train_0             15                9                    5   ক্ট্রো\n",
       "1  Train_1            159                0                    0        হ\n",
       "2  Train_2             22                3                    5     খ্রী\n",
       "3  Train_3             53                2                    2     র্টি\n",
       "4  Train_4             71                9                    5     থ্রো"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Split the dataset (Train/Valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['id'] = df_train['image_id'].apply(lambda x:int(x.split('_')[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train[['id','grapheme_root','vowel_diacritic','consonant_diacritic']].values[:,0]\n",
    "y = df_train[['id','grapheme_root','vowel_diacritic','consonant_diacritic']].values[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mskf = MultilabelStratifiedKFold(n_splits = 6, random_state= 623)\n",
    "df_train['fold'] = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,(train_idx, vld_idx) in enumerate(mskf.split(X,y)):\n",
    "    df_train.loc[vld_idx, 'fold'] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5    33474\n",
       "0    33474\n",
       "4    33473\n",
       "3    33473\n",
       "2    33473\n",
       "1    33473\n",
       "Name: fold, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# look at the eqaul sample keeping distribution of all 3 cahraters\n",
    "df_train['fold'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>grapheme_root</th>\n",
       "      <th>vowel_diacritic</th>\n",
       "      <th>consonant_diacritic</th>\n",
       "      <th>grapheme</th>\n",
       "      <th>id</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Train_0</td>\n",
       "      <td>15</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>ক্ট্রো</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Train_1</td>\n",
       "      <td>159</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>হ</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Train_2</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>খ্রী</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Train_3</td>\n",
       "      <td>53</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>র্টি</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Train_4</td>\n",
       "      <td>71</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>থ্রো</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200835</th>\n",
       "      <td>Train_200835</td>\n",
       "      <td>22</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>র্খে</td>\n",
       "      <td>200835</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200836</th>\n",
       "      <td>Train_200836</td>\n",
       "      <td>65</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>ত্তো</td>\n",
       "      <td>200836</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200837</th>\n",
       "      <td>Train_200837</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>অ্যা</td>\n",
       "      <td>200837</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200838</th>\n",
       "      <td>Train_200838</td>\n",
       "      <td>152</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>স্নো</td>\n",
       "      <td>200838</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200839</th>\n",
       "      <td>Train_200839</td>\n",
       "      <td>127</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>ল্টি</td>\n",
       "      <td>200839</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200840 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            image_id  grapheme_root  vowel_diacritic  consonant_diacritic  \\\n",
       "0            Train_0             15                9                    5   \n",
       "1            Train_1            159                0                    0   \n",
       "2            Train_2             22                3                    5   \n",
       "3            Train_3             53                2                    2   \n",
       "4            Train_4             71                9                    5   \n",
       "...              ...            ...              ...                  ...   \n",
       "200835  Train_200835             22                7                    2   \n",
       "200836  Train_200836             65                9                    0   \n",
       "200837  Train_200837              2                1                    4   \n",
       "200838  Train_200838            152                9                    0   \n",
       "200839  Train_200839            127                2                    0   \n",
       "\n",
       "       grapheme      id  fold  \n",
       "0        ক্ট্রো       0     1  \n",
       "1             হ       1     1  \n",
       "2          খ্রী       2     0  \n",
       "3          র্টি       3     4  \n",
       "4          থ্রো       4     2  \n",
       "...         ...     ...   ...  \n",
       "200835     র্খে  200835     2  \n",
       "200836     ত্তো  200836     5  \n",
       "200837     অ্যা  200837     0  \n",
       "200838     স্নো  200838     1  \n",
       "200839     ল্টি  200839     2  \n",
       "\n",
       "[200840 rows x 7 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_train['fold'] = pd.read_csv(os.path.join(path, 'df_folds.csv'))['fold']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_fold = [i for i in range(6) if i not in [5]]\n",
    "vld_fold = [5]\n",
    "trn_idx = df_train.loc[df_train['fold'].isin(trn_fold)].index\n",
    "vld_idx = df_train.loc[df_train['fold'].isin(vld_fold)].index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Define the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_parquet = ['train_image_data_0.parquet',\n",
    "        'train_image_data_1.parquet',\n",
    "        'train_image_data_2.parquet',\n",
    "        'train_image_data_3.parquet']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50210/50210 [02:14<00:00, 373.10it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 50210/50210 [02:07<00:00, 394.28it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 50210/50210 [02:09<00:00, 386.54it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 50210/50210 [02:07<00:00, 392.41it/s]\n"
     ]
    }
   ],
   "source": [
    "# Careful (take long)\n",
    "for fname in train_parquet:\n",
    "    F = os.path.join(path, fname)\n",
    "    df_train = pd.read_parquet(F)\n",
    "    img_ids = df_train['image_id'].values\n",
    "    img_array = df_train.iloc[:,1:].values\n",
    "    for idx in tqdm(range(len(df_train))):\n",
    "        img_id = img_ids[idx]\n",
    "        img = img_array[idx]\n",
    "        joblib.dump(img, f'C:\\\\Users\\\\bokhy\\\\Desktop\\\\kaggle\\\\bengaliai-cv19\\\\train_images\\\\{img_id}.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BengaliDataset(Dataset):\n",
    "    def __init__(self, csv, img_height, img_width, transform):\n",
    "        self.csv = csv.reset_index()\n",
    "        self.img_ids = csv['image_id'].values\n",
    "        self.img_height = img_height\n",
    "        self.img_width = img_width\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.csv)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        img_id = self.img_ids[index]\n",
    "        img = joblib.load(f'C:\\\\Users\\\\bokhy\\\\Desktop\\\\kaggle\\\\bengaliai-cv19\\\\train_images\\\\{img_id}.pkl' )\n",
    "        img = img.reshape(self.img_height, self.img_width).astype(np.uint8)\n",
    "        img = 255 - img\n",
    "        \n",
    "        img = img[:, :, np.newaxis]\n",
    "        img = np.repeat(img, 3, 2)\n",
    "        \n",
    "        if self.transform is not None:\n",
    "            img = self.transform(image = img)['image']\n",
    "            \n",
    "        label_1 = self.csv.iloc[index].grapheme_root\n",
    "        label_2 = self.csv.iloc[index].vowel_diacritic\n",
    "        label_3 = self.csv.iloc[index].consonant_diacritic\n",
    "        \n",
    "        return img, np.array([label_1, label_2, label_3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Define Image Augmenation (Albumentation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_augmentation = Compose([\n",
    "    Rotate(20),\n",
    "    ToTensor()\n",
    "])\n",
    "\n",
    "# We dont do augmentation on Validation set\n",
    "valid_augmentation = Compose([\n",
    "    ToTensor()\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Load Dataset on DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train and Valid Data sets are created per below code\n",
    "trn_dataset = BengaliDataset(csv = df_train.loc[trn_idx][:1000],\n",
    "                            img_height = 137,\n",
    "                            img_width = 236,\n",
    "                            transform = train_augmentation)\n",
    "\n",
    "vld_dataset = BengaliDataset(csv = df_train.loc[vld_idx][:1000],\n",
    "                            img_height = 137,\n",
    "                            img_width = 236,\n",
    "                            transform = valid_augmentation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create batch to use it for learning\n",
    "trn_loader = DataLoader(trn_dataset,\n",
    "                       shuffle = True,\n",
    "                       num_workers = 0, \n",
    "                       batch_size = 128 # or 256\n",
    "                       )\n",
    "\n",
    "vld_loader = DataLoader(vld_dataset,\n",
    "                       shuffle = False,\n",
    "                       num_workers = 0, \n",
    "                       batch_size = 128 # or 256\n",
    "                       )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Create a model, optimizer and criterion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pretrainedmodels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'resnet50'\n",
    "model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the last layer to fit our task\n",
    "in_features = model.last_linear.in_features\n",
    "# total output numbers are 186 \n",
    "model.last_linear = torch.nn.Linear(in_features, 186)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use GPU computing\n",
    "model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Optimizer (make it learn parameters of our model)\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr = 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, \n",
    "                                                      mode = 'max',\n",
    "                                                      verbose = True,\n",
    "                                                      patience = 7,\n",
    "                                                      factor = 0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_score = - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86372a98c35247ea87f3cfcba0bb40c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 260.00 MiB (GPU 0; 4.00 GiB total capacity; 2.84 GiB already allocated; 46.96 MiB free; 54.21 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-605dfd1c31f1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mtargets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtargets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mlogits\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mgrapheme\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlogits\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;36m168\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pretrainedmodels\\models\\torchvision_models.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    338\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    339\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 340\u001b[1;33m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    341\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlogits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    342\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\pretrainedmodels\\models\\torchvision_models.py\u001b[0m in \u001b[0;36mfeatures\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    325\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmaxpool\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 327\u001b[1;33m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayer1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    328\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayer2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    329\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayer3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\container.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m             \u001b[0minput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torchvision\\models\\resnet.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    104\u001b[0m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    105\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 106\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconv3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    107\u001b[0m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbn3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\conv.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    343\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 345\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconv2d_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    346\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\conv.py\u001b[0m in \u001b[0;36mconv2d_forward\u001b[1;34m(self, input, weight)\u001b[0m\n\u001b[0;32m    340\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[0;32m    341\u001b[0m         return F.conv2d(input, weight, self.bias, self.stride,\n\u001b[1;32m--> 342\u001b[1;33m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[0;32m    343\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 260.00 MiB (GPU 0; 4.00 GiB total capacity; 2.84 GiB already allocated; 46.96 MiB free; 54.21 MiB cached)"
     ]
    }
   ],
   "source": [
    "#for e in range(50):\n",
    "train_loss = []\n",
    "# this means that 'we start the training!'\n",
    "model.train()\n",
    "\n",
    "# tqdm_notebook wrap would show the whole epochs\n",
    "for inputs, targets in tqdm_notebook(trn_loader):\n",
    "\n",
    "    inputs = inputs.cuda()\n",
    "    targets = targets.cuda()\n",
    "\n",
    "    logits = model(inputs)\n",
    "\n",
    "    grapheme = logits[:, :168]\n",
    "    vowel = logits[:, 168:179]\n",
    "    cons = logits[:, 179:]\n",
    "\n",
    "    # targets[:,0] --> first column in targets is the grapheme\n",
    "    # targets[:,1] --> second column is the vowel\n",
    "    # targets[:,2] --> third  column is the consonants\n",
    "    loss = loss_fn(grapheme, targets[:,0]) + loss_fn(vowel, targets[:,1]) + loss_fn(cons, targets[:,2]) \n",
    "\n",
    "    loss.backward()\n",
    "\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    # loss.item would make is a value from tensor\n",
    "    train_loss.append(loss.item())\n",
    "\n",
    "val_loss = []\n",
    "val_true = []\n",
    "val_pred = []\n",
    "\n",
    "model.eval()\n",
    "\n",
    "with torch.no_grad():\n",
    "    for inputs, targets in tqdm_notebook(vld_loader):\n",
    "\n",
    "        inputs = inputs.cuda()\n",
    "        targets = targets.cuda()\n",
    "\n",
    "        logits = model(inputs)\n",
    "\n",
    "        grapheme = logits[:, :168]\n",
    "        vowel = logits[:, 168:179]\n",
    "        cons = logits[:, 179:]\n",
    "\n",
    "        # targets[:,0] --> first column in targets is the grapheme\n",
    "        # targets[:,1] --> second column is the vowel\n",
    "        # targets[:,2] --> third  column is the consonants\n",
    "        loss = loss_fn(grapheme, targets[:,0]) + loss_fn(vowel, targets[:,1]) + loss_fn(cons, targets[:,2]) \n",
    "\n",
    "        val_loss.append(loss.item())\n",
    "\n",
    "        grapheme = grapheme.cpu().argmax(dim=1).data.numpy()\n",
    "        vowel    = vowel.cpu().argmax(dim=1).data.numpy()\n",
    "        cons     = cons.cpu().argmax(dim=1).data.numpy()\n",
    "\n",
    "        val_true.append(targets.cpu().numpy())\n",
    "        val_pred.append(np.stack([grapheme,vowel,cons], axis = 1))\n",
    "\n",
    "val_true = np.concatenate(val_true)\n",
    "val_pred = np.concatenate(val_pred)\n",
    "\n",
    "val_loss = np.mean(val_loss)\n",
    "train_loss = np.mean(train_loss)\n",
    "\n",
    "score_g = recall_score(val_true[: , 0], val_pred[:, 0], average = 'macro')\n",
    "score_v = recall_score(val_true[: , 1], val_pred[:, 1], average = 'macro')\n",
    "score_c = recall_score(val_true[: , 2], val_pred[:, 2], average = 'macro')\n",
    "\n",
    "final_score = np.average([score_g , score_v , score_c], weights = [2,1,1])\n",
    "\n",
    "print(f'train_loss: {train_loss: .5f}; val_loss: {val_loss: .5f}; score: {final_score: .5f}')\n",
    "print(f'score_g: {score_g: .5f}; score_v: {score_v: .5f}; score_c: {score_c: .5f}')\n",
    "\n",
    "if final_score > best_score:\n",
    "    best_score = final_score\n",
    "\n",
    "    stat_dict = model.cpu().stat_dict()\n",
    "    model = model.cpu()\n",
    "    torch.save(stat_dict, 'model.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To see model's weight\n",
    "model.load_stat_dict(torch.load('./model.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}