notebooks/evaluate/zero_shot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fbc7e716-f7f4-42a8-bdec-0a40a4b2d405",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/alireza/vlm_toolbox\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.chdir('../../vlm_toolbox/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ec644e8-79c4-41df-9187-7f11e765181c",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3406a0cd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "import warnings\n",
    "\n",
    "import torch\n",
    "\n",
    "from config.enums import (\n",
    "    ImageDatasets,\n",
    "    ModelType,\n",
    "    Setups,\n",
    "    Sources,\n",
    "    Stages,\n",
    "    Trainers,\n",
    ")\n",
    "from config.setup import Setup\n",
    "from pipeline.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecc0afe6-e276-4d7b-8b59-c0a8ff23a9d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "959be6b9-c2be-4f5b-98d7-0722f62c84f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flush():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe3c82c9",
   "metadata": {},
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dbb87d95-9c99-4648-94d4-80a89e28656b",
   "metadata": {},
   "outputs": [],
   "source": [
    "HIERARCHY_LEVELS = {\n",
    "    'coarse': None,\n",
    "    'default': 'coarse',\n",
    "}\n",
    "\n",
    "REVERSED_HIERARCHY_LEVELS = {\n",
    "    'coarse': 'default',\n",
    "    'default': None,\n",
    "}\n",
    "\n",
    "LEVELS_TOP_N_CHILDREN_TO_PROPAGATE = {\n",
    "    'coarse': 3,\n",
    "    'default': None,\n",
    "}\n",
    "\n",
    "TOP_K = 5\n",
    "LEVEL_NAMES = list(HIERARCHY_LEVELS.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "13028e23-f756-48c6-9e53-8d686771958f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>Labels Summary...</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Current label column: simplified</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Number of classes: 1000</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Number of labels: 1000</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Prompt mode: Soft</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Soft prompt group: generic</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Number of soft prompts: 1</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Dataset's prompt context initialization: it is an image of</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>\n",
       "---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Loading Model...</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>\n",
       "---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model weights loaded successfully from /home/alireza/io/model/zero_shot/imagenet1k/clip_vit_b_16/open_ai/coop/16_shots/default/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>\n",
       "---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>Model Summary...</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>coop - clip_vit_b_16</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Total model parameters: 149628929</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>🔥 Trainable model parameters: 8192 (0.01%)</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>🧊 Frozen model parameters: 149620737 (99.99%)\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Attribute Name: model, Type: CustomCLIP, Param Share: 100.00%:</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>-    Attribute Name: prompt_learner, Type: PromptLearner, Param Share: 0.01%:</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>--       🔥 Trainable: 100.00%, 🧊 Frozen: 0.00%</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>-    Attribute Name: image_encoder, Type: VisionTransformer, Param Share: 57.60%:</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>--       🔥 Trainable: 0.00%, 🧊 Frozen: 100.00%</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>-    Attribute Name: text_encoder, Type: TextEncoder, Param Share: 25.48%:</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>--       🔥 Trainable: 0.00%, 🧊 Frozen: 100.00%</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>-    Attribute Name: token_embedding, Type: Embedding, Param Share: 16.91%:</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>--       🔥 Trainable: 0.00%, 🧊 Frozen: 100.00%</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Device: cuda:0</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Model size: 453.882MB</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Logit scale: 100.0000</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Modalities @ VALIDATION\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>M1: Modality(modality_type='image', identifier='class_id', raw_keys=['image'], preprocessed_keys={'input': ['image'], 'output': ['pixel_values']}, key='m1', stage='validation', embedding_key={'input': ['pixel_values'], 'output': ['image_embeds']}, status='raw', requires_grad=false, requires_preprocess=true, perform_augmentation=false)\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>M2: Modality(modality_type='text', identifier='label_id', raw_keys=['label_id'], preprocessed_keys={'input': ['label'], 'output': ['input_ids']}, key='m2', stage='validation', embedding_key={'input': ['label_id'], 'output': ['text_embeds']}, status='raw', requires_grad=false, requires_preprocess=false, perform_augmentation=false)</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>\n",
       "---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Loading Datasets...</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>m1 @ validation (image -> raw) Loaded.</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>m2 @ validation (text -> raw) Loaded.</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>\n",
       "---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>Datasets Summary...</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>m1 @ validation (image -> embedding):\n",
       " Dataset({\n",
       "    features: ['class_id', 'image_embeds'],\n",
       "    num_rows: 50000\n",
       "})\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>m2 @ validation (text -> raw):\n",
       " Dataset({\n",
       "    features: ['label_id', 'label'],\n",
       "    num_rows: 1000\n",
       "})\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>\n",
       "---------------------------</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: blue;'>---------------------------\n",
       "</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='49' max='49' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [49/49 00:27]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 1.1691420078277588, 'eval_top_1': 0.69364, 'eval_top_2': 0.81794, 'eval_top_3': 0.86794, 'eval_top_4': 0.8945, 'eval_top_5': 0.91232, 'eval_runtime': 102.4459, 'eval_samples_per_second': 488.063, 'eval_steps_per_second': 0.478}\n",
      "{'per_sample': '/home/alireza/novel_coop/zero_shot/imagenet1k/clip_vit_b_16/open_ai/coop/16_shots/default/per_sample.parquet', 'overall': '/home/alireza/novel_coop/zero_shot/imagenet1k/clip_vit_b_16/open_ai/coop/16_shots/default/overall.parquet'}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Tearing down resources and cleaning up...</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute model</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute trainer</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute trainer_args</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute trainer_callbacks</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute metric_evaluator</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute annotations_config</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute label_handler</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute model_config</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute stage_handlers</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute stage_handlers_configs</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute original_class_ids</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Deleted attribute original_class_id_label_id_adj_matrix</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style='color: black;'>Resources have been freed up!</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for col_name in cols[-1:]:\n",
    "    train_stage_setup = Setup(\n",
    "        dataset_name=ImageDatasets.IMAGENET_1K,\n",
    "        trainer_name=Trainers.COOP,\n",
    "        setup_type=Setups.EVAL_ONLY,\n",
    "        model_type=ModelType.ZERO_SHOT,\n",
    "        source=Sources.OPEN_AI,\n",
    "        train_split='validation',\n",
    "        label_column_name=col_name,\n",
    "        n_shots=16,\n",
    "        # annotations_key_value_criteria={'kingdom': ['Animalia']},\n",
    "        top_k=len(labels_df[col_name].value_counts()),\n",
    "        load_from_checkpoint=True,\n",
    "    )\n",
    "\n",
    "    # print(train_stage_setup.get_model_checkpoint_path())\n",
    "\n",
    "    pipeline = Pipeline(train_stage_setup, device_type=DEVICE_TYPE, logger=logger)\n",
    "    pipeline.setup_labels()\n",
    "    pipeline.setup_model()\n",
    "    pipeline.setup_data(stages=[Stages.EVAL])\n",
    "    metrics = pipeline.evaluate()\n",
    "    print(metrics)\n",
    "    to_save_dir = '/home/alireza/novel_coop/' + train_stage_setup.get_relative_save_path()\n",
    "    saved_dirs_dict = pipeline.save_metrics(directory=to_save_dir, main_metric_only=False)\n",
    "    print(saved_dirs_dict)\n",
    "    pipeline.tear_down()\n",
    "    del pipeline\n",
    "    del train_stage_setup\n",
    "    flush()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "clip",
   "language": "python",
   "name": "clip"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}