analysis_clevr_transfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import jacinle\n",
    "import jactorch\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "\n",
    "from experiments.desc_clevr_nesycoco import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import jacinle.io as io\n",
    "from os import path as osp\n",
    "\n",
    "data_parses = [\n",
    "    \"data/clevr-parsings/transfer-questions-ncprogram-gt.json\"\n",
    "    ]\n",
    "all_parses = dict()\n",
    "for filename in data_parses:\n",
    "    if filename.endswith('.p'):\n",
    "        content = io.load_pkl(filename)\n",
    "    else:\n",
    "        content = io.load(filename)\n",
    "    all_parses.update(content)\n",
    "\n",
    "data_dir = \"data/clevr-mini\"\n",
    "data_scenes_json = osp.join(data_dir, 'scenes.json')\n",
    "data_image_root = osp.join(data_dir, 'images')\n",
    "data_vocab_json = osp.join(data_dir, 'vocab.json')\n",
    "data_output_vocab_json = osp.join(data_dir, 'output-vocab.json')\n",
    "from left.domain import create_domain_from_parsing\n",
    "### limit the number of parses\n",
    "all_parses = dict(list(all_parses.items()))\n",
    "\n",
    "domain = create_domain_from_parsing(all_parses)\n",
    "\n",
    "evaluate_custom = \"puzzle\"\n",
    "data_questions_json = \"data/clevr-transfer/puzzle-20230513.json\"\n",
    "\n",
    "\n",
    "from left.data.clevr_custom_transfer import make_dataset\n",
    "dataset = make_dataset(evaluate_custom, data_scenes_json, data_questions_json, data_image_root, data_output_vocab_json)\n",
    "train_dataset = validation_dataset = dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1\n",
    "num_workers = 96\n",
    "train_dataloader = train_dataset.make_dataloader(batch_size, shuffle=False, drop_last=True, nr_workers=num_workers)\n",
    "validation_dataloader = validation_dataset.make_dataloader(batch_size, shuffle=False, drop_last=False, nr_workers=num_workers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "from jacinle.config.g import g\n",
    "g.concept_mapping = None\n",
    "domain.print_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jactorch.io import state_dict, load_state_dict\n",
    "from left.data.referit3d.vocabulary import Vocabulary\n",
    "from jactorch.train import TrainerEnv\n",
    "from jactorch.optim import AdamW\n",
    "from jactorch.cuda.copy import async_copy_to\n",
    "\n",
    "vocab = Vocabulary()\n",
    "model = make_model(None, domain, all_parses, train_dataset.output_vocab if hasattr(train_dataset, 'output_vocab') else train_dataset.unwrapped.output_vocab, custom_transfer=None)\n",
    "model.cuda()\n",
    "trainable_parameters = filter(lambda x: x.requires_grad, model.parameters())\n",
    "\n",
    "optimizer = AdamW(trainable_parameters, 0.001 , weight_decay=1e-2)\n",
    "\n",
    "trainer = TrainerEnv(model, optimizer)\n",
    "\n",
    "\n",
    "filename = \"[your model path]\"\n",
    "trainer.load_checkpoint(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "model.eval()\n",
    "group_meters = jacinle.GroupMeters()\n",
    "group_meters.reset()\n",
    "from jacinle.utils.tqdm import tqdm_pbar\n",
    "wrong_indices = []\n",
    "correct_indices = []\n",
    "parse_fails = []\n",
    "all_accuracies = []\n",
    "with tqdm_pbar(total=len(train_dataloader)) as pbar:\n",
    "    for (index, feed_dict) in enumerate(train_dataloader):\n",
    "        feed_dict = async_copy_to(feed_dict, 0)\n",
    "        try:\n",
    "            output_dict, extra_info = trainer.evaluate(feed_dict)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            parse_fails.append(index)\n",
    "            \n",
    "            continue\n",
    "\n",
    "        if evaluate_custom == 'ref':\n",
    "            for result, groundtruth in zip(output_dict['executions'], feed_dict['answer']):\n",
    "                if result is None:\n",
    "                    this_accuracy = 0\n",
    "                    wrong_indices.append(index)\n",
    "                elif result.dtype.typename != 'Object' or result.total_batch_dims != 1:\n",
    "                    this_accuracy = 0\n",
    "                    wrong_indices.append(index)\n",
    "                else:\n",
    "                    this_accuracy = int(result.tensor.argmax().item() == groundtruth)\n",
    "                    if this_accuracy == 0:\n",
    "                        wrong_indices.append(index)\n",
    "                    else:\n",
    "                        correct_indices.append(index)\n",
    "                all_accuracies.append(this_accuracy)\n",
    "        elif evaluate_custom in ('puzzle', 'rpm'):\n",
    "            for result, groundtruth in zip(output_dict['executions'], feed_dict['answer']):\n",
    "                if result is None:\n",
    "                    this_accuracy = 0\n",
    "                    wrong_indices.append(index)\n",
    "                elif result.dtype.typename != 'bool' or result.total_batch_dims != 0:\n",
    "                    this_accuracy = 0\n",
    "                    wrong_indices.append(index)\n",
    "                else:\n",
    "                    pred = (result.tensor.item() > 0.4)\n",
    "                    \n",
    "                    this_accuracy = int(pred == groundtruth)\n",
    "                    if this_accuracy == 0:\n",
    "                        print(result.tensor.item(), pred, groundtruth)\n",
    "                    if this_accuracy == 0:\n",
    "                        wrong_indices.append(index)\n",
    "                    else:\n",
    "                        correct_indices.append(index)\n",
    "                all_accuracies.append(this_accuracy)\n",
    "        else:\n",
    "            raise NotImplementedError()\n",
    "        acc = np.mean(np.array(all_accuracies))\n",
    "        pbar.set_description(group_meters.format_simple(\n",
    "            f'Validation Acc: {acc}',\n",
    "            {k: v for k, v in group_meters.val.items() if k.startswith('validation') and k.count('/') <= 2},\n",
    "            compressed=True\n",
    "        ), refresh=False)\n",
    "        pbar.update()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(parse_fails), len(wrong_indices), len(correct_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "iterator = iter(train_dataloader)\n",
    "data_all = [next(iterator) for _ in range(100)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import sys\n",
    "## Open a real image from address draw bouding boxes for objects\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "import numpy as np\n",
    "import json\n",
    "# plt.close()\n",
    "from copy import deepcopy\n",
    "from reascan import get_image\n",
    "feed_dict = data_all[wrong_indices[3]]\n",
    "# feed_dict = data_all[61]\n",
    "# feed_dict = deepcopy(feed_dict)\n",
    "# feed_dict[\"program\"][0] = feed_dict[\"program\"][0].replace(\"iota\",\"miota\")\n",
    "# print(feed_dict[\"program\"][0])\n",
    "feed_dict = async_copy_to(feed_dict, 0)\n",
    "print(feed_dict)\n",
    "q = feed_dict[\"question_raw\"][0]\n",
    "print(\"program:\", all_parses[q])\n",
    "\n",
    "\n",
    "output_dict, extra_info = trainer.evaluate(feed_dict)\n",
    "\n",
    "image = Image.open(osp.join(data_image_root, feed_dict[\"image_filename\"][0])).convert('RGB')\n",
    "print(feed_dict[\"question_raw\"])\n",
    "for i in range(len(output_dict[\"execution_traces\"][0])):\n",
    "    \n",
    "    if len(str(output_dict[\"execution_traces\"][0][i][0])) > 6:\n",
    "        tensor_values = np.array(output_dict[\"execution_traces\"][0][i][1].tensor.detach().cpu().numpy())\n",
    "        ## normalize tensor_values from 0-1\n",
    "        # tensor_values = (tensor_values - tensor_values.min()) / (tensor_values.max() - tensor_values.min())\n",
    "        fig, ax = plt.subplots()\n",
    "        ax.imshow(image)\n",
    "        for obj_index,obj in enumerate(feed_dict[\"objects_raw\"][0]):\n",
    "            # Create a Rectangle patch\n",
    "            rect = patches.Rectangle((obj[0], obj[1]), obj[2] - obj[0], obj[3] - obj[1], linewidth=1, edgecolor='r', facecolor=\"none\")\n",
    "            ax.add_patch(rect)\n",
    "            #if round(float(tensor_values[obj_index]),4)\n",
    "            ## check for nan values\n",
    "            # if (isinstance(tensor_values, np.array) and np.isnan(tensor_values[obj_index])) or ( not isinstance(tensor_values, np.array) and np.isnan(tensor_values)):\n",
    "            #     text = f\"{obj_index} nan \"\n",
    "            # else:\n",
    "            try:\n",
    "                if len(tensor_values.shape) > 1:\n",
    "                    text = f\"{obj_index}\"\n",
    "                else:\n",
    "                    text = f\"{obj_index} {round(float(tensor_values[obj_index]),4)} \"\n",
    "            except:\n",
    "                    \n",
    "                print(\"##\"*100)\n",
    "                text = f\"{obj_index} {tensor_values} \"\n",
    "            ax.text(obj[0], obj[1], text, color='white', fontsize=10, bbox=dict(facecolor='red', alpha=0.5))\n",
    "        plt.title(str(output_dict[\"execution_traces\"][0][i][0]))\n",
    "        plt.show()\n",
    "        from pprint import pprint\n",
    "        if len(tensor_values.shape) == 2:\n",
    "            for i in range(tensor_values.shape[0]):\n",
    "                print(i, [round(x,3) for x in tensor_values[i]])\n",
    "        else:\n",
    "            pprint(tensor_values)\n",
    "\n",
    "print(tensor_values.argmax().item())\n",
    "print(feed_dict[\"answer\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrong_indices\n",
    "address = \"./data/clevr-transfer/puzzle-20230513.json\"\n",
    "with open(address, \"r\") as f:\n",
    "    data = json.load(f)\n",
    "    for i in wrong_indices:\n",
    "        print(i)\n",
    "        print(data[\"puzzles\"][i][\"question\"])\n",
    "        print(data[\"puzzles\"][i][\"answer\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os\n",
    "from PIL import Image\n",
    "\n",
    "hf_cache = \"./cache\"\n",
    "os.environ[\"HF_HOME\"] = hf_cache\n",
    "\n",
    "import torch\n",
    "from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration\n",
    "\n",
    "processor = LlavaNextProcessor.from_pretrained(\"llava-hf/llava-v1.6-vicuna-7b-hf\",cache_dir=hf_cache)\n",
    "\n",
    "model = LlavaNextForConditionalGeneration.from_pretrained(\"llava-hf/llava-v1.6-vicuna-7b-hf\", torch_dtype=torch.float16, low_cpu_mem_usage=True,cache_dir=hf_cache).to(torch.device(\"cuda:0\"))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def answer(question, image_address):\n",
    "  \n",
    "  # Define a chat histiry and use `apply_chat_template` to get correctly formatted prompt\n",
    "  # Each value in \"content\" has to be a list of dicts with types (\"text\", \"image\") \n",
    "  conversation = [\n",
    "      {\n",
    "\n",
    "        \"role\": \"user\",\n",
    "        \"content\": [\n",
    "            {\"type\": \"text\", \"text\": question},\n",
    "            {\"type\": \"image\"},\n",
    "          ],\n",
    "      },\n",
    "  ]\n",
    "  prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)\n",
    "\n",
    "  raw_image = Image.open(image_address).convert(\"RGB\")\n",
    "  inputs = processor(prompt, raw_image, return_tensors='pt').to(0, torch.float16)\n",
    "\n",
    "  output = model.generate(**inputs, max_new_tokens=200, do_sample=False)\n",
    "  return processor.decode(output[0][2:])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import jacinle\n",
    "import jactorch\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "# from experiments.desc_neuro_codex_clevr_vec import *\n",
    "\n",
    "# with set_configs():\n",
    "#     configs.model.embedding_type = 'fasttex'\n",
    "import jacinle.io as io\n",
    "from os import path as osp\n",
    "\n",
    "data_dir = \"data/clevr-mini\"\n",
    "data_scenes_json = osp.join(data_dir, 'scenes.json')\n",
    "data_image_root = osp.join(data_dir, 'images')\n",
    "data_vocab_json = osp.join(data_dir, 'vocab.json')\n",
    "data_output_vocab_json = osp.join(data_dir, 'output-vocab.json')\n",
    "evaluate_custom = \"rpm\"\n",
    "data_questions_json = \"data/clevr-transfer/rpm-20230513.json\"\n",
    "\n",
    "from left.data.clevr_custom_transfer import make_dataset\n",
    "dataset = make_dataset(evaluate_custom, data_scenes_json, data_questions_json, data_image_root, data_output_vocab_json)\n",
    "validation_dataset = dataset\n",
    "batch_size = 1\n",
    "num_workers = 4\n",
    "validation_dataloader = validation_dataset.make_dataloader(batch_size, shuffle=False, drop_last=False, nr_workers=num_workers)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "group_meters = jacinle.GroupMeters()\n",
    "group_meters.reset()\n",
    "from jacinle.utils.tqdm import tqdm_pbar\n",
    "wrong_indices = []\n",
    "correct_indices = []\n",
    "parse_fails = []\n",
    "all_accuracies = []\n",
    "with tqdm_pbar(total=len(validation_dataloader)) as pbar:\n",
    "    for (index, feed_dict) in enumerate(validation_dataloader):\n",
    "        if evaluate_custom in ('puzzle', 'rpm'):\n",
    "            question = feed_dict[\"question_raw\"][0] +  \" \\n Yes or No?\"\n",
    "            image_address = f\"data/clevr-mini/images/{feed_dict['image_filename'][0]}\"\n",
    "            response = answer(question, image_address)\n",
    "            gt_answer = \"Yes\" if feed_dict['answer'][0] else \"No\"\n",
    "            ### extract yes ir no from response with regex\n",
    "            response = response.split(\"ASSISTANT: \")[-1].lower()\n",
    "            gt_answer = gt_answer.lower()\n",
    "            ### extract yes|no from response regex\n",
    "            response_extracted = re.findall(r\"yes|no\", response)\n",
    "            if response_extracted:\n",
    "                response_extracted = response_extracted[0]\n",
    "                if response_extracted == gt_answer:\n",
    "                    this_accuracy = 1\n",
    "                else:\n",
    "                    this_accuracy = 0  \n",
    "            else:\n",
    "                this_accuracy = 0\n",
    "                print(response, response_extracted, gt_answer, this_accuracy)\n",
    "                print(\"#####################################\")\n",
    "            all_accuracies.append(this_accuracy)\n",
    "            \n",
    "        else:### do ref manually\n",
    "            raise NotImplementedError()\n",
    "        acc = np.mean(np.array(all_accuracies))\n",
    "        pbar.set_description(group_meters.format_simple(\n",
    "            f'Validation Acc: {acc}',\n",
    "            {k: v for k, v in group_meters.val.items() if k.startswith('validation') and k.count('/') <= 2},\n",
    "            compressed=True\n",
    "        ), refresh=False)\n",
    "        pbar.update()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}