analysis_clevr_left.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import torch\n",
    "import jacinle\n",
    "import json\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "import jactorch\n",
    "\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from typing import Optional, Union, List, Dict\n",
    "from jacinle.config.environ_v2 import configs, set_configs\n",
    "from left.domain import create_domain_from_parsing\n",
    "from left.models.model import LeftModel\n",
    "import re\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from typing import Union, List, Dict\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "\n",
    "import jacinle\n",
    "from jacinle.utils.container import GView\n",
    "from jacinle.config.environ_v2 import configs, set_configs, def_configs\n",
    "from concepts.benchmark.clevr.clevr_constants import g_attribute_concepts, g_relational_concepts\n",
    "from left.models.model import LeftModel\n",
    "import json\n",
    "import os\n",
    "\n",
    "from experiments.desc_neuro_codex_clevr_learned_belongings import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import jacinle.io as io\n",
    "from os import path as osp\n",
    "\n",
    "# data_dir = \"data/CLEVR_CoGenT_v1.0/valB/\"\n",
    "data_dir = \"data/clevr/val/\"\n",
    "\n",
    "data_parses = [\n",
    "    f\"{data_dir}/questions-ncprogram-gt.pkl\"\n",
    "    ]\n",
    "all_parses = dict()\n",
    "for filename in data_parses:\n",
    "    if filename.endswith('.p'):\n",
    "        content = io.load_pkl(filename)\n",
    "    else:\n",
    "        content = io.load(filename)\n",
    "    all_parses.update(content)\n",
    "\n",
    "data_questions_json = osp.join(data_dir, 'questions.json')\n",
    "data_scenes_json = osp.join(data_dir, 'scenes.json')\n",
    "data_image_root = osp.join(data_dir, 'images')\n",
    "data_vocab_json = osp.join(data_dir, 'vocab.json')\n",
    "data_output_vocab_json = osp.join(data_dir, 'output-vocab.json')\n",
    "from left.domain import create_domain_from_parsing\n",
    "\n",
    "domain = create_domain_from_parsing(all_parses)\n",
    "\n",
    "\n",
    "\n",
    "from concepts.benchmark.clevr.dataset import make_dataset\n",
    "validation_dataset = make_dataset(\n",
    "    data_scenes_json,\n",
    "    data_questions_json,\n",
    "    data_image_root,\n",
    "    vocab_json=data_vocab_json,\n",
    "    output_vocab_json=data_output_vocab_json,\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1\n",
    "num_workers = 96\n",
    "validation_dataloader = validation_dataset.make_dataloader(batch_size, shuffle=False, drop_last=False, nr_workers=num_workers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "from jacinle.config.g import g\n",
    "g.concept_mapping = None\n",
    "domain.print_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jactorch.io import state_dict, load_state_dict\n",
    "from left.data.referit3d.vocabulary import Vocabulary\n",
    "from jactorch.train import TrainerEnv\n",
    "from jactorch.optim import AdamW\n",
    "from jactorch.cuda.copy import async_copy_to\n",
    "\n",
    "vocab = Vocabulary()\n",
    "model = make_model(None, domain, all_parses, validation_dataset.output_vocab if hasattr(validation_dataset, 'output_vocab') else validation_dataset.unwrapped.output_vocab, custom_transfer=None)\n",
    "model.cuda()\n",
    "trainable_parameters = filter(lambda x: x.requires_grad, model.parameters())\n",
    "\n",
    "optimizer = AdamW(trainable_parameters, 0.001 , weight_decay=1e-2)\n",
    "\n",
    "trainer = TrainerEnv(model, optimizer)\n",
    "\n",
    "filename = \"[PATH_TO_MODEL]\"\n",
    "\n",
    "trainer.load_checkpoint(filename)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "group_meters = jacinle.GroupMeters()\n",
    "group_meters.reset()\n",
    "from jacinle.utils.tqdm import tqdm_pbar\n",
    "from jactorch.utils.meta import as_float\n",
    "wrong_indices = []\n",
    "correct_indices = []\n",
    "parse_fails = []\n",
    "with tqdm_pbar(total=len(validation_dataloader)) as pbar:\n",
    "    for (index, feed_dict) in enumerate(validation_dataloader):\n",
    "        feed_dict = async_copy_to(feed_dict, 0)\n",
    "        output_dict, extra_info = trainer.evaluate(feed_dict)\n",
    "\n",
    "        if output_dict[\"results\"][0][2] is None:\n",
    "            parse_fails.append(index)\n",
    "            correct = False\n",
    "        if output_dict[\"monitors\"][\"acc/qa\"] == 1:\n",
    "            correct =  True\n",
    "        else:\n",
    "            correct = False\n",
    "\n",
    "        if not correct:\n",
    "            wrong_indices.append(index)\n",
    "        else:\n",
    "            correct_indices.append(index)\n",
    "            \n",
    "\n",
    "        acc = len(correct_indices) / (len(correct_indices) + len(wrong_indices))\n",
    "\n",
    "        pbar.set_description(group_meters.format_simple(\n",
    "            f'Validation Acc: {acc}',\n",
    "            {k: v for k, v in group_meters.val.items() if k.startswith('validation') and k.count('/') <= 2},\n",
    "            compressed=True\n",
    "        ), refresh=False)\n",
    "        pbar.update()\n",
    "        if index == 200:\n",
    "            break\n",
    "# data[\"program\"][0] = \"exists(Object, lambda x:same_row(x,iota(Object, lambda y: yellow(y))))\"\n",
    "# # print(data[\"program\"])\n",
    "# loss, monitors, outputs = model(data)\n",
    "# print(i,outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parse_fails = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(parse_fails), len(wrong_indices), len(correct_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "wrong_indices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "iterator = iter(validation_dataloader)\n",
    "data_all = [next(iterator) for _ in range(200)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in data_all:\n",
    "    if d[\"question_raw\"][0] not in all_parses:\n",
    "        print(\"not found\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import sys\n",
    "\n",
    "## Open a real image from address draw bouding boxes for objects\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "import numpy as np\n",
    "import json\n",
    "# plt.close()\n",
    "from copy import deepcopy\n",
    "from reascan import get_image\n",
    "feed_dict = data_all[wrong_indices[11]]\n",
    "feed_dict = async_copy_to(feed_dict, 0)\n",
    "print(feed_dict)\n",
    "q = feed_dict[\"question_raw\"][0]\n",
    "print(\"program:\", all_parses[q])\n",
    "\n",
    "\n",
    "output_dict, extra_info = trainer.evaluate(feed_dict)\n",
    "image = Image.open(osp.join(data_image_root, feed_dict[\"image_filename\"][0])).convert('RGB')\n",
    "print(feed_dict[\"question_raw\"])\n",
    "for i in range(len(output_dict[\"execution_traces\"][0])):\n",
    "    \n",
    "    if len(str(output_dict[\"execution_traces\"][0][i][0])) > 6:\n",
    "        tensor_values = np.array(output_dict[\"execution_traces\"][0][i][1].tensor.detach().cpu().numpy())\n",
    "        tensor_values = (tensor_values - tensor_values.min()) / (tensor_values.max() - tensor_values.min())\n",
    "        fig, ax = plt.subplots()\n",
    "        ax.imshow(image)\n",
    "        for obj_index,obj in enumerate(feed_dict[\"objects_raw\"][0]):\n",
    "            # Create a Rectangle patch\n",
    "            rect = patches.Rectangle((obj[0], obj[1]), obj[2] - obj[0], obj[3] - obj[1], linewidth=1, edgecolor='r', facecolor=\"none\")\n",
    "            ax.add_patch(rect)\n",
    "            try:\n",
    "                text = f\"{obj_index} {round(float(tensor_values[obj_index]),4)} \"\n",
    "            except:\n",
    "                text = f\"{obj_index} {tensor_values} \"\n",
    "            ax.text(obj[0], obj[1], text, color='white', fontsize=10, bbox=dict(facecolor='red', alpha=0.5))\n",
    "        plt.title(str(output_dict[\"execution_traces\"][0][i][0]))\n",
    "        plt.show()\n",
    "        print(tensor_values)\n",
    "\n",
    "plt.show()\n",
    "print(tensor_values.argmax().item())\n",
    "print(validation_dataset.unwrapped.output_vocab.idx2word[tensor_values.argmax().item()])\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}