Simple Models Results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kpkn3mPKEKCX"
   },
   "source": [
    "# Covid Chest Project\n",
    "## Simple Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G7_xE6-zEKCh"
   },
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from utils import load_covid_data, create_dataset_xray, create_dataset_ct\n",
    "from models import simple_cnn_model\n",
    "from plots import plot_cm_handy, plot_roc_handy\n",
    "from evaluation import mode_robustness, evaluation\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rvZSoeEVEKCi"
   },
   "source": [
    "## Dataset1 (X-Ray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 549
    },
    "executionInfo": {
     "elapsed": 70795,
     "status": "error",
     "timestamp": 1612858364416,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "z3QHd4uKEKCj",
    "outputId": "b0e0a232-b138-4694-f626-27b9272fed55"
   },
   "outputs": [],
   "source": [
    "x_train, y_train = load_covid_data(path='Data/train', shuffle=True, class_frequency=True)\n",
    "x_test,y_test = load_covid_data(path='Data/test')\n",
    "\n",
    "class_weights = compute_class_weight('balanced',np.unique(np.argmax(y_train,axis=1)), np.argmax(y_train,axis=1))\n",
    "class_weights = {0:class_weights[0],\n",
    "                 1:class_weights[1],\n",
    "                 2:class_weights[2]}\n",
    "BATCH_SIZE=64\n",
    "train_dataset, validation_dataset = create_dataset_xray(x_train, y_train, x_test, y_test, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset2 (CT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.load('CT_X.npy')\n",
    "Y = np.load('CT_Y.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE=256\n",
    "train_dataset, validation_dataset, X_train, X_test, y_train, y_test = create_dataset_ct(X, Y, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WIc0byuxGEn4"
   },
   "source": [
    "## Without Monte-Carlo Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WS-4sSQKIW8o"
   },
   "source": [
    "### Simple CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 7026,
     "status": "ok",
     "timestamp": 1615809861845,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhKpImYys1sQuY9zSkc3NW-sL3jIBU62aUGbslLoQ=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "rPlSXAGrIVT2"
   },
   "outputs": [],
   "source": [
    "model, callbacks=simple_cnn_model(mc=False, lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 853379,
     "status": "ok",
     "timestamp": 1607964880676,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "2_crYJJ1I5M-",
    "outputId": "e3b3280d-9679-4850-cab4-8d0abd9d6475"
   },
   "outputs": [],
   "source": [
    "hist = model.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n",
    "                          class_weight=class_weights, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fl3qTM7E_x5g"
   },
   "source": [
    "#### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11256,
     "status": "ok",
     "timestamp": 1611135065887,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "eoU0tkU0Lbnk",
    "outputId": "26a0c45a-de5a-44d4-d105-3bb2436acacc"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix,precision_score,recall_score,f1_score\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "model = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/simple_cnn_model_covid_simple.h5')\n",
    "\n",
    "preds = model.predict(x_test)\n",
    "acc = accuracy_score(np.argmax(y_test, axis=1), np.argmax(preds, axis=1))*100\n",
    "\n",
    "cm = confusion_matrix(np.argmax(y_test, axis=1)\n",
    ", np.argmax(preds, axis=1))\n",
    "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "print('CONFUSION MATRIX ------------------')\n",
    "print(cm)\n",
    "\n",
    "print('\\nTEST METRICS ----------------------')\n",
    "\n",
    "precision = precision_score(np.argmax(y_test, axis=1),\n",
    "                            np.argmax(preds, axis=1), average='weighted')*100\n",
    "recall = recall_score(np.argmax(y_test, axis=1),\n",
    "                      np.argmax(preds, axis=1), average='weighted')*100\n",
    "\n",
    "print('Accuracy: {}%'.format(acc))\n",
    "print('Precision: {}%'.format(precision))\n",
    "print('Recall: {}%'.format(recall))\n",
    "print('F1-score: {}'.format( 2*precision*recall/(precision+recall) ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 3120,
     "status": "ok",
     "timestamp": 1612627874978,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "o6TO6sghouGX",
    "outputId": "c114d070-526e-4cea-a4d0-12dc39e18e3f"
   },
   "outputs": [],
   "source": [
    "model = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/simple_cnn_model_covid_simple.h5')\n",
    "\n",
    "y_p = model.predict(x_test, batch_size=BATCH_SIZE)\n",
    "\n",
    "\n",
    "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Simple CNN model without uncertainty (X-Ray)',\n",
    "               class_name=['COVID19','Normal','Pneumonia'])\n",
    "\n",
    "plot_cm_handy(y_test, y_p,\n",
    "              lw=2, name='Confusion Matrix of Simple CNN model without uncertainty (X-Ray)',\n",
    "              class_name=['COVID19','Normal','Pneumonia'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BZCN1ZW7_7qA"
   },
   "source": [
    "#### T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1878,
     "status": "ok",
     "timestamp": 1612632381081,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "ROdgMjO3BE7S",
    "outputId": "785cf394-b056-44c8-e4e7-8713e9734702"
   },
   "outputs": [],
   "source": [
    "from models import simple_cnn_trunc_model\n",
    "model = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/simple_cnn_model_covid_simple.h5')\n",
    "trunc_model = simple_cnn_trunc_model(model, mc=False)\n",
    "hidden_features = trunc_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9143,
     "status": "ok",
     "timestamp": 1612632444898,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "xt3pJy-pB27U",
    "outputId": "43d37fc8-488e-4c2d-e7ba-d05104513eb2"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "pca = PCA(n_components=120)\n",
    "pca_result = pca.fit_transform(hidden_features)\n",
    "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n",
    "\n",
    "tsne = TSNE(n_components=2, verbose = 1)\n",
    "tsne_results = tsne.fit_transform(pca_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 917
    },
    "executionInfo": {
     "elapsed": 1365,
     "status": "ok",
     "timestamp": 1612632448301,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "rmbdCrESCCYF",
    "outputId": "b5fc4bd6-1b67-4e6b-e2b7-bd1c4e81d408"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib \n",
    "matplotlib.rc('xtick', labelsize=20) \n",
    "matplotlib.rc('ytick', labelsize=20) \n",
    "\n",
    "plt.rcParams.update({'font.size': 25})\n",
    "%matplotlib inline\n",
    "Name='T-SNE Visualization of Simple CNN model without uncertainty (X-Ray)'\n",
    "fig = plt.figure(figsize=[15, 15])\n",
    "color_map = np.argmax(y_test, axis=1)\n",
    "classes=['COVID19','Normal','Pneumonia']\n",
    "for cl in range(3):\n",
    "    indices = np.where(color_map==cl)\n",
    "    indices = indices[0]\n",
    "    plt.title(Name, fontsize=20)\n",
    "    plt.ylabel('Dim_2', fontsize=20)\n",
    "    plt.xlabel('Dim_1', fontsize=20)\n",
    "    matplotlib.rc('xtick', labelsize=20) \n",
    "    matplotlib.rc('ytick', labelsize=20) \n",
    "    plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n",
    "\n",
    "plt.rcParams.update({'font.size': 20})\n",
    "\n",
    "plt.legend()\n",
    "plt.show()\n",
    "fig.savefig('{}.pdf'.format(Name),dpi=300)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Tqr7C97x2qsZ"
   },
   "source": [
    "### Multi-headed Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 996,
     "status": "ok",
     "timestamp": 1615812562739,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhKpImYys1sQuY9zSkc3NW-sL3jIBU62aUGbslLoQ=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "sb7CG05q2vRe"
   },
   "outputs": [],
   "source": [
    "model_mh, callbacks_mh = multi_headed_model(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 655818,
     "status": "ok",
     "timestamp": 1607965960940,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "kn0c2XOTB5EA",
    "outputId": "21850153-d14e-41e9-879f-18751ef7f885"
   },
   "outputs": [],
   "source": [
    "hist_mh = model_mh.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n",
    "                          class_weight=class_weights, callbacks=callbacks_mh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wQIac5kbEY_4"
   },
   "source": [
    "#### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9313,
     "status": "ok",
     "timestamp": 1611147157527,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "S52xhcUvpTmW",
    "outputId": "c3d4cc15-f585-4df7-9305-0b116965e0f7"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix,precision_score,recall_score,f1_score\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "model_mh = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/multi_headed_model_covid_simple.h5')\n",
    "\n",
    "preds = model_mh.predict(x_test)\n",
    "acc = accuracy_score(np.argmax(y_test, axis=1), np.argmax(preds, axis=1))*100\n",
    "\n",
    "cm = confusion_matrix(np.argmax(y_test, axis=1)\n",
    ", np.argmax(preds, axis=1))\n",
    "cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "print('CONFUSION MATRIX ------------------')\n",
    "print(cm)\n",
    "\n",
    "print('\\nTEST METRICS ----------------------')\n",
    "\n",
    "precision = precision_score(np.argmax(y_test, axis=1),\n",
    "                            np.argmax(preds, axis=1), average='weighted')*100\n",
    "recall = recall_score(np.argmax(y_test, axis=1),\n",
    "                      np.argmax(preds, axis=1), average='weighted')*100\n",
    "\n",
    "print('Accuracy: {}%'.format(acc))\n",
    "print('Precision: {}%'.format(precision))\n",
    "print('Recall: {}%'.format(recall))\n",
    "print('F1-score: {}'.format( 2*precision*recall/(precision+recall) ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 3393,
     "status": "ok",
     "timestamp": 1612627905296,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "4038KiUU2J-v",
    "outputId": "1778c1a3-08ff-4d23-a212-3760d6a95239"
   },
   "outputs": [],
   "source": [
    "model_mh = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/multi_headed_model_covid_simple.h5')\n",
    "\n",
    "y_p = model_mh.predict(x_test, batch_size=BATCH_SIZE)\n",
    "\n",
    "\n",
    "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Multi-headed model without uncertainty (X-Ray)',\n",
    "               class_name=['COVID19','Normal','Pneumonia'], axis=[0.0, 0.18, 0.85, 1.0])\n",
    "\n",
    "plot_cm_handy(y_test, y_p,\n",
    "              lw=2, name='Confusion Matrix of Multi-headed model without uncertainty (X-Ray)',\n",
    "              class_name=['COVID19','Normal','Pneumonia'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "e3os807LEcwv"
   },
   "source": [
    "#### T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 7827,
     "status": "ok",
     "timestamp": 1612632482228,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "6UD3sXtaFAV0",
    "outputId": "5f3bc017-a539-465c-bd46-a4a5b892faae"
   },
   "outputs": [],
   "source": [
    "from models import multi_headed_trunc_model\n",
    "model_mh = load_model('drive/My Drive/Chest_Covid/Simple Models/X Ray Results/multi_headed_model_covid_simple.h5')\n",
    "trunc_model = multi_headed_trunc_model(model_mh, mc=False)\n",
    "hidden_features = trunc_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9492,
     "status": "ok",
     "timestamp": 1612632501164,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "xqtX8FjEFhPK",
    "outputId": "ed832807-dc79-4bb6-a815-a1a3375a6608"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "pca = PCA(n_components=120)\n",
    "pca_result = pca.fit_transform(hidden_features)\n",
    "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n",
    "\n",
    "tsne = TSNE(n_components=2, verbose = 1)\n",
    "tsne_results = tsne.fit_transform(pca_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 917
    },
    "executionInfo": {
     "elapsed": 1578,
     "status": "ok",
     "timestamp": 1612632508139,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "lNQ0sK47Flnv",
    "outputId": "79279490-5f11-492c-ae04-f95e2844993d"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib \n",
    "matplotlib.rc('xtick', labelsize=20) \n",
    "matplotlib.rc('ytick', labelsize=20) \n",
    "\n",
    "plt.rcParams.update({'font.size': 25})\n",
    "%matplotlib inline\n",
    "Name='T-SNE Visualization of Multi-headed model without uncertainty (X-Ray)'\n",
    "fig = plt.figure(figsize=[15, 15])\n",
    "color_map = np.argmax(y_test,  axis=1)\n",
    "classes=['COVID19','Normal','Pneumonia']\n",
    "for cl in range(3):\n",
    "    indices = np.where(color_map==cl)\n",
    "    indices = indices[0]\n",
    "    plt.title(Name, fontsize=20)\n",
    "    plt.ylabel('Dim_2', fontsize=20)\n",
    "    plt.xlabel('Dim_1', fontsize=20)\n",
    "    matplotlib.rc('xtick', labelsize=20) \n",
    "    matplotlib.rc('ytick', labelsize=20) \n",
    "    plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n",
    "\n",
    "plt.rcParams.update({'font.size': 20})\n",
    "\n",
    "plt.legend()\n",
    "plt.show()\n",
    "fig.savefig('{}.pdf'.format(Name),dpi=300)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9q6QoPz1PzaE"
   },
   "source": [
    "## With Monte-Carlo Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zgJdbhoMQIJx"
   },
   "source": [
    "### Simple CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5XFLZ_DEPynC"
   },
   "outputs": [],
   "source": [
    "mc_model, mc_callbacks=simple_cnn_model(mc=True, lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1134160,
     "status": "ok",
     "timestamp": 1607970919286,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "bSIlot9nQaKR",
    "outputId": "2988cfe4-55ff-4243-ae82-f5678e5ebae8"
   },
   "outputs": [],
   "source": [
    "mc_hist = mc_model.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n",
    "                          class_weight=class_weights, callbacks=mc_callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tvxtdGOYHnTC"
   },
   "source": [
    "#### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 80458,
     "status": "ok",
     "timestamp": 1613650889182,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "KvHuWn993fOV",
    "outputId": "7e4f8f77-5384-4f57-c851-fa54914cc076"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score,recall_score,precision_score\n",
    "import tqdm\n",
    "\n",
    "mc_model = load_model('path/simple_cnn_model_covid_mc.h5')\n",
    "\n",
    "number_prediction=200\n",
    "mc_predictions = []\n",
    "for i in tqdm.tqdm(range(number_prediction)):\n",
    "    y_p = mc_model.predict(x_test)\n",
    "    mc_predictions.append(y_p)\n",
    "\n",
    "accs=recalls=precisions=F1s=[]\n",
    "for y_p in mc_predictions:\n",
    "    acc = accuracy_score(y_test.argmax(axis=1), y_p.argmax(axis=1))\n",
    "    recall=recall_score(y_test.argmax(axis=1), y_p.argmax(axis=1),average='weighted')\n",
    "    precision=precision_score(y_test.argmax(axis=1), y_p.argmax(axis=1),average='weighted')\n",
    "    F1=(2*precision*recall)/(precision+recall)\n",
    "    accs.append(acc)\n",
    "    recalls.append(recall)\n",
    "    precisions.append(precision)\n",
    "    F1s.append(F1)\n",
    "\n",
    "\n",
    "print(\"MC accuracy: {:.5%}\".format(sum(accs)/len(accs)))\n",
    "print(\"MC precision: {:.5%}\".format(sum(precisions)/len(precisions)))\n",
    "print(\"MC recall: {:.5%}\".format(sum(recalls)/len(recalls)))\n",
    "print(\"MC F1: {:.5%}\".format(sum(F1s)/len(F1s)))\n",
    "\n",
    "mc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1)\n",
    "ensemble_acc = accuracy_score(y_test.argmax(axis=1), mc_ensemble_pred)\n",
    "ensemble_precision=precision_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n",
    "ensemble_recall=recall_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n",
    "ensemble_F1=(2*ensemble_precision*ensemble_recall)/(ensemble_precision+ensemble_recall)\n",
    "\n",
    "print(\"MC-ensemble accuracy: {:.5%}\".format(ensemble_acc))\n",
    "print(\"MC-ensemble precision: {:.5%}\".format(ensemble_precision))\n",
    "print(\"MC-ensemble recall: {:.5%}\".format(ensemble_recall))\n",
    "print(\"MC-ensemble F1: {:.5%}\".format(ensemble_F1))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 449
    },
    "executionInfo": {
     "elapsed": 1137,
     "status": "ok",
     "timestamp": 1612215394628,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "zINmKa3OheBj",
    "outputId": "bb0c18e8-3b5c-4984-9da1-e5f0996bd3d9"
   },
   "outputs": [],
   "source": [
    "Name = 'Histogram of Simple CNN model with uncertainty (X-Ray)'\n",
    "f, ax = plt.subplots(figsize=[10, 7])\n",
    "plt.hist(accs)\n",
    "plt.axvline(x=ensemble_acc, color=\"r\")\n",
    "ax.set_title(Name,fontsize=19)\n",
    "plt.show()\n",
    "f.savefig('{}.pdf'.format(Name))\n",
    "ax.figure.savefig(\"{}.pdf\".format(Name), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 2445,
     "status": "ok",
     "timestamp": 1612628062671,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "thp5jb-p3kIa",
    "outputId": "d65da74f-f6a8-43a0-951e-7d372792c0d5"
   },
   "outputs": [],
   "source": [
    "plot_roc_handy(y_test, y_p, zoom=True, lw=2, name='Roc of Simple CNN model with uncertainty (X-Ray)',\n",
    "               class_name=['COVID19','Normal','Pneumonia'], axis=[0.0, 0.21, 0.85, 1.0])\n",
    "\n",
    "plot_cm_handy(y_test, y_p,\n",
    "              lw=2, name='Confusion Matrix of Simple CNN model with uncertainty (X-Ray)',\n",
    "              class_name=['COVID19','Normal','Pneumonia'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Bm0SUpOJHs32"
   },
   "source": [
    "#### Noise Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4463,
     "status": "ok",
     "timestamp": 1612105203067,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "o76otCkJ88v3",
    "outputId": "6948a184-f852-48bc-a7c1-9ed8fb12c169"
   },
   "outputs": [],
   "source": [
    "mc_model=load_model('path/simple_cnn_model_covid_mc.h5')\n",
    "std_coef=[1e-4,1e-3,1e-2,1e-1, 0.2,0.3,0.4,0.5,0.6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 163152,
     "status": "ok",
     "timestamp": 1612105390317,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "Vx2E8JoI4IuT",
    "outputId": "3208144e-4b3f-436c-ef2f-37530bb991ac"
   },
   "outputs": [],
   "source": [
    "mode_robustness(x_test, y_test, mc_model, std_coef)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6LtcdA_2HxJi"
   },
   "source": [
    "#### T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5745,
     "status": "ok",
     "timestamp": 1612632524315,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "4MgSCxJ6H0H3",
    "outputId": "80611d75-e482-44dc-eebe-31713d214507"
   },
   "outputs": [],
   "source": [
    "mc_model = load_model('path/simple_cnn_model_covid_mc.h5')\n",
    "trunc_model = simple_cnn_trunc_model(mc_model, mc=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kPKFW4cSJFmN"
   },
   "outputs": [],
   "source": [
    "hidden_features=[]\n",
    "for i in range(200):\n",
    "  hidden_features.append(trunc_model.predict(x_test))\n",
    "\n",
    "hidden_features=np.array(hidden_features).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8019,
     "status": "ok",
     "timestamp": 1612632613186,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "pWFzWCHZIzq0",
    "outputId": "ab05f0ed-2ffc-4cae-8551-3f5ccb3903bd"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "pca = PCA(n_components=120)\n",
    "pca_result = pca.fit_transform(hidden_features)\n",
    "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n",
    "\n",
    "tsne = TSNE(n_components=2, verbose = 1)\n",
    "tsne_results = tsne.fit_transform(pca_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 917
    },
    "executionInfo": {
     "elapsed": 1774,
     "status": "ok",
     "timestamp": 1612632620393,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "5IJq0K-hIzdP",
    "outputId": "4de066da-43a5-47e9-f400-523b6132bf72"
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib \n",
    "matplotlib.rc('xtick', labelsize=20) \n",
    "matplotlib.rc('ytick', labelsize=20) \n",
    "\n",
    "plt.rcParams.update({'font.size': 25})\n",
    "%matplotlib inline\n",
    "Name='T-SNE Visualization of Simple CNN model with uncertainty (X-Ray)'\n",
    "fig = plt.figure(figsize=[15, 15])\n",
    "color_map = np.argmax(y_test, axis=1)\n",
    "classes=['COVID19','Normal','Pneumonia']\n",
    "for cl in range(3):\n",
    "    indices = np.where(color_map==cl)\n",
    "    indices = indices[0]\n",
    "    plt.title(Name, fontsize=20)\n",
    "    plt.ylabel('Dim_2', fontsize=20)\n",
    "    plt.xlabel('Dim_1', fontsize=20)\n",
    "    matplotlib.rc('xtick', labelsize=20) \n",
    "    matplotlib.rc('ytick', labelsize=20) \n",
    "    plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n",
    "\n",
    "plt.rcParams.update({'font.size': 20})\n",
    "\n",
    "plt.legend()\n",
    "plt.show()\n",
    "fig.savefig('{}.pdf'.format(Name),dpi=300)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ctVi8yViTTe0"
   },
   "source": [
    "### Multi-headed Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4k6hlLSyTUO2"
   },
   "outputs": [],
   "source": [
    "mc_model_mh, mc_callbacks_mh = multi_headed_model(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1472938,
     "status": "ok",
     "timestamp": 1607975975007,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "WgHcK7D1Terz",
    "outputId": "7348ff07-6a2d-4415-c45f-c1132bdb6403"
   },
   "outputs": [],
   "source": [
    "mc_hist_mh = mc_model_mh.fit(train_dataset, epochs=200, validation_data=validation_dataset,\n",
    "                          class_weight=class_weights, callbacks=mc_callbacks_mh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IoctPxwUIIx5"
   },
   "source": [
    "#### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 113061,
     "status": "ok",
     "timestamp": 1612628193425,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "BQabg4To-SUP",
    "outputId": "7c5c5af5-c294-4cc6-8580-275dfe4c3f65"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score,recall_score,precision_score\n",
    "import tqdm\n",
    "\n",
    "mc_model_mh = load_model('path/multi_headed_model_covid_mc.h5')\n",
    "\n",
    "number_prediction=200\n",
    "mc_predictions = []\n",
    "for i in tqdm.tqdm(range(number_prediction)):\n",
    "    y_p = mc_model_mh.predict(x_test)\n",
    "    mc_predictions.append(y_p)\n",
    "\n",
    "accs=recalls=precisions=F1s=[]\n",
    "for y_p in mc_predictions:\n",
    "    acc = accuracy_score(y_test.argmax(axis=1), y_p.argmax(axis=1))\n",
    "    recall=recall_score(y_test.argmax(axis=1), y_p.argmax(axis=1), average='weighted')\n",
    "    precision=precision_score(y_test.argmax(axis=1), y_p.argmax(axis=1), average='weighted')\n",
    "    F1=(2*precision*recall)/(precision+recall)\n",
    "    accs.append(acc)\n",
    "    recalls.append(recall)\n",
    "    precisions.append(precision)\n",
    "    F1s.append(F1)\n",
    "\n",
    "\n",
    "print(\"MC accuracy: {:.5%}\".format(sum(accs)/len(accs)))\n",
    "print(\"MC precision: {:.5%}\".format(sum(precisions)/len(precisions)))\n",
    "print(\"MC recall: {:.5%}\".format(sum(recalls)/len(recalls)))\n",
    "print(\"MC F1: {:.5%}\".format(sum(F1s)/len(F1s)))\n",
    "\n",
    "mc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1)\n",
    "ensemble_acc = accuracy_score(y_test.argmax(axis=1), mc_ensemble_pred)\n",
    "ensemble_precision=precision_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n",
    "ensemble_recall=recall_score(y_test.argmax(axis=1), mc_ensemble_pred, average='weighted')\n",
    "ensemble_F1=(2*ensemble_precision*ensemble_recall)/(ensemble_precision+ensemble_recall)\n",
    "\n",
    "print(\"MC-ensemble accuracy: {:.5%}\".format(ensemble_acc))\n",
    "print(\"MC-ensemble precision: {:.5%}\".format(ensemble_precision))\n",
    "print(\"MC-ensemble recall: {:.5%}\".format(ensemble_recall))\n",
    "print(\"MC-ensemble F1: {:.5%}\".format(ensemble_F1))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 449
    },
    "executionInfo": {
     "elapsed": 1122,
     "status": "ok",
     "timestamp": 1612215642550,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "nMPCIyVihzl-",
    "outputId": "233d492c-c0b3-440e-a18d-8d5995ee1453"
   },
   "outputs": [],
   "source": [
    "Name = 'Histogram of Multi-headed model with uncertainty (X-Ray)'\n",
    "f, ax = plt.subplots(figsize=[10, 7])\n",
    "plt.hist(accs);\n",
    "plt.axvline(x=ensemble_acc, color=\"r\")\n",
    "ax.set_title(Name,fontsize=19)\n",
    "plt.show()\n",
    "f.savefig('{}.pdf'.format(Name))\n",
    "ax.figure.savefig(\"{}.pdf\".format(Name), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 1903,
     "status": "ok",
     "timestamp": 1612628220225,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "RbPuXZqX5IrZ",
    "outputId": "a3d59402-a499-4c4d-a28e-3c076ed3ea68"
   },
   "outputs": [],
   "source": [
    "plot_roc_handy(y_test, np.array(mc_predictions).mean(axis=0) ,zoom=True, lw=2, name='Roc of Multi-headed model with uncertainty (X-Ray)',\n",
    "               class_name=['COVID19','Normal','Pneumonia'], axis=[0.0, 0.21, 0.85, 1.0])\n",
    "\n",
    "plot_cm_handy(y_test, np.array(mc_predictions).mean(axis=0),\n",
    "              lw=2, name='Confusion Matrix of Multi-headed model with uncertainty (X-Ray)',\n",
    "              class_name=['COVID19','Normal','Pneumonia'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JDutmYYmIPrU"
   },
   "source": [
    "#### Noise Robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9924,
     "status": "ok",
     "timestamp": 1612107340331,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "AnWZdMkr_zHx",
    "outputId": "673e2167-d9bc-4d72-9324-baeb4d276efd"
   },
   "outputs": [],
   "source": [
    "mc_model_mh = load_model('path/multi_headed_model_covid_mc.h5')\n",
    "std_coef=[1e-4,1e-3,1e-2,1e-1, 0.2,0.3,0.4,0.5,0.6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 246515,
     "status": "ok",
     "timestamp": 1612107592091,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "gGwv-LEEFVqk",
    "outputId": "74a6ffac-e552-4fd7-bc45-e72b39c779f7"
   },
   "outputs": [],
   "source": [
    "mode_robustness(x_test, y_test, mc_model_mh, std_coef)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jq8OYXpaIUch"
   },
   "source": [
    "#### T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1709,
     "status": "ok",
     "timestamp": 1612632666788,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "wjPTphhEKQcu",
    "outputId": "5db610ee-c78e-40e4-983a-f561e077e5c4"
   },
   "outputs": [],
   "source": [
    "mc_model_mh = load_model('path/multi_headed_model_covid_mc.h5')\n",
    "trunc_model = multi_headed_trunc_model(mc_model_mh, mc=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SXhCOKG2KU3T"
   },
   "outputs": [],
   "source": [
    "hidden_features=[]\n",
    "for i in range(200):\n",
    "  hidden_features.append(trunc_model.predict(x_test))\n",
    "\n",
    "hidden_features=np.array(hidden_features).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8529,
     "status": "ok",
     "timestamp": 1612632796935,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "uEGkXjZ3KUnJ",
    "outputId": "ae9cc5f4-10fd-422f-e520-a0228b7fc38b"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "pca = PCA(n_components=120)\n",
    "pca_result = pca.fit_transform(hidden_features)\n",
    "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n",
    "\n",
    "tsne = TSNE(n_components=2, verbose = 1)\n",
    "tsne_results = tsne.fit_transform(pca_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 917
    },
    "executionInfo": {
     "elapsed": 1745,
     "status": "ok",
     "timestamp": 1612632803522,
     "user": {
      "displayName": "Sina Qahremani",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh-ZisuRC_RaGfXurPEq8lcklkHXJNZRrfk3xd4=s64",
      "userId": "15549406949129485910"
     },
     "user_tz": -210
    },
    "id": "i1a-kstDKUTq",
    "outputId": "a63403e0-a38d-4986-9e09-643f7fc75c69"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib \n",
    "matplotlib.rc('xtick', labelsize=20) \n",
    "matplotlib.rc('ytick', labelsize=20) \n",
    "\n",
    "plt.rcParams.update({'font.size': 25})\n",
    "%matplotlib inline\n",
    "Name='T-SNE Visualization of Multi-headed model with uncertainty (X-Ray)'\n",
    "fig = plt.figure(figsize=[15, 15])\n",
    "color_map = np.argmax(y_test, axis=1)\n",
    "classes=['COVID19','Normal','Pneumonia']\n",
    "for cl in range(3):\n",
    "    indices = np.where(color_map==cl)\n",
    "    indices = indices[0]\n",
    "    plt.title(Name,fontsize=20)\n",
    "    plt.ylabel('Dim_2',fontsize=20)\n",
    "    plt.xlabel('Dim_1',fontsize=20)\n",
    "    matplotlib.rc('xtick', labelsize=20) \n",
    "    matplotlib.rc('ytick', labelsize=20) \n",
    "    plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=classes[cl])\n",
    "\n",
    "plt.rcParams.update({'font.size': 20})\n",
    "\n",
    "plt.legend()\n",
    "plt.show()\n",
    "fig.savefig('{}.pdf'.format(Name),dpi=300)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "xray_simple_models.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}