how_to_combine_WS_labels_and_AL_labels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_samples</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_informative</th>\n",
       "      <th>n_redundant</th>\n",
       "      <th>n_repeated</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>n_clusters_per_class</th>\n",
       "      <th>weights</th>\n",
       "      <th>flip_y</th>\n",
       "      <th>class_sep</th>\n",
       "      <th>...</th>\n",
       "      <th>JOB_ID</th>\n",
       "      <th>AL_SAMPLES_WEIGHT</th>\n",
       "      <th>AMOUNT_OF_LFS</th>\n",
       "      <th>CLUSTERED_AL_WS_COMBINATION</th>\n",
       "      <th>DATASET</th>\n",
       "      <th>DATASET_RANDOM_GENERATION_SEED</th>\n",
       "      <th>FRACTION_OF_INITIALLY_LABELLED_SAMPLES</th>\n",
       "      <th>FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES</th>\n",
       "      <th>LF_RANDOM_SEED</th>\n",
       "      <th>MERGE_WS_SAMPLES_STRATEGY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>178</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>21</td>\n",
       "      <td>62</td>\n",
       "      <td>1.667661</td>\n",
       "      <td>True</td>\n",
       "      <td>wine</td>\n",
       "      <td>914092</td>\n",
       "      <td>0.244126</td>\n",
       "      <td>0.168291</td>\n",
       "      <td>61088</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>62</td>\n",
       "      <td>3.462121</td>\n",
       "      <td>False</td>\n",
       "      <td>FERTILITY</td>\n",
       "      <td>408924</td>\n",
       "      <td>0.356298</td>\n",
       "      <td>0.906828</td>\n",
       "      <td>685441</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>351</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>27</td>\n",
       "      <td>2.673670</td>\n",
       "      <td>False</td>\n",
       "      <td>IONOSPHERE</td>\n",
       "      <td>579305</td>\n",
       "      <td>0.304781</td>\n",
       "      <td>0.164656</td>\n",
       "      <td>660891</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100</td>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>47</td>\n",
       "      <td>4.153230</td>\n",
       "      <td>False</td>\n",
       "      <td>FERTILITY</td>\n",
       "      <td>328948</td>\n",
       "      <td>0.592415</td>\n",
       "      <td>0.046450</td>\n",
       "      <td>528179</td>\n",
       "      <td>RandomLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>306</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>1.001795</td>\n",
       "      <td>False</td>\n",
       "      <td>HABERMAN</td>\n",
       "      <td>214177</td>\n",
       "      <td>0.304242</td>\n",
       "      <td>0.524756</td>\n",
       "      <td>258796</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8475</th>\n",
       "      <td>48842</td>\n",
       "      <td>105</td>\n",
       "      <td>105</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>9918</td>\n",
       "      <td>53</td>\n",
       "      <td>4.143662</td>\n",
       "      <td>True</td>\n",
       "      <td>adult</td>\n",
       "      <td>100406</td>\n",
       "      <td>0.406360</td>\n",
       "      <td>0.073848</td>\n",
       "      <td>245517</td>\n",
       "      <td>RandomLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8476</th>\n",
       "      <td>10000</td>\n",
       "      <td>3073</td>\n",
       "      <td>3073</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>9610</td>\n",
       "      <td>47</td>\n",
       "      <td>7.759934</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>82165</td>\n",
       "      <td>0.294491</td>\n",
       "      <td>0.644126</td>\n",
       "      <td>66889</td>\n",
       "      <td>SnorkelLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8477</th>\n",
       "      <td>48842</td>\n",
       "      <td>105</td>\n",
       "      <td>105</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>9848</td>\n",
       "      <td>14</td>\n",
       "      <td>4.310431</td>\n",
       "      <td>True</td>\n",
       "      <td>adult</td>\n",
       "      <td>257669</td>\n",
       "      <td>0.310385</td>\n",
       "      <td>0.922679</td>\n",
       "      <td>212332</td>\n",
       "      <td>SnorkelLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8478</th>\n",
       "      <td>10000</td>\n",
       "      <td>3073</td>\n",
       "      <td>3073</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>9732</td>\n",
       "      <td>66</td>\n",
       "      <td>7.084622</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>242013</td>\n",
       "      <td>0.286538</td>\n",
       "      <td>0.003731</td>\n",
       "      <td>784560</td>\n",
       "      <td>RandomLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8479</th>\n",
       "      <td>10000</td>\n",
       "      <td>3073</td>\n",
       "      <td>3073</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>9783</td>\n",
       "      <td>33</td>\n",
       "      <td>7.216152</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>484338</td>\n",
       "      <td>0.094111</td>\n",
       "      <td>0.399906</td>\n",
       "      <td>501539</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8480 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      n_samples  n_features  n_informative  n_redundant  n_repeated  \\\n",
       "0           178          14             14            0           0   \n",
       "1           100          21             21            0           0   \n",
       "2           351          35             35            0           0   \n",
       "3           100          21             21            0           0   \n",
       "4           306           4              4            0           0   \n",
       "...         ...         ...            ...          ...         ...   \n",
       "8475      48842         105            105            0           0   \n",
       "8476      10000        3073           3073            0           0   \n",
       "8477      48842         105            105            0           0   \n",
       "8478      10000        3073           3073            0           0   \n",
       "8479      10000        3073           3073            0           0   \n",
       "\n",
       "      n_classes n_clusters_per_class weights flip_y class_sep  ... JOB_ID  \\\n",
       "0             3                    ?       ?      ?         ?  ...     21   \n",
       "1             2                    ?       ?      ?         ?  ...     19   \n",
       "2             2                    ?       ?      ?         ?  ...     20   \n",
       "3             2                    ?       ?      ?         ?  ...      4   \n",
       "4             2                    ?       ?      ?         ?  ...      2   \n",
       "...         ...                  ...     ...    ...       ...  ...    ...   \n",
       "8475          4                    ?       ?      ?         ?  ...   9918   \n",
       "8476         10                    ?       ?      ?         ?  ...   9610   \n",
       "8477          4                    ?       ?      ?         ?  ...   9848   \n",
       "8478         10                    ?       ?      ?         ?  ...   9732   \n",
       "8479         10                    ?       ?      ?         ?  ...   9783   \n",
       "\n",
       "     AL_SAMPLES_WEIGHT  AMOUNT_OF_LFS CLUSTERED_AL_WS_COMBINATION     DATASET  \\\n",
       "0                   62       1.667661                        True        wine   \n",
       "1                   62       3.462121                       False   FERTILITY   \n",
       "2                   27       2.673670                       False  IONOSPHERE   \n",
       "3                   47       4.153230                       False   FERTILITY   \n",
       "4                   38       1.001795                       False    HABERMAN   \n",
       "...                ...            ...                         ...         ...   \n",
       "8475                53       4.143662                        True       adult   \n",
       "8476                47       7.759934                        True     cifar10   \n",
       "8477                14       4.310431                        True       adult   \n",
       "8478                66       7.084622                        True     cifar10   \n",
       "8479                33       7.216152                        True     cifar10   \n",
       "\n",
       "     DATASET_RANDOM_GENERATION_SEED FRACTION_OF_INITIALLY_LABELLED_SAMPLES  \\\n",
       "0                            914092                               0.244126   \n",
       "1                            408924                               0.356298   \n",
       "2                            579305                               0.304781   \n",
       "3                            328948                               0.592415   \n",
       "4                            214177                               0.304242   \n",
       "...                             ...                                    ...   \n",
       "8475                         100406                               0.406360   \n",
       "8476                          82165                               0.294491   \n",
       "8477                         257669                               0.310385   \n",
       "8478                         242013                               0.286538   \n",
       "8479                         484338                               0.094111   \n",
       "\n",
       "     FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES  LF_RANDOM_SEED  \\\n",
       "0                                  0.168291           61088   \n",
       "1                                  0.906828          685441   \n",
       "2                                  0.164656          660891   \n",
       "3                                  0.046450          528179   \n",
       "4                                  0.524756          258796   \n",
       "...                                     ...             ...   \n",
       "8475                               0.073848          245517   \n",
       "8476                               0.644126           66889   \n",
       "8477                               0.922679          212332   \n",
       "8478                               0.003731          784560   \n",
       "8479                               0.399906          501539   \n",
       "\n",
       "           MERGE_WS_SAMPLES_STRATEGY  \n",
       "0     MajorityVoteLabelMergeStrategy  \n",
       "1     MajorityVoteLabelMergeStrategy  \n",
       "2     MajorityVoteLabelMergeStrategy  \n",
       "3           RandomLabelMergeStrategy  \n",
       "4     MajorityVoteLabelMergeStrategy  \n",
       "...                              ...  \n",
       "8475        RandomLabelMergeStrategy  \n",
       "8476       SnorkelLabelMergeStrategy  \n",
       "8477       SnorkelLabelMergeStrategy  \n",
       "8478        RandomLabelMergeStrategy  \n",
       "8479  MajorityVoteLabelMergeStrategy  \n",
       "\n",
       "[8480 rows x 59 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from active_learning.learner.standard import Learner, get_classifier\n",
    "from active_learning.weak_supervision.SelfTraining import SelfTraining\n",
    "import argparse\n",
    "import numpy as np\n",
    "import copy\n",
    "import pandas as pd\n",
    "import math\n",
    "import random\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "from timeit import default_timer as timer\n",
    "from typing import List, Dict\n",
    "from active_learning.config import get_active_config\n",
    "from active_learning.dataStorage import DataStorage\n",
    "from active_learning.datasets import load_synthetic\n",
    "from active_learning.logger import init_logger\n",
    "from active_learning.merge_weak_supervision_label_strategies.MajorityVoteLabelMergeStrategy import (\n",
    "    MajorityVoteLabelMergeStrategy,\n",
    ")\n",
    "from collections import Counter\n",
    "\n",
    "from active_learning.weak_supervision import SyntheticLabelingFunctions\n",
    "from active_learning.weak_supervision.BaseWeakSupervision import BaseWeakSupervision\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "#sns.set_theme(style=\"whitegrid\")\n",
    "\n",
    "\n",
    "font_size = 8\n",
    "\n",
    "tex_fonts = {\n",
    "    # Use LaTeX to write all text\n",
    "    # \"text.usetex\": True,\n",
    "    \"text.usetex\": False,\n",
    "    \"font.family\": [\"Times New Roman\", 'sans-serif'],\n",
    "    # Use 10pt font in plots, to match 10pt font in document\n",
    "    \"axes.labelsize\": font_size,\n",
    "    \"font.size\": font_size,\n",
    "    # Make the legend/label fonts a little smaller\n",
    "    \"legend.fontsize\": font_size,\n",
    "    \"xtick.labelsize\": font_size,\n",
    "    \"ytick.labelsize\": font_size,\n",
    "    \"xtick.bottom\": True,\n",
    "    \"figure.autolayout\": True,\n",
    "}\n",
    "\n",
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_context(\"paper\")\n",
    "plt.rcParams.update(tex_fonts)  # type: ignore\n",
    "\n",
    "\n",
    "# https://jwalton.info/Embed-Publication-Matplotlib-Latex/\n",
    "def set_matplotlib_size(width, fraction=1):\n",
    "    \"\"\"Set figure dimensions to avoid scaling in LaTeX.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    width: float\n",
    "            Document textwidth or columnwidth in pts\n",
    "    fraction: float, optional\n",
    "            Fraction of the width which you wish the figure to occupy\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    fig_dim: tuple\n",
    "            Dimensions of figure in inches\n",
    "    \"\"\"\n",
    "    # Width of figure (in pts)\n",
    "    fig_width_pt = width * fraction\n",
    "\n",
    "    # Convert from pt to inches\n",
    "    inches_per_pt = 1 / 72.27\n",
    "\n",
    "    # Golden ratio to set aesthetic figure height\n",
    "    # https://disq.us/p/2940ij3\n",
    "    golden_ratio = (5 ** 0.5 - 1) / 2\n",
    "\n",
    "    # Figure width in inches\n",
    "    fig_width_in = fig_width_pt * inches_per_pt\n",
    "    # Figure height in inches\n",
    "    fig_height_in = fig_width_in * golden_ratio\n",
    "\n",
    "    fig_dim = (fig_width_in, fig_height_in)\n",
    "\n",
    "    return fig_dim\n",
    "\n",
    "\n",
    "#width = 505.89\n",
    "width = 1500\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"~/exp_results/run_how_to/exp_results.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['n_samples', 'n_features', 'n_informative', 'n_redundant', 'n_repeated',\n",
       "       'n_classes', 'n_clusters_per_class', 'weights', 'flip_y', 'class_sep',\n",
       "       'hypercube', 'scale', 'random_state', 'ABSTAIN_THRESHOLDS',\n",
       "       'LF_CLASSIFIERS', 'AMOUNT_OF_LF_FEATURESSS', 'acc_WS', 'f1_WS',\n",
       "       'f1_initial', 'acc_initial', 'f1_ws', 'acc_ws',\n",
       "       'acc_ws_and_al_UncertaintyMaxMargin_no_ws',\n",
       "       'f1_ws_and_al_UncertaintyMaxMargin_no_ws',\n",
       "       'acc_al_and_al_UncertaintyMaxMargin_no_ws',\n",
       "       'f1_al_and_al_UncertaintyMaxMargin_no_ws',\n",
       "       'acc_ws_and_al_UncertaintyMaxMargin_with_ws',\n",
       "       'f1_ws_and_al_UncertaintyMaxMargin_with_ws',\n",
       "       'acc_al_and_al_UncertaintyMaxMargin_with_ws',\n",
       "       'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random',\n",
       "       'f1_ws_and_al_Random', 'acc_al_and_al_Random', 'f1_al_and_al_Random',\n",
       "       'acc_ws_and_al_CoveredByLeastAmountOfLf',\n",
       "       'f1_ws_and_al_CoveredByLeastAmountOfLf',\n",
       "       'acc_al_and_al_CoveredByLeastAmountOfLf',\n",
       "       'f1_al_and_al_CoveredByLeastAmountOfLf',\n",
       "       'acc_ws_and_al_ClassificationIsMostWrong',\n",
       "       'f1_ws_and_al_ClassificationIsMostWrong',\n",
       "       'acc_al_and_al_ClassificationIsMostWrong',\n",
       "       'f1_al_and_al_ClassificationIsMostWrong',\n",
       "       'acc_ws_and_al_GreatestDisagreement',\n",
       "       'f1_ws_and_al_GreatestDisagreement',\n",
       "       'acc_al_and_al_GreatestDisagreement',\n",
       "       'f1_al_and_al_GreatestDisagreement', 'amount_of_initial_al_samples',\n",
       "       'amount_of_lastly_al_samples', 'amount_of_ws_labelled_samples',\n",
       "       'JOB_ID', 'AL_SAMPLES_WEIGHT', 'AMOUNT_OF_LFS',\n",
       "       'CLUSTERED_AL_WS_COMBINATION', 'DATASET',\n",
       "       'DATASET_RANDOM_GENERATION_SEED',\n",
       "       'FRACTION_OF_INITIALLY_LABELLED_SAMPLES',\n",
       "       'FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES', 'LF_RANDOM_SEED',\n",
       "       'MERGE_WS_SAMPLES_STRATEGY'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.keys()#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "General Improvement after AL + WS acc\n",
      "                                                               Mean     \n",
      "acc_initial                                        61.69% ±0.53%\n",
      "acc_ws                                             61.80% ±0.52%\n",
      "acc_ws_and_al_UncertaintyMaxMargin_no_ws           61.91% ±0.54%\n",
      "acc_al_and_al_UncertaintyMaxMargin_no_ws           62.12% ±0.55%\n",
      "acc_ws_and_al_UncertaintyMaxMargin_with_ws         61.91% ±0.54%\n",
      "acc_al_and_al_UncertaintyMaxMargin_with_ws         62.12% ±0.55%\n",
      "acc_ws_and_al_Random                               61.84% ±0.53%\n",
      "acc_al_and_al_Random                               61.94% ±0.55%\n",
      "acc_ws_and_al_CoveredByLeastAmountOfLf             61.80% ±0.53%\n",
      "acc_al_and_al_CoveredByLeastAmountOfLf             62.02% ±0.54%\n",
      "acc_ws_and_al_ClassificationIsMostWrong            61.79% ±0.52%\n",
      "acc_al_and_al_ClassificationIsMostWrong            62.02% ±0.53%\n",
      "acc_ws_and_al_GreatestDisagreement                 61.68% ±0.53%\n",
      "acc_al_and_al_GreatestDisagreement                 61.96% ±0.54%\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong']\n",
      "Where is al+ws+al better than al+al? []\n",
      "General Improvement after AL + WS f1\n",
      "                                                               Mean     \n",
      "f1_initial                                         59.27% ±0.54%\n",
      "f1_ws                                              58.04% ±0.54%\n",
      "f1_ws_and_al_UncertaintyMaxMargin_no_ws            59.53% ±0.55%\n",
      "f1_al_and_al_UncertaintyMaxMargin_no_ws            60.27% ±0.55%\n",
      "f1_ws_and_al_UncertaintyMaxMargin_with_ws          59.53% ±0.55%\n",
      "f1_al_and_al_UncertaintyMaxMargin_with_ws          60.27% ±0.55%\n",
      "f1_ws_and_al_Random                                59.41% ±0.54%\n",
      "f1_al_and_al_Random                                60.02% ±0.55%\n",
      "f1_ws_and_al_CoveredByLeastAmountOfLf              59.25% ±0.54%\n",
      "f1_al_and_al_CoveredByLeastAmountOfLf              59.86% ±0.54%\n",
      "f1_ws_and_al_ClassificationIsMostWrong             58.74% ±0.54%\n",
      "f1_al_and_al_ClassificationIsMostWrong             59.19% ±0.54%\n",
      "f1_ws_and_al_GreatestDisagreement                  59.10% ±0.54%\n",
      "f1_al_and_al_GreatestDisagreement                  59.75% ±0.54%\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random']\n",
      "Where is al+ws+al better than al+al? []\n"
     ]
    }
   ],
   "source": [
    "def hist_plot(df: pd.DataFrame, COMPARE_KEYS:List[str], TITLE:str=None, OUTPUT_PATH: str=None, LABEL:str=\"AUC-F1 (\\\\%)\"):\n",
    "    fig = plt.figure(figsize=set_matplotlib_size(width, fraction=0.5))\n",
    "\n",
    "    # fig = plt.gcf()  # type: ignore\n",
    "    # fig.set_size_inches(18.5, 10.5)\n",
    "\n",
    "    selection_list = []\n",
    "    for key in COMPARE_KEYS:\n",
    "        selection_list.append((df[key], key))\n",
    "    #for strategy in df[\"strategy\"].unique():\n",
    "    #    selection_list.append((df[df[\"strategy\"] == strategy][\"f1_auc\"], strategy))\n",
    "\n",
    "    for selection, label in sorted(selection_list, key=lambda tup: tup[1]):\n",
    "        #  ax = sns.distplot(selection, label=label, **kwargs)\n",
    "        #  label = \"{}-{:>3}: {:.4g}% \".format(label[0], label[1], selection.mean() * 100)\n",
    "        label = \"{:<10}{:>5.4g}% \".format(label + \":\", selection.mean() * 100)\n",
    "        ax = sns.kdeplot(selection, label=label)\n",
    "        # ax = sns.distplot(selection, label=label, kde=True)\n",
    "        #ax.set_xlim(0, 1)\n",
    "        ax.set_xlim(0.5, 0.6)\n",
    "        #  ax.set_xlim(0.75, 0.8)\n",
    "        #  ax.set_xlim(80, 875)\n",
    "\n",
    "        mean = selection.mean()\n",
    "        if selection.count() == 0:\n",
    "            low = high = mean\n",
    "        else:\n",
    "            low = selection.mean() - 1.96 * selection.std() / math.sqrt(\n",
    "                selection.count()\n",
    "            )\n",
    "            high = selection.mean() + 1.96 * selection.std() / math.sqrt(\n",
    "                selection.count()\n",
    "            )\n",
    "        print(label)\n",
    "        ax.axvline(mean, color=plt.gca().lines[-1].get_color())  # type: ignore\n",
    "        ax.axvspan(low, high, alpha=0.2, color=plt.gca().lines[-1].get_color())  # type: ignore\n",
    "\n",
    "        ax.set_xticks([x for x in ax.get_xticks()]) # stupid \"workaround\" for https://github.com/matplotlib/matplotlib/issues/18848\n",
    "        ax.set_xticklabels([\"{:.0%}\\\\%\".format(x) for x in ax.get_xticks()])\n",
    "    ax.set_title(TITLE)  # type: ignore\n",
    "    # ax.set_ylabel(\"\")\n",
    "    ax.set_xlabel(LABEL)  # type: ignore\n",
    "    ax.legend(  # type: ignore\n",
    "        loc=\"center left\"\n",
    "    )  # , bbox_to_anchor=(0.0, 1.01), borderaxespad=0, frameon=False, ncol=len(selection))  # type: ignore\n",
    "    # ax.xa\n",
    "\n",
    "def comparison_table(df: pd.DataFrame, COMPARE_KEYS:List[str], TITLE:str=\"\"):\n",
    "    results = []\n",
    "    for key in COMPARE_KEYS:\n",
    "        selection = df[key]\n",
    "\n",
    "        mean = selection.mean()\n",
    "        error_area = 1.96 * selection.std() / math.sqrt(\n",
    "                selection.count()\n",
    "            )\n",
    "        results.append([key, mean, error_area])\n",
    "    print(TITLE)\n",
    "    print(\"{:>60} {:>6} {:>4}\".format(\"\", \"Mean\", \"\"))\n",
    "    for result in results:\n",
    "        print(\"{:<50} {:>6.2%} ±{:>4.2%}\".format(result[0], float(result[1]), float(result[2])))\n",
    "\n",
    "def answer_questions(df: pd.DataFrame, COMPARE_KEYS:List[str], metric):\n",
    "    results = {}\n",
    "    for key in COMPARE_KEYS:\n",
    "        selection = df[key]\n",
    "\n",
    "        mean = selection.mean()\n",
    "        error_area = 1.96 * selection.std() / math.sqrt(\n",
    "                selection.count()\n",
    "            )\n",
    "        results[key] = (mean, error_area)\n",
    "    \n",
    "    better_than_initial = [k for k,v in results.items() if v[0] > results[metric + \"_initial\"][0]]\n",
    "    better_than_initial_with_ws = [k for k in better_than_initial if \"ws_and_al\" in k]\n",
    "    print(\"Which are better than initial?\", better_than_initial)\n",
    "    print(\"Which are better than initial, with WS?\", better_than_initial_with_ws)\n",
    "\n",
    "    al_ws_al_better_than_al_al = []\n",
    "\n",
    "    for k in results.keys():\n",
    "        if k.startswith(metric + \"_ws_and_al\"):\n",
    "            if results[k][0] > results[k.replace(\"_ws_and_al\", \"_al_and_al\")][0]:\n",
    "                al_ws_al_better_than_al_al.append(k)\n",
    "\n",
    "    print(\"Where is al+ws+al better than al+al?\", al_ws_al_better_than_al_al)\n",
    "    \n",
    "    #print(\"Where is the mean improvement within the 95% error range?\")\n",
    "    \n",
    "    \n",
    "# 01 general improvement\n",
    "for metric in ['acc', 'f1']:\n",
    "    COMPARE_KEYS = [metric+\"_initial\", metric +\"_ws\"]\n",
    "    for AL_SAMPLING_STRATEGY in [\"UncertaintyMaxMargin_no_ws\",\n",
    "            \"UncertaintyMaxMargin_with_ws\",\n",
    "            \"Random\",\n",
    "            \"CoveredByLeastAmountOfLf\",\n",
    "            \"ClassificationIsMostWrong\",\n",
    "            \"GreatestDisagreement\"]:\n",
    "            COMPARE_KEYS.append(metric + '_ws_and_al_'+AL_SAMPLING_STRATEGY)\n",
    "            COMPARE_KEYS.append(metric + '_al_and_al_'+AL_SAMPLING_STRATEGY)\n",
    "    comparison_table(df, COMPARE_KEYS=COMPARE_KEYS, TITLE=\"General Improvement after AL + WS \" + metric)\n",
    "    answer_questions(df, COMPARE_KEYS=COMPARE_KEYS, metric=metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "################################################################################\n",
      "DATASET\n",
      "################################################################################\n",
      "\n",
      "wine :  271\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "FERTILITY :  271\n",
      "Which are better than initial? ['acc_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_Random', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "IONOSPHERE :  299\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "HABERMAN :  310\n",
      "Which are better than initial? ['acc_ws']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf']\n",
      "Which are better than initial? ['f1_ws']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf']\n",
      "\n",
      "parkinsons :  269\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "flag :  277\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "PLANNING :  297\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "ILPD :  269\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "PIMA :  249\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "DIABETES :  288\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "synthetic :  2745\n",
      "Which are better than initial? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "BREAST :  268\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "zoo :  239\n",
      "Which are better than initial? ['acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_Random', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "GERMAN :  296\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "australian :  296\n",
      "Which are better than initial? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_Random', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "glass :  253\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws', 'f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "HEART :  262\n",
      "Which are better than initial? ['acc_ws']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "abalone :  263\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "olivetti :  246\n",
      "Which are better than initial? []\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_al_and_al_Random']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "dwtc :  255\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? ['f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_ClassificationIsMostWrong']\n",
      "\n",
      "cifar10 :  288\n",
      "Which are better than initial? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "adult :  269\n",
      "Which are better than initial? ['acc_ws']\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Which are better than initial? []\n",
      "Which are better than initial, with WS? []\n",
      "Where is al+ws+al better than al+al? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement']\n",
      "\n",
      "################################################################################\n",
      "MERGE_WS_SAMPLES_STRATEGY\n",
      "################################################################################\n",
      "\n",
      "MajorityVoteLabelMergeStrategy :  3157\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "RandomLabelMergeStrategy :  3121\n",
      "Which are better than initial? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong']\n",
      "Where is al+ws+al better than al+al? []\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random']\n",
      "Where is al+ws+al better than al+al? []\n",
      "\n",
      "SnorkelLabelMergeStrategy :  2202\n",
      "Which are better than initial? ['acc_ws', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_al_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? ['acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random']\n",
      "Which are better than initial? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_al_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_GreatestDisagreement']\n",
      "Which are better than initial, with WS? ['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_GreatestDisagreement']\n",
      "Where is al+ws+al better than al+al? []\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# group by stuff\n",
    "for group_by in ['DATASET', 'MERGE_WS_SAMPLES_STRATEGY']:\n",
    "    print()\n",
    "    print(\"#\"*80)\n",
    "    print(group_by)\n",
    "    print(\"#\"*80)\n",
    "    for group_by_value in df[group_by].unique():\n",
    "        print()\n",
    "        print(group_by_value, \": \", len(df[df[group_by] == group_by_value]))\n",
    "        for metric in ['acc', 'f1']:\n",
    "            COMPARE_KEYS = [metric+\"_initial\", metric +\"_ws\"]\n",
    "            for AL_SAMPLING_STRATEGY in [\"UncertaintyMaxMargin_no_ws\",\n",
    "                    \"UncertaintyMaxMargin_with_ws\",\n",
    "                    \"Random\",\n",
    "                    \"CoveredByLeastAmountOfLf\",\n",
    "                    \"ClassificationIsMostWrong\",\n",
    "                    \"GreatestDisagreement\"]:\n",
    "                    COMPARE_KEYS.append(metric + '_ws_and_al_'+AL_SAMPLING_STRATEGY)\n",
    "                    COMPARE_KEYS.append(metric + '_al_and_al_'+AL_SAMPLING_STRATEGY)\n",
    "            #comparison_table(df, COMPARE_KEYS=COMPARE_KEYS, TITLE=\"General Improvement after AL + WS \" + metric)\n",
    "            answer_questions(df[df[group_by] == group_by_value], COMPARE_KEYS=COMPARE_KEYS, metric=metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  city keep1  f1_ws_and_al_U  f1_al_and_al_U  f1_ws_and_al_R  f1_al_and_al_R  \\\n",
      "0    A     D               1               4               7              10   \n",
      "1    B     E               2               5               8              11   \n",
      "2    C     F               3               6               9              12   \n",
      "\n",
      "   f1_ws_and_al_X  f1_al_and_al_X  acc_ws_and_al_U  acc_al_and_al_U  \\\n",
      "0              13              16               19               22   \n",
      "1              14              17               20               23   \n",
      "2              15              18               21               24   \n",
      "\n",
      "   acc_ws_and_al_R  acc_al_and_al_R  acc_ws_and_al_X  acc_al_and_al_X  \n",
      "0               25               27               30               33  \n",
      "1               26               28               31               34  \n",
      "2               27               29               32               35  \n",
      "['f1_ws_and_al_U', 'f1_ws_and_al_R', 'f1_ws_and_al_X', 'f1_al_and_al_U', 'f1_al_and_al_R', 'f1_al_and_al_X', 'acc_ws_and_al_U', 'acc_ws_and_al_R', 'acc_ws_and_al_X', 'acc_al_and_al_U', 'acc_al_and_al_R', 'acc_al_and_al_X']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['city', 'keep1', 'f1_al_and_ws_and_al', 'f1_al_and_al',\n",
       "       'acc_al_and_ws_and_al', 'acc_al_and_al'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#andersherum an die Sache gehen : im großen df schauen wo al_ws_al better than al+al -> und dann schauen wie viel prozent davon jeweils auf die einzelnen werte von datasets, … verteilt sind!!\n",
    "df1 = pd.DataFrame({'city':['A','B','C'],\n",
    "                    'keep1':['D', 'E', 'F'],\n",
    "                    'f1_ws_and_al_U':[1,2,3],\n",
    "                    'f1_al_and_al_U':[4,5,6],\n",
    "                    'f1_ws_and_al_R':[7,8,9],\n",
    "                    'f1_al_and_al_R':[10,11,12],\n",
    "                    'f1_ws_and_al_X':[13,14,15],\n",
    "                    'f1_al_and_al_X':[16,17,18],\n",
    "\n",
    "                    'acc_ws_and_al_U':[19,20,21],\n",
    "                    'acc_al_and_al_U':[22,23,24],\n",
    "                    'acc_ws_and_al_R':[25,26,27],\n",
    "                    'acc_al_and_al_R':[27,28,29],\n",
    "                    'acc_ws_and_al_X':[30,31,32],\n",
    "                    'acc_al_and_al_X':[33,34,35],\n",
    "})\n",
    "print(df1)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "df_new = pd.DataFrame()\n",
    "al_sampling_strategies = [\"U\", \"R\", \"X\"]\n",
    "to_split_metrics = (\n",
    "    [\"f1_ws_and_al_\" + k for k in al_sampling_strategies]\n",
    "    + [\"f1_al_and_al_\" + k for k in al_sampling_strategies]\n",
    "    + [\"acc_ws_and_al_\" + k for k in al_sampling_strategies]\n",
    "    + [\"acc_al_and_al_\" + k for k in al_sampling_strategies]\n",
    ")\n",
    "print(to_split_metrics)\n",
    "\n",
    "for _, row in df1.iterrows():\n",
    "    for al_sampling_strategy in al_sampling_strategies:\n",
    "        # remove all other metrics\n",
    "        row2 = row.drop(\n",
    "            [m for m in to_split_metrics if not m.endswith(al_sampling_strategy)]\n",
    "        )\n",
    "        row2 = row2.rename(\n",
    "            {\n",
    "                \"f1_ws_and_al_\" + al_sampling_strategy: \"f1_al_and_ws_and_al\",\n",
    "                \"acc_ws_and_al_\" + al_sampling_strategy: \"acc_al_and_ws_and_al\",\n",
    "                \"f1_al_and_al_\" + al_sampling_strategy: \"f1_al_and_al\",\n",
    "                \"acc_al_and_al_\" + al_sampling_strategy: \"acc_al_and_al\",\n",
    "            }\n",
    "        )\n",
    "        df_new = df_new.append(row2)\n",
    "\n",
    "df_new.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['f1_ws_and_al_UncertaintyMaxMargin_no_ws', 'f1_ws_and_al_UncertaintyMaxMargin_with_ws', 'f1_ws_and_al_Random', 'f1_ws_and_al_CoveredByLeastAmountOfLf', 'f1_ws_and_al_ClassificationIsMostWrong', 'f1_ws_and_al_GreatestDisagreement', 'f1_al_and_al_UncertaintyMaxMargin_no_ws', 'f1_al_and_al_UncertaintyMaxMargin_with_ws', 'f1_al_and_al_Random', 'f1_al_and_al_CoveredByLeastAmountOfLf', 'f1_al_and_al_ClassificationIsMostWrong', 'f1_al_and_al_GreatestDisagreement', 'acc_ws_and_al_UncertaintyMaxMargin_no_ws', 'acc_ws_and_al_UncertaintyMaxMargin_with_ws', 'acc_ws_and_al_Random', 'acc_ws_and_al_CoveredByLeastAmountOfLf', 'acc_ws_and_al_ClassificationIsMostWrong', 'acc_ws_and_al_GreatestDisagreement', 'acc_al_and_al_UncertaintyMaxMargin_no_ws', 'acc_al_and_al_UncertaintyMaxMargin_with_ws', 'acc_al_and_al_Random', 'acc_al_and_al_CoveredByLeastAmountOfLf', 'acc_al_and_al_ClassificationIsMostWrong', 'acc_al_and_al_GreatestDisagreement']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['n_samples', 'n_features', 'n_informative', 'n_redundant', 'n_repeated',\n",
       "       'n_classes', 'n_clusters_per_class', 'weights', 'flip_y', 'class_sep',\n",
       "       'hypercube', 'scale', 'random_state', 'ABSTAIN_THRESHOLDS',\n",
       "       'LF_CLASSIFIERS', 'AMOUNT_OF_LF_FEATURESSS', 'acc_WS', 'f1_WS',\n",
       "       'f1_initial', 'acc_initial', 'f1_ws', 'acc_ws', 'acc_al_and_ws_and_al',\n",
       "       'f1_al_and_ws_and_al', 'acc_al_and_al', 'f1_al_and_al',\n",
       "       'amount_of_initial_al_samples', 'amount_of_lastly_al_samples',\n",
       "       'amount_of_ws_labelled_samples', 'JOB_ID', 'AL_SAMPLES_WEIGHT',\n",
       "       'AMOUNT_OF_LFS', 'CLUSTERED_AL_WS_COMBINATION', 'DATASET',\n",
       "       'DATASET_RANDOM_GENERATION_SEED',\n",
       "       'FRACTION_OF_INITIALLY_LABELLED_SAMPLES',\n",
       "       'FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES', 'LF_RANDOM_SEED',\n",
       "       'MERGE_WS_SAMPLES_STRATEGY', 'al_sampling_strategy'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CREATE_CSV = True\n",
    "if CREATE_CSV:\n",
    "    # unravel al sampling strat metrics\n",
    "    al_sampling_strategies = [\n",
    "        \"UncertaintyMaxMargin_no_ws\",\n",
    "        \"UncertaintyMaxMargin_with_ws\",\n",
    "        \"Random\",\n",
    "        \"CoveredByLeastAmountOfLf\",\n",
    "        \"ClassificationIsMostWrong\",\n",
    "        \"GreatestDisagreement\",\n",
    "    ]\n",
    "\n",
    "    df_new = pd.DataFrame()\n",
    "    to_split_metrics = (\n",
    "        [\"f1_ws_and_al_\" + k for k in al_sampling_strategies]\n",
    "        + [\"f1_al_and_al_\" + k for k in al_sampling_strategies]\n",
    "        + [\"acc_ws_and_al_\" + k for k in al_sampling_strategies]\n",
    "        + [\"acc_al_and_al_\" + k for k in al_sampling_strategies]\n",
    "    )\n",
    "    print(to_split_metrics)\n",
    "\n",
    "    for _, row in df.iterrows():\n",
    "        for al_sampling_strategy in al_sampling_strategies:\n",
    "            # remove all other metrics\n",
    "            row2 = row.drop(\n",
    "                [m for m in to_split_metrics if not m.endswith(al_sampling_strategy)]\n",
    "            )\n",
    "            row2 = row2.rename(\n",
    "                {\n",
    "                    \"f1_ws_and_al_\" + al_sampling_strategy: \"f1_al_and_ws_and_al\",\n",
    "                    \"acc_ws_and_al_\" + al_sampling_strategy: \"acc_al_and_ws_and_al\",\n",
    "                    \"f1_al_and_al_\" + al_sampling_strategy: \"f1_al_and_al\",\n",
    "                    \"acc_al_and_al_\" + al_sampling_strategy: \"acc_al_and_al\",\n",
    "                }\n",
    "            )\n",
    "            row2['al_sampling_strategy'] = al_sampling_strategy\n",
    "            df_new = df_new.append(row2)\n",
    "\n",
    "    df_new\n",
    "df_new.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "if CREATE_CSV:\n",
    "    df_new.to_csv(\"how_to.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['n_samples', 'n_features', 'n_informative', 'n_redundant', 'n_repeated',\n",
      "       'n_classes', 'n_clusters_per_class', 'weights', 'flip_y', 'class_sep',\n",
      "       'hypercube', 'scale', 'random_state', 'ABSTAIN_THRESHOLDS',\n",
      "       'LF_CLASSIFIERS', 'AMOUNT_OF_LF_FEATURESSS', 'acc_WS', 'f1_WS',\n",
      "       'f1_initial', 'acc_initial', 'f1_ws', 'acc_ws', 'acc_al_and_ws_and_al',\n",
      "       'f1_al_and_ws_and_al', 'acc_al_and_al', 'f1_al_and_al',\n",
      "       'amount_of_initial_al_samples', 'amount_of_lastly_al_samples',\n",
      "       'amount_of_ws_labelled_samples', 'JOB_ID', 'AL_SAMPLES_WEIGHT',\n",
      "       'AMOUNT_OF_LFS', 'CLUSTERED_AL_WS_COMBINATION', 'DATASET',\n",
      "       'DATASET_RANDOM_GENERATION_SEED',\n",
      "       'FRACTION_OF_INITIALLY_LABELLED_SAMPLES',\n",
      "       'FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES', 'LF_RANDOM_SEED',\n",
      "       'MERGE_WS_SAMPLES_STRATEGY', 'al_sampling_strategy'],\n",
      "      dtype='object')\n",
      "Total:  50880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/julius/.local/share/virtualenvs/code-oYwF_TsS/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (32) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_samples</th>\n",
       "      <th>n_features</th>\n",
       "      <th>n_informative</th>\n",
       "      <th>n_redundant</th>\n",
       "      <th>n_repeated</th>\n",
       "      <th>n_classes</th>\n",
       "      <th>n_clusters_per_class</th>\n",
       "      <th>weights</th>\n",
       "      <th>flip_y</th>\n",
       "      <th>class_sep</th>\n",
       "      <th>...</th>\n",
       "      <th>AL_SAMPLES_WEIGHT</th>\n",
       "      <th>AMOUNT_OF_LFS</th>\n",
       "      <th>CLUSTERED_AL_WS_COMBINATION</th>\n",
       "      <th>DATASET</th>\n",
       "      <th>DATASET_RANDOM_GENERATION_SEED</th>\n",
       "      <th>FRACTION_OF_INITIALLY_LABELLED_SAMPLES</th>\n",
       "      <th>FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES</th>\n",
       "      <th>LF_RANDOM_SEED</th>\n",
       "      <th>MERGE_WS_SAMPLES_STRATEGY</th>\n",
       "      <th>al_sampling_strategy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>178.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1.667661</td>\n",
       "      <td>1.0</td>\n",
       "      <td>wine</td>\n",
       "      <td>914092.0</td>\n",
       "      <td>0.244126</td>\n",
       "      <td>0.168291</td>\n",
       "      <td>61088.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>UncertaintyMaxMargin_no_ws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>178.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1.667661</td>\n",
       "      <td>True</td>\n",
       "      <td>wine</td>\n",
       "      <td>914092.0</td>\n",
       "      <td>0.244126</td>\n",
       "      <td>0.168291</td>\n",
       "      <td>61088.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>UncertaintyMaxMargin_with_ws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>178.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1.667661</td>\n",
       "      <td>True</td>\n",
       "      <td>wine</td>\n",
       "      <td>914092.0</td>\n",
       "      <td>0.244126</td>\n",
       "      <td>0.168291</td>\n",
       "      <td>61088.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>Random</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>178.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1.667661</td>\n",
       "      <td>True</td>\n",
       "      <td>wine</td>\n",
       "      <td>914092.0</td>\n",
       "      <td>0.244126</td>\n",
       "      <td>0.168291</td>\n",
       "      <td>61088.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>CoveredByLeastAmountOfLf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>178.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1.667661</td>\n",
       "      <td>True</td>\n",
       "      <td>wine</td>\n",
       "      <td>914092.0</td>\n",
       "      <td>0.244126</td>\n",
       "      <td>0.168291</td>\n",
       "      <td>61088.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>ClassificationIsMostWrong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50875</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>7.216152</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>484338.0</td>\n",
       "      <td>0.094111</td>\n",
       "      <td>0.399906</td>\n",
       "      <td>501539.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>UncertaintyMaxMargin_with_ws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50876</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>7.216152</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>484338.0</td>\n",
       "      <td>0.094111</td>\n",
       "      <td>0.399906</td>\n",
       "      <td>501539.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>Random</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50877</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>7.216152</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>484338.0</td>\n",
       "      <td>0.094111</td>\n",
       "      <td>0.399906</td>\n",
       "      <td>501539.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>CoveredByLeastAmountOfLf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50878</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>7.216152</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>484338.0</td>\n",
       "      <td>0.094111</td>\n",
       "      <td>0.399906</td>\n",
       "      <td>501539.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>ClassificationIsMostWrong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50879</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>3073.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>...</td>\n",
       "      <td>33.0</td>\n",
       "      <td>7.216152</td>\n",
       "      <td>True</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>484338.0</td>\n",
       "      <td>0.094111</td>\n",
       "      <td>0.399906</td>\n",
       "      <td>501539.0</td>\n",
       "      <td>MajorityVoteLabelMergeStrategy</td>\n",
       "      <td>GreatestDisagreement</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50880 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       n_samples  n_features  n_informative  n_redundant  n_repeated  \\\n",
       "0          178.0        14.0           14.0          0.0         0.0   \n",
       "1          178.0        14.0           14.0          0.0         0.0   \n",
       "2          178.0        14.0           14.0          0.0         0.0   \n",
       "3          178.0        14.0           14.0          0.0         0.0   \n",
       "4          178.0        14.0           14.0          0.0         0.0   \n",
       "...          ...         ...            ...          ...         ...   \n",
       "50875    10000.0      3073.0         3073.0          0.0         0.0   \n",
       "50876    10000.0      3073.0         3073.0          0.0         0.0   \n",
       "50877    10000.0      3073.0         3073.0          0.0         0.0   \n",
       "50878    10000.0      3073.0         3073.0          0.0         0.0   \n",
       "50879    10000.0      3073.0         3073.0          0.0         0.0   \n",
       "\n",
       "       n_classes n_clusters_per_class weights flip_y class_sep  ...  \\\n",
       "0            3.0                    ?       ?      ?         ?  ...   \n",
       "1            3.0                    ?       ?      ?         ?  ...   \n",
       "2            3.0                    ?       ?      ?         ?  ...   \n",
       "3            3.0                    ?       ?      ?         ?  ...   \n",
       "4            3.0                    ?       ?      ?         ?  ...   \n",
       "...          ...                  ...     ...    ...       ...  ...   \n",
       "50875       10.0                    ?       ?      ?         ?  ...   \n",
       "50876       10.0                    ?       ?      ?         ?  ...   \n",
       "50877       10.0                    ?       ?      ?         ?  ...   \n",
       "50878       10.0                    ?       ?      ?         ?  ...   \n",
       "50879       10.0                    ?       ?      ?         ?  ...   \n",
       "\n",
       "      AL_SAMPLES_WEIGHT AMOUNT_OF_LFS  CLUSTERED_AL_WS_COMBINATION  DATASET  \\\n",
       "0                  62.0      1.667661                          1.0     wine   \n",
       "1                  62.0      1.667661                         True     wine   \n",
       "2                  62.0      1.667661                         True     wine   \n",
       "3                  62.0      1.667661                         True     wine   \n",
       "4                  62.0      1.667661                         True     wine   \n",
       "...                 ...           ...                          ...      ...   \n",
       "50875              33.0      7.216152                         True  cifar10   \n",
       "50876              33.0      7.216152                         True  cifar10   \n",
       "50877              33.0      7.216152                         True  cifar10   \n",
       "50878              33.0      7.216152                         True  cifar10   \n",
       "50879              33.0      7.216152                         True  cifar10   \n",
       "\n",
       "      DATASET_RANDOM_GENERATION_SEED FRACTION_OF_INITIALLY_LABELLED_SAMPLES  \\\n",
       "0                           914092.0                               0.244126   \n",
       "1                           914092.0                               0.244126   \n",
       "2                           914092.0                               0.244126   \n",
       "3                           914092.0                               0.244126   \n",
       "4                           914092.0                               0.244126   \n",
       "...                              ...                                    ...   \n",
       "50875                       484338.0                               0.094111   \n",
       "50876                       484338.0                               0.094111   \n",
       "50877                       484338.0                               0.094111   \n",
       "50878                       484338.0                               0.094111   \n",
       "50879                       484338.0                               0.094111   \n",
       "\n",
       "      FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES LF_RANDOM_SEED  \\\n",
       "0                                   0.168291        61088.0   \n",
       "1                                   0.168291        61088.0   \n",
       "2                                   0.168291        61088.0   \n",
       "3                                   0.168291        61088.0   \n",
       "4                                   0.168291        61088.0   \n",
       "...                                      ...            ...   \n",
       "50875                               0.399906       501539.0   \n",
       "50876                               0.399906       501539.0   \n",
       "50877                               0.399906       501539.0   \n",
       "50878                               0.399906       501539.0   \n",
       "50879                               0.399906       501539.0   \n",
       "\n",
       "            MERGE_WS_SAMPLES_STRATEGY          al_sampling_strategy  \n",
       "0      MajorityVoteLabelMergeStrategy    UncertaintyMaxMargin_no_ws  \n",
       "1      MajorityVoteLabelMergeStrategy  UncertaintyMaxMargin_with_ws  \n",
       "2      MajorityVoteLabelMergeStrategy                        Random  \n",
       "3      MajorityVoteLabelMergeStrategy      CoveredByLeastAmountOfLf  \n",
       "4      MajorityVoteLabelMergeStrategy     ClassificationIsMostWrong  \n",
       "...                               ...                           ...  \n",
       "50875  MajorityVoteLabelMergeStrategy  UncertaintyMaxMargin_with_ws  \n",
       "50876  MajorityVoteLabelMergeStrategy                        Random  \n",
       "50877  MajorityVoteLabelMergeStrategy      CoveredByLeastAmountOfLf  \n",
       "50878  MajorityVoteLabelMergeStrategy     ClassificationIsMostWrong  \n",
       "50879  MajorityVoteLabelMergeStrategy          GreatestDisagreement  \n",
       "\n",
       "[50880 rows x 40 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"how_to.csv\")\n",
    "\n",
    "print(df.keys())\n",
    "print(\"Total: \", len(df))\n",
    "\n",
    "no_hist_keys = [\n",
    "    \"ABSTAIN_THRESHOLDS\",\n",
    "    \"AL_SAMPLES_WEIGTHS\",\n",
    "    \"DATASET_RANDOM_GENERATION_SEED\",\n",
    "    \"JOB_ID\",\n",
    "    \"LF_CLASSIFIERS\",\n",
    "    \"LF_RANDOM_SEED\",\n",
    "    \"random_state\",\n",
    "    \"AMOUNT_OF_LF_FEATURESSS\",\n",
    "]\n",
    "\n",
    "hist_keys_that_are_lists = [\n",
    "    \"ABSTAIN_THRESHOLDS\",\n",
    "    \"LF_CLASSIFIERS\",\n",
    "    \"AMOUNT_OF_LF_FEATURESSS\",\n",
    "]\n",
    "\n",
    "# 1. check the value ranges -> does it make sense to go one_hot_encoding? for LF_CLASSIFIERS definitely\n",
    "# 2. for the rest not so much -,-\n",
    "\n",
    "\n",
    "title=\"all\"\n",
    "'''\n",
    "for key in df:\n",
    "    if key.startswith(\"acc_\") or key.startswith(\"f1_\") or key in no_hist_keys:\n",
    "        continue\n",
    "    print(key)\n",
    "    sns.histplot(df, y=key).set(title=title)\n",
    "    plt.savefig('plots/' + title+ '_' + key + '.jpg', dpi=300, bbox_inches=\"tight\")\n",
    "    plt.savefig('plots/' + title+ '_' + key + '.pdf',dpi=300, format=\"pdf\", bbox_inches=\"tight\")\n",
    "    plt.clf()\n",
    "\n",
    "\n",
    "def analyse_value_distributions(df, df_a, df_not_a, title):\n",
    "    for key in df_a:\n",
    "        if key.startswith(\"acc_\") or key.startswith(\"f1_\") or key in no_hist_keys:\n",
    "            continue\n",
    "        print(key)\n",
    "\n",
    "        f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, sharex=True)\n",
    "        ax1.set_title(title)\n",
    "        sns.histplot(df_a, y=key, ax=ax1)\n",
    "        sns.histplot(df_not_a, y=key, ax=ax2)\n",
    "        plt.savefig(\"plots/\" + title + \"_\" + key + \".jpg\", dpi=300, bbox_inches=\"tight\")\n",
    "        plt.savefig(\n",
    "            \"plots/\" + title + \"_\" + key + \".pdf\",\n",
    "            dpi=300,\n",
    "            format=\"pdf\",\n",
    "            bbox_inches=\"tight\",\n",
    "        )\n",
    "        plt.clf()\n",
    "        #acc per lf als stärke der lf\n",
    "\n",
    "for metric in [\"acc\", \"f1\"]:\n",
    "    print(metric)\n",
    "        \n",
    "    analyse_value_distributions(\n",
    "        df,\n",
    "        df.loc[(df[metric + \"_initial\"] > df[metric + \"_al_and_al\"])\n",
    "            & (df[metric + \"_initial\"] > df[metric + \"_al_and_ws_and_al\"])],\n",
    "        df.loc[(df[metric + \"_initial\"] <= df[metric + \"_al_and_al\"])\n",
    "            & (df[metric + \"_initial\"] <= df[metric + \"_al_and_ws_and_al\"])],\n",
    "        metric + \" Initial was better than everything else\",\n",
    "    )\n",
    "    analyse_value_distributions(\n",
    "        df,\n",
    "        df.loc[(df[metric + \"_ws\"] > df[metric + \"_initial\"])],\n",
    "        df.loc[(df[metric + \"_ws\"] >= df[metric + \"_initial\"])],\n",
    "         metric + \"Only WS > Initial\",\n",
    "    )\n",
    "    analyse_value_distributions(\n",
    "        df,\n",
    "        df.loc[(df[metric + \"_al_and_ws_and_al\"] > df[metric + \"_ws\"])],\n",
    "        df.loc[(df[metric + \"_al_and_ws_and_al\"] <= df[metric + \"_ws\"])],\n",
    "         metric + \" AL and WS and AL > Only WS\",\n",
    "    )\n",
    "    analyse_value_distributions(\n",
    "        df,\n",
    "        df.loc[ (df[metric + \"_al_and_al\"] > df[metric + \"_initial\"])],\n",
    "        df.loc[ (df[metric + \"_al_and_al\"] <= df[metric + \"_initial\"])],\n",
    "         metric + \" AL and AL > AL\",\n",
    "    )\n",
    "    analyse_value_distributions(\n",
    "        df,\n",
    "        df.loc[(df[metric + \"_al_and_ws_and_al\"] < df[metric + \"_ws\"])],\n",
    "        df.loc[(df[metric + \"_al_and_ws_and_al\"] >= df[metric + \"_ws\"])],\n",
    "        metric + \" AL and WS and AL < Only WS\",\n",
    "    )'''\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/julius/.local/share/virtualenvs/code-oYwF_TsS/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (32) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ABSTAIN_THRESHOLDS': defaultdict(<function <lambda> at 0x7f1070d290d0>, {0.2: 23916, 0.8: 23682, 0.4: 24228, 0.7: 23604, 0.1: 24060, 0.5: 23694, 0.0: 12318, 0.6: 24570, 0.3: 24162, 0.9: 23868, 1.0: 11640}), 'LF_CLASSIFIERS': defaultdict(<function <lambda> at 0x7f106fe34280>, {'dt': 80352, 'lr': 79866, 'knn': 79524}), 'AMOUNT_OF_LF_FEATURESSS': defaultdict(<function <lambda> at 0x7f106fe3c820>, {1: 145410, 3: 24174, 2: 41256, 4: 16524, 5: 12378})}\n",
      "ABSTAIN_THRESHOLDS\n",
      "LF_CLASSIFIERS\n",
      "AMOUNT_OF_LF_FEATURESSS\n",
      "Index(['n_samples', 'n_features', 'n_informative', 'n_redundant', 'n_repeated',\n",
      "       'n_classes', 'n_clusters_per_class', 'weights', 'flip_y', 'class_sep',\n",
      "       'hypercube', 'scale', 'random_state', 'ABSTAIN_THRESHOLDS',\n",
      "       'LF_CLASSIFIERS', 'AMOUNT_OF_LF_FEATURESSS', 'acc_WS', 'f1_WS',\n",
      "       'f1_initial', 'acc_initial', 'f1_ws', 'acc_ws', 'acc_al_and_ws_and_al',\n",
      "       'f1_al_and_ws_and_al', 'acc_al_and_al', 'f1_al_and_al',\n",
      "       'amount_of_initial_al_samples', 'amount_of_lastly_al_samples',\n",
      "       'amount_of_ws_labelled_samples', 'JOB_ID', 'AL_SAMPLES_WEIGHT',\n",
      "       'AMOUNT_OF_LFS', 'CLUSTERED_AL_WS_COMBINATION', 'DATASET',\n",
      "       'DATASET_RANDOM_GENERATION_SEED',\n",
      "       'FRACTION_OF_INITIALLY_LABELLED_SAMPLES',\n",
      "       'FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES', 'LF_RANDOM_SEED',\n",
      "       'MERGE_WS_SAMPLES_STRATEGY', 'al_sampling_strategy',\n",
      "       'ABSTAIN_THRESHOLDS_0.2', 'ABSTAIN_THRESHOLDS_0.8',\n",
      "       'ABSTAIN_THRESHOLDS_0.4', 'ABSTAIN_THRESHOLDS_0.7',\n",
      "       'ABSTAIN_THRESHOLDS_0.1', 'ABSTAIN_THRESHOLDS_0.5',\n",
      "       'ABSTAIN_THRESHOLDS_0.0', 'ABSTAIN_THRESHOLDS_0.6',\n",
      "       'ABSTAIN_THRESHOLDS_0.3', 'ABSTAIN_THRESHOLDS_0.9',\n",
      "       'ABSTAIN_THRESHOLDS_1.0', 'LF_CLASSIFIERS_dt', 'LF_CLASSIFIERS_lr',\n",
      "       'LF_CLASSIFIERS_knn', 'AMOUNT_OF_LF_FEATURESSS_1',\n",
      "       'AMOUNT_OF_LF_FEATURESSS_3', 'AMOUNT_OF_LF_FEATURESSS_2',\n",
      "       'AMOUNT_OF_LF_FEATURESSS_4', 'AMOUNT_OF_LF_FEATURESSS_5'],\n",
      "      dtype='object')\n",
      "Index(['n_samples', 'n_features', 'n_informative', 'n_redundant', 'n_repeated',\n",
      "       'n_classes', 'n_clusters_per_class', 'weights', 'flip_y', 'class_sep',\n",
      "       'hypercube', 'scale', 'random_state', 'acc_WS', 'f1_WS', 'f1_initial',\n",
      "       'acc_initial', 'f1_ws', 'acc_ws', 'acc_al_and_ws_and_al',\n",
      "       'f1_al_and_ws_and_al', 'acc_al_and_al', 'f1_al_and_al',\n",
      "       'amount_of_initial_al_samples', 'amount_of_lastly_al_samples',\n",
      "       'amount_of_ws_labelled_samples', 'JOB_ID', 'AL_SAMPLES_WEIGHT',\n",
      "       'AMOUNT_OF_LFS', 'CLUSTERED_AL_WS_COMBINATION', 'DATASET',\n",
      "       'DATASET_RANDOM_GENERATION_SEED',\n",
      "       'FRACTION_OF_INITIALLY_LABELLED_SAMPLES',\n",
      "       'FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES', 'LF_RANDOM_SEED',\n",
      "       'MERGE_WS_SAMPLES_STRATEGY', 'al_sampling_strategy',\n",
      "       'ABSTAIN_THRESHOLDS_0.2', 'ABSTAIN_THRESHOLDS_0.8',\n",
      "       'ABSTAIN_THRESHOLDS_0.4', 'ABSTAIN_THRESHOLDS_0.7',\n",
      "       'ABSTAIN_THRESHOLDS_0.1', 'ABSTAIN_THRESHOLDS_0.5',\n",
      "       'ABSTAIN_THRESHOLDS_0.0', 'ABSTAIN_THRESHOLDS_0.6',\n",
      "       'ABSTAIN_THRESHOLDS_0.3', 'ABSTAIN_THRESHOLDS_0.9',\n",
      "       'ABSTAIN_THRESHOLDS_1.0', 'LF_CLASSIFIERS_dt', 'LF_CLASSIFIERS_lr',\n",
      "       'LF_CLASSIFIERS_knn', 'AMOUNT_OF_LF_FEATURESSS_1',\n",
      "       'AMOUNT_OF_LF_FEATURESSS_3', 'AMOUNT_OF_LF_FEATURESSS_2',\n",
      "       'AMOUNT_OF_LF_FEATURESSS_4', 'AMOUNT_OF_LF_FEATURESSS_5'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "import ast\n",
    "from collections import defaultdict\n",
    "\n",
    "df = pd.read_csv(\"how_to.csv\")\n",
    "\n",
    "hist_keys_that_are_lists =  [\n",
    "    \"ABSTAIN_THRESHOLDS\", # first binarize, then count values?\n",
    "    \"LF_CLASSIFIERS\", # -> #dt #knn #lr -> count values\n",
    "    \"AMOUNT_OF_LF_FEATURESSS\", # -> #1lfs #2lfs #3fs -> count values\n",
    "]\n",
    "key_values = {}\n",
    "for key in hist_keys_that_are_lists:\n",
    "    key_values[key] = defaultdict(lambda: 0)\n",
    "\n",
    "    for row in df[key]:\n",
    "        if key == 'AL_SAMPLES_WEIGHT':\n",
    "            parsed_object = int(row)\n",
    "        else:\n",
    "            parsed_object = ast.literal_eval(row)\n",
    "\n",
    "        if isinstance(parsed_object, list):\n",
    "            for value in parsed_object:\n",
    "                if key != 'ABSTAIN_THRESHOLDS':\n",
    "                    key_values[key][value] += 1 \n",
    "                else:\n",
    "                    key_values[key][round(value,1)] += 1\n",
    "        else:\n",
    "            key_values[key][10*round(parsed_object/10)] += 1\n",
    "    #print(key)\n",
    "    #print(key_values)\n",
    "print(key_values)\n",
    "CREATE_CSV = True\n",
    "if CREATE_CSV:\n",
    "    # initialize new colums with 0\n",
    "    for k,v in key_values.items():\n",
    "        for value in v.keys():\n",
    "            df[k+'_'+str(value)] = 0\n",
    "\n",
    "    for key in hist_keys_that_are_lists:\n",
    "        print(key)\n",
    "        for ix, row in df[key].iteritems():\n",
    "\n",
    "            if key == 'AL_SAMPLES_WEIGHT':\n",
    "                parsed_object = int(row)\n",
    "            else:\n",
    "                parsed_object = ast.literal_eval(row)\n",
    "                \n",
    "            if isinstance(parsed_object, list):\n",
    "                for value in parsed_object:\n",
    "                    if key != 'ABSTAIN_THRESHOLDS':\n",
    "                        df.loc[ix, key + '_' + str(value)] += 1\n",
    "                    else:\n",
    "                        df.loc[ix, key + '_' + str(round(value,1))] += 1\n",
    "            else:\n",
    "                df.loc[ix, key + '_' + str(10*round(parsed_object/10))] += 1\n",
    "    print(df.keys())\n",
    "    df.drop(hist_keys_that_are_lists, axis=1, inplace=True)\n",
    "    print(df.keys())\n",
    "    df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['n_samples', 'n_features', 'n_informative', 'n_redundant', 'n_repeated',\n",
       "       'n_classes', 'n_clusters_per_class', 'weights', 'flip_y', 'class_sep',\n",
       "       'hypercube', 'scale', 'random_state', 'acc_WS', 'f1_WS', 'f1_initial',\n",
       "       'acc_initial', 'f1_ws', 'acc_ws', 'acc_al_and_ws_and_al',\n",
       "       'f1_al_and_ws_and_al', 'acc_al_and_al', 'f1_al_and_al',\n",
       "       'amount_of_initial_al_samples', 'amount_of_lastly_al_samples',\n",
       "       'amount_of_ws_labelled_samples', 'JOB_ID', 'AL_SAMPLES_WEIGHT',\n",
       "       'AMOUNT_OF_LFS', 'CLUSTERED_AL_WS_COMBINATION', 'DATASET',\n",
       "       'DATASET_RANDOM_GENERATION_SEED',\n",
       "       'FRACTION_OF_INITIALLY_LABELLED_SAMPLES',\n",
       "       'FRACTION_OF_LASTLY_AL_LABELLED_SAMPLES', 'LF_RANDOM_SEED',\n",
       "       'MERGE_WS_SAMPLES_STRATEGY', 'al_sampling_strategy',\n",
       "       'ABSTAIN_THRESHOLDS_0.2', 'ABSTAIN_THRESHOLDS_0.8',\n",
       "       'ABSTAIN_THRESHOLDS_0.4', 'ABSTAIN_THRESHOLDS_0.7',\n",
       "       'ABSTAIN_THRESHOLDS_0.1', 'ABSTAIN_THRESHOLDS_0.5',\n",
       "       'ABSTAIN_THRESHOLDS_0.0', 'ABSTAIN_THRESHOLDS_0.6',\n",
       "       'ABSTAIN_THRESHOLDS_0.3', 'ABSTAIN_THRESHOLDS_0.9',\n",
       "       'ABSTAIN_THRESHOLDS_1.0', 'LF_CLASSIFIERS_dt', 'LF_CLASSIFIERS_lr',\n",
       "       'LF_CLASSIFIERS_knn', 'AMOUNT_OF_LF_FEATURESSS_1',\n",
       "       'AMOUNT_OF_LF_FEATURESSS_3', 'AMOUNT_OF_LF_FEATURESSS_2',\n",
       "       'AMOUNT_OF_LF_FEATURESSS_4', 'AMOUNT_OF_LF_FEATURESSS_5'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv('new.csv', index=False)\n",
    "df.keys()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "178c561f06d1c9b562d26393e106d1bb838dc5f57458805df561d13c135e6443"
  },
  "kernelspec": {
   "display_name": "Python 3.8.11 64-bit ('code-oYwF_TsS': venv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}