Experiment_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7049be18-2025-4ed0-b06f-b0269fbdbb93",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from transformers import BertTokenizer\n",
    "import pickle\n",
    "\n",
    "from textattack.models.wrappers import ModelWrapper\n",
    "from textattack.datasets import HuggingFaceDataset\n",
    "from textattack.attack_recipes import PWWSRen2019\n",
    "from textattack import Attacker, AttackArgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9033dc5d-ad95-400b-95af-79a713169aac",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn import CrossEntropyLoss\n",
    "import textattack\n",
    "from model_wrapper import ModelWrapper\n",
    "\n",
    "from models import model_1 as mm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d1a81f4e-44c5-4d2f-a422-c6f8f4e2b81c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42ac3741-10da-4013-8925-1f0d8ce411a7",
   "metadata": {
    "tags": []
   },
   "source": [
    "### PyTorchModelWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3916c7f2-9acc-4cf6-9447-1cb42d824813",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class PyTorchModelWrapper(ModelWrapper):\n",
    "    \"\"\"Loads a PyTorch model (`nn.Module`) and tokenizer.\n",
    "\n",
    "    Args:\n",
    "        model (torch.nn.Module): PyTorch model\n",
    "        tokenizer: tokenizer whose output can be packed as a tensor and passed to the model.\n",
    "            No type requirement, but most have `tokenizer` method that accepts list of strings.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, model, tokenizer):\n",
    "        if not isinstance(model, torch.nn.Module):\n",
    "            raise TypeError(\n",
    "                f\"PyTorch model must be torch.nn.Module, got type {type(model)}\"\n",
    "            )\n",
    "\n",
    "        self.model = model\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def to(self, device):\n",
    "        self.model.to(device)\n",
    "\n",
    "    def __call__(self, text_input_list, batch_size=32):\n",
    "        model_device = next(self.model.parameters()).device\n",
    "        ############################################################ Change 1\n",
    "        ids = self.tokenizer(text_input_list, return_tensors=\"pt\", padding=True)\n",
    "        ids = ids.to(model_device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = textattack.shared.utils.batch_model_predict(\n",
    "                self.model, ids, batch_size=batch_size\n",
    "            )\n",
    "\n",
    "        return outputs\n",
    "\n",
    "    def get_grad(self, text_input, loss_fn=CrossEntropyLoss()):\n",
    "        \"\"\"Get gradient of loss with respect to input tokens.\n",
    "\n",
    "        Args:\n",
    "            text_input (str): input string\n",
    "            loss_fn (torch.nn.Module): loss function. Default is `torch.nn.CrossEntropyLoss`\n",
    "        Returns:\n",
    "            Dict of ids, tokens, and gradient as numpy array.\n",
    "        \"\"\"\n",
    "\n",
    "        if not hasattr(self.model, \"get_input_embeddings\"):\n",
    "            raise AttributeError(\n",
    "                f\"{type(self.model)} must have method `get_input_embeddings` that returns `torch.nn.Embedding` object that represents input embedding layer\"\n",
    "            )\n",
    "        if not isinstance(loss_fn, torch.nn.Module):\n",
    "            raise ValueError(\"Loss function must be of type `torch.nn.Module`.\")\n",
    "\n",
    "        self.model.train()\n",
    "\n",
    "        embedding_layer = self.model.get_input_embeddings()\n",
    "        original_state = embedding_layer.weight.requires_grad\n",
    "        embedding_layer.weight.requires_grad = True\n",
    "\n",
    "        emb_grads = []\n",
    "\n",
    "        def grad_hook(module, grad_in, grad_out):\n",
    "            emb_grads.append(grad_out[0])\n",
    "\n",
    "        emb_hook = embedding_layer.register_backward_hook(grad_hook)\n",
    "\n",
    "        self.model.zero_grad()\n",
    "        model_device = next(self.model.parameters()).device\n",
    "        \n",
    "        ######################################################### Change 2\n",
    "        ids = self.tokenizer(text_input_list, return_tensors=\"pt\", padding=True)\n",
    "        ids = ids.to(model_device)\n",
    "\n",
    "        predictions = self.model(ids)\n",
    "\n",
    "        output = predictions.argmax(dim=1)\n",
    "        loss = loss_fn(predictions, output)\n",
    "        loss.backward()\n",
    "\n",
    "        # grad w.r.t to word embeddings\n",
    "\n",
    "        # Fix for Issue #601\n",
    "\n",
    "        # Check if gradient has shape [max_sequence,1,_] ( when model input in transpose of input sequence)\n",
    "\n",
    "        if emb_grads[0].shape[1] == 1:\n",
    "            grad = torch.transpose(emb_grads[0], 0, 1)[0].cpu().numpy()\n",
    "        else:\n",
    "            # gradient has shape [1,max_sequence,_]\n",
    "            grad = emb_grads[0][0].cpu().numpy()\n",
    "\n",
    "        embedding_layer.weight.requires_grad = original_state\n",
    "        emb_hook.remove()\n",
    "        self.model.eval()\n",
    "\n",
    "        output = {\"ids\": ids[0].tolist(), \"gradient\": grad}\n",
    "\n",
    "        return output\n",
    "\n",
    "    def _tokenize(self, inputs):\n",
    "        \"\"\"Helper method that for `tokenize`\n",
    "        Args:\n",
    "            inputs (list[str]): list of input strings\n",
    "        Returns:\n",
    "            tokens (list[list[str]]): List of list of tokens as strings\n",
    "        \"\"\"\n",
    "        return [self.tokenizer.convert_ids_to_tokens(self.tokenizer(x)) for x in inputs]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0580a9c-44b6-4bc9-9cd1-5bcdb0c19f15",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Self Tx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a031a345-3045-460a-b645-722db0226563",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "d_model = 768\n",
    "max_token = 512\n",
    "nhead = 12\n",
    "nlayer = 6\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d85c977d-8aba-460e-afb0-474b636ee2e1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(\"bert_embedding.pkl\", \"rb\") as file:\n",
    "    embd = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f2e89fd2-a358-4b23-b041-32c94e2ca1b5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = mm.ad_transformer(d_model,nhead, nlayer, max_token, embd, device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "89254143-a950-4897-aba7-4d3cdca9e44f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b504c8db-b6e6-4f29-a188-2f2cc26e63fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(\"weight_trained_1.pth\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8ce8152-23ec-49bd-a94b-a40a13654076",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Spam Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bbbadc2d-fed9-4268-b05c-d511694bff8b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#f = pd.read_csv(\"dataset/spam_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ec987be9-14c2-434c-b029-bc29f04eb71d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df = df.drop(\"len\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "09fc61b8-bb0e-4f75-b0da-debb5aa1bc2d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Removing Sequence more than 500 tokens\n",
    "#tok_threshold = []\n",
    "#for i in range(len(df) - 1, -1, -1):\n",
    "#    tok_len = len(tokenizer(df.iloc[i,1], return_tensors=\"pt\", padding=True)[\"input_ids\"][0])\n",
    "#    if tok_len > 500:\n",
    "#        tok_threshold.append(i)\n",
    "#        \n",
    "#df.drop(859, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6a7e333a-a3cf-4095-bdfe-2b18ad513b50",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df = df[df[\"text_type\"] == \"spam\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "047cca20-5893-4e61-9293-aca91a1577e6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#df_test.to_csv(\"attack_spam_only.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3aa6e39e-a4ae-4645-a807-d123ef095cdb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\"dataset/attack_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ac673a5a-d863-4828-9d6b-c894d30b31be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = df_test.replace(\"spam\",0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5946d094-a653-4cdb-bfd8-274627f15105",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Testing Acurracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "cca8c6be-5ed1-4d6f-a837-8ead76728b98",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Actual Value (SPAM, HAM)  ## Y - Predicted Value (SPAM, HAM)\n",
    "#conf_matrix = np.zeros((2,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "63c28c47-cda0-491c-b7d0-35d5116fe270",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfor i in range(len(df_test)):\\n    input_list = df_test.iloc[i,1]\\n    tok_input = tokenizer(input_list, return_tensors=\"pt\", padding=True).to(device)\\n    out = model(tok_input)\\n    \\n    index_1 = torch.argmax(out[0]).tolist()\\n    \\n    if df_test.iloc[i,0] == 0:\\n        index_2 = 0\\n    else:\\n        index_2 = 1\\n        \\n    conf_matrix[index_1][index_2] += 1\\n'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "for i in range(len(df_test)):\n",
    "    input_list = df_test.iloc[i,1]\n",
    "    tok_input = tokenizer(input_list, return_tensors=\"pt\", padding=True).to(device)\n",
    "    out = model(tok_input)\n",
    "    \n",
    "    index_1 = torch.argmax(out[0]).tolist()\n",
    "    \n",
    "    if df_test.iloc[i,0] == 0:\n",
    "        index_2 = 0\n",
    "    else:\n",
    "        index_2 = 1\n",
    "        \n",
    "    conf_matrix[index_1][index_2] += 1\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "afaccf29-a4ba-4cd2-b595-2c6643b4ee03",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#conf_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61c33c6-8611-41cb-88e6-e07b450f23b8",
   "metadata": {},
   "source": [
    "### Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7e2b0de6-f07d-4d1f-a52f-c93d988500e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test = df_test.rename(columns={'text_type': 'label'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a7a14008-a471-4843-9de0-2bc4b10a7ed2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_wrapper = PyTorchModelWrapper(model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "85db722d-9611-4aaa-98d0-450653bbd8df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from textattack import Attack\n",
    "from textattack.search_methods import GreedySearch\n",
    "from textattack.constraints.pre_transformation import (\n",
    "    RepeatModification,\n",
    "    StopwordModification,\n",
    ")\n",
    "from textattack.goal_functions import UntargetedClassification, TargetedClassification\n",
    "from textattack.transformations import WordSwapEmbedding\n",
    "from textattack.constraints.pre_transformation import RepeatModification\n",
    "from textattack.constraints.pre_transformation import StopwordModification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c447e937-f2ef-438b-a96b-b402d1dbd13d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n"
     ]
    }
   ],
   "source": [
    "goal_function = TargetedClassification(model_wrapper, target_class = 1)\n",
    "\n",
    "transformation = WordSwapEmbedding()\n",
    "\n",
    "constraints = [RepeatModification(), StopwordModification()]\n",
    "\n",
    "search_method = GreedySearch()\n",
    "\n",
    "attack = Attack(goal_function, constraints, transformation, search_method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "367153a4-ef8c-4f98-81c2-67e37da48e0b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "custom_ds = []\n",
    "for i in range(len(df_test)):\n",
    "    custom_ds.append((df_test.iloc[i,1], int(df_test.iloc[i,0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "415e4dd0-77e1-4795-9d70-4f75309f3f77",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = textattack.datasets.Dataset(custom_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fd58a638-8402-42b2-b70d-c609cb3661ad",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### 21 success 9 failed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4631a6-626f-4899-ac4a-1a84713bbfd5",
   "metadata": {},
   "source": [
    "### Getting Low Rank of only Attention Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e621695c-96fb-4001-a366-c35e920578ee",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_model_weight = model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4c15323f-826f-4e89-8207-c66a332c3c8e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2048, 768])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_model_weight['transformer_encoder.layers.0.linear1.weight'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75269675-832c-4ee2-8622-885b88e6561f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ff31a33-1867-42ec-9f69-a6c29d3cb69e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "d170a09d-d159-4c0a-a113-5756cb2a0f5f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def svd_sa(model, rank):\n",
    "    all_model_weight = model.state_dict()\n",
    "    \n",
    "    for layer_num in range(nlayer):\n",
    "        layer_name = 'transformer_encoder.layers.' + str(layer_num) + '.linear1.weight'\n",
    "\n",
    "        in_proj_weight = all_model_weight[layer_name]\n",
    "\n",
    "        U, s, V = np.linalg.svd(in_proj_weight.to(\"cpu\").numpy())\n",
    "        low_rank_mat = np.dot(U[:, :rank], np.dot(np.diag(s[:rank]), V[:rank, :]))\n",
    "        in_proj_weight = low_rank_mat\n",
    "\n",
    "        U, s, V = np.linalg.svd(k.to(\"cpu\").numpy())\n",
    "        low_rank_mat = np.dot(U[:, :rank], np.dot(np.diag(s[:rank]), V[:rank, :]))\n",
    "        k = low_rank_mat\n",
    "\n",
    "        U, s, V = np.linalg.svd(v.to(\"cpu\").numpy())\n",
    "        low_rank_mat = np.dot(U[:, :rank], np.dot(np.diag(s[:rank]), V[:rank, :]))\n",
    "        v = low_rank_mat\n",
    "\n",
    "        all_model_weight[layer_name] = torch.concat((torch.tensor(q),torch.tensor(k), torch.tensor(v))).to(\"cuda\")\n",
    "        \n",
    "        return all_model_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0486247-b0b1-417e-93d1-9b18d1a092d3",
   "metadata": {},
   "source": [
    "### Experimenting on different rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "b03e61e4-cdab-4535-a035-0d0a9b153326",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "transformation = WordSwapEmbedding()\n",
    "constraints = [RepeatModification(), StopwordModification()]\n",
    "search_method = GreedySearch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08f829d2-61e3-4ce9-bc58-26891d3b5ff5",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_50.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 1 / 1:   3%|██▎                                                                   | 1/30 [00:00<00:01, 20.69it/s]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 0 / 0 / 1 / 1:   7%|████▋                                                                 | 2/30 [00:03<00:42,  1.52s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 1 / 2:   7%|████▋                                                                 | 2/30 [00:03<00:42,  1.53s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3:  10%|███████                                                               | 3/30 [00:06<00:56,  2.11s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 1 / 3:  13%|█████████▎                                                            | 4/30 [00:09<01:04,  2.48s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 1 / 4:  13%|█████████▎                                                            | 4/30 [00:09<01:04,  2.49s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 1 / 5:  17%|███████████▋                                                          | 5/30 [00:13<01:05,  2.61s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513339230.ta.chkpt\" at 2024-04-19 07:55:39 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 1 / 5:  20%|██████████████                                                        | 6/30 [00:16<01:05,  2.72s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 1 / 6:  20%|██████████████                                                        | 6/30 [00:16<01:05,  2.73s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 0 / 1 / 7:  23%|████████████████▎                                                     | 7/30 [00:20<01:05,  2.86s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 0 / 1 / 7:  27%|██████████████████▋                                                   | 8/30 [00:20<00:56,  2.59s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 1 / 1 / 8:  27%|██████████████████▋                                                   | 8/30 [00:20<00:57,  2.59s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 1 / 9:  30%|█████████████████████                                                 | 9/30 [00:24<00:57,  2.73s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 1 / 9:  33%|███████████████████████                                              | 10/30 [00:27<00:54,  2.74s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 1 / 10:  33%|██████████████████████▋                                             | 10/30 [00:27<00:54,  2.74s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513353603.ta.chkpt\" at 2024-04-19 07:55:53 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 1 / 11:  37%|████████████████████████▉                                           | 11/30 [00:30<00:53,  2.82s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 1 / 11:  40%|███████████████████████████▏                                        | 12/30 [00:39<00:59,  3.30s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 1 / 12:  40%|███████████████████████████▏                                        | 12/30 [00:39<00:59,  3.30s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 2 / 1 / 12:  43%|█████████████████████████████▍                                      | 13/30 [00:47<01:02,  3.66s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 3 / 1 / 13:  43%|█████████████████████████████▍                                      | 13/30 [00:47<01:02,  3.66s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 3 / 1 / 13:  47%|███████████████████████████████▋                                    | 14/30 [00:51<00:58,  3.64s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 1 / 14:  47%|███████████████████████████████▎                                   | 14/30 [00:51<00:58,  3.65s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 3 / 1 / 14:  50%|█████████████████████████████████▌                                 | 15/30 [00:59<00:59,  3.98s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 1 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [00:59<00:59,  3.98s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513385857.ta.chkpt\" at 2024-04-19 07:56:25 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 4 / 1 / 15:  53%|███████████████████████████████████▋                               | 16/30 [01:12<01:03,  4.52s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 5 / 1 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:12<01:03,  4.52s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 5 / 1 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:15<00:57,  4.45s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 5 / 1 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:15<00:57,  4.46s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 5 / 1 / 17:  60%|████████████████████████████████████████▏                          | 18/30 [01:18<00:52,  4.38s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 5 / 1 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:18<00:52,  4.38s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 5 / 1 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:28<00:51,  4.68s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 5 / 1 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:29<00:51,  4.69s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 5 / 1 / 19:  67%|████████████████████████████████████████████▋                      | 20/30 [01:33<00:46,  4.69s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 5 / 1 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:33<00:46,  4.69s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513420070.ta.chkpt\" at 2024-04-19 07:57:00 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 5 / 1 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:39<00:42,  4.76s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 1 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:39<00:42,  4.76s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 1 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:47<00:39,  4.90s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 1 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:47<00:39,  4.91s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 5 / 1 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:53<00:34,  4.92s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 1 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [01:53<00:34,  4.92s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 5 / 1 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:58<00:29,  4.94s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 5 / 1 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [01:58<00:29,  4.94s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 5 / 1 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:20<00:28,  5.64s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 6 / 1 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:20<00:28,  5.64s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513467157.ta.chkpt\" at 2024-04-19 07:57:47 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 6 / 1 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:30<00:23,  5.78s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 1 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:30<00:23,  5.78s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 1 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:45<00:18,  6.14s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 6 / 1 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [02:45<00:18,  6.14s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 6 / 1 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:56<00:12,  6.31s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 7 / 1 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [02:56<00:12,  6.31s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 7 / 1 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:59<00:06,  6.20s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 7 / 1 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [02:59<00:06,  6.20s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 7 / 1 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:03<00:00,  6.12s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 7 / 1 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:03<00:00,  6.12s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513509671.ta.chkpt\" at 2024-04-19 07:58:29 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 7 / 1 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:03<00:00,  6.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 22     |\n",
      "| Number of failed attacks:     | 7      |\n",
      "| Number of skipped attacks:    | 1      |\n",
      "| Original accuracy:            | 96.67% |\n",
      "| Accuracy under attack:        | 23.33% |\n",
      "| Attack success rate:          | 75.86% |\n",
      "| Average perturbed word %:     | 9.0%   |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 399.28 |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_100.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "  0%|                                                                                                                                    | 0/30 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
      "\n",
      "  3%|████▏                                                                                                                       | 1/30 [00:02<01:18,  2.71s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1:   3%|██▎                                                                   | 1/30 [00:02<01:19,  2.73s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 1 / 0 / 0 / 1:   7%|████▋                                                                 | 2/30 [00:05<01:22,  2.93s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 0 / 2:   7%|████▋                                                                 | 2/30 [00:05<01:22,  2.94s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 0 / 2:  10%|███████                                                               | 3/30 [00:09<01:25,  3.18s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 0 / 3:  10%|███████                                                               | 3/30 [00:09<01:25,  3.19s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 3 / 0 / 0 / 3:  13%|█████████▎                                                            | 4/30 [00:13<01:27,  3.36s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 0 / 4:  13%|█████████▎                                                            | 4/30 [00:13<01:27,  3.36s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 4 / 0 / 0 / 4:  17%|███████████▋                                                          | 5/30 [00:16<01:23,  3.35s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:16<01:23,  3.35s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513527749.ta.chkpt\" at 2024-04-19 07:58:47 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  20%|██████████████                                                        | 6/30 [00:20<01:20,  3.37s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 0 / 0 / 6:  20%|██████████████                                                        | 6/30 [00:20<01:20,  3.37s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 6 / 0 / 0 / 6:  23%|████████████████▎                                                     | 7/30 [00:23<01:18,  3.42s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 0 / 0 / 7:  23%|████████████████▎                                                     | 7/30 [00:23<01:18,  3.42s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 0 / 0 / 7:  27%|██████████████████▋                                                   | 8/30 [00:24<01:07,  3.08s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 0 / 8:  27%|██████████████████▋                                                   | 8/30 [00:24<01:08,  3.09s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 7 / 1 / 0 / 8:  30%|█████████████████████                                                 | 9/30 [00:33<01:18,  3.75s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 0 / 9:  30%|█████████████████████                                                 | 9/30 [00:33<01:18,  3.75s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 8 / 1 / 0 / 9:  33%|███████████████████████                                              | 10/30 [00:36<01:13,  3.67s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:36<01:13,  3.68s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513547753.ta.chkpt\" at 2024-04-19 07:59:07 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  37%|████████████████████████▉                                           | 11/30 [00:40<01:09,  3.67s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 1 / 0 / 11:  37%|████████████████████████▌                                          | 11/30 [00:40<01:09,  3.67s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 10 / 1 / 0 / 11:  40%|██████████████████████████▊                                        | 12/30 [00:44<01:06,  3.68s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 1 / 0 / 12:  40%|██████████████████████████▊                                        | 12/30 [00:44<01:06,  3.69s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 1 / 0 / 12:  43%|█████████████████████████████                                      | 13/30 [00:52<01:09,  4.07s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 2 / 0 / 13:  43%|█████████████████████████████                                      | 13/30 [00:52<01:09,  4.07s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 11 / 2 / 0 / 13:  47%|███████████████████████████████▎                                   | 14/30 [00:58<01:06,  4.16s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 2 / 0 / 14:  47%|███████████████████████████████▎                                   | 14/30 [00:58<01:06,  4.16s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 2 / 0 / 14:  50%|█████████████████████████████████▌                                 | 15/30 [01:06<01:06,  4.46s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:06<01:06,  4.47s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513577985.ta.chkpt\" at 2024-04-19 07:59:37 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  53%|███████████████████████████████████▋                               | 16/30 [01:20<01:10,  5.03s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 0 / 16:  53%|███████████████████████████████████▋                               | 16/30 [01:20<01:10,  5.03s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 12 / 4 / 0 / 16:  57%|█████████████████████████████████████▉                             | 17/30 [01:24<01:04,  4.95s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 0 / 17:  57%|█████████████████████████████████████▉                             | 17/30 [01:24<01:04,  4.95s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 13 / 4 / 0 / 17:  60%|████████████████████████████████████████▏                          | 18/30 [01:27<00:58,  4.86s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 0 / 18:  60%|████████████████████████████████████████▏                          | 18/30 [01:27<00:58,  4.86s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 14 / 4 / 0 / 18:  63%|██████████████████████████████████████████▍                        | 19/30 [01:40<00:58,  5.30s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 0 / 19:  63%|██████████████████████████████████████████▍                        | 19/30 [01:40<00:58,  5.30s/it]\u001b[A\u001b[Atextattack: CSVLogger exiting without calling flush().\n",
      "[Succeeded / Failed / Skipped / Total] 2 / 0 / 3 / 5:  17%|███████████▏                                                       | 5/30 [43:23<3:36:59, 520.79s/it]\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 15 / 4 / 0 / 19:  67%|████████████████████████████████████████████▋                      | 20/30 [01:46<00:53,  5.32s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:46<00:53,  5.32s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513617491.ta.chkpt\" at 2024-04-19 08:00:17 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:52<00:48,  5.36s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 0 / 21:  70%|██████████████████████████████████████████████▉                    | 21/30 [01:52<00:48,  5.36s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 17 / 4 / 0 / 21:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:57<00:42,  5.35s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 4 / 0 / 22:  73%|█████████████████████████████████████████████████▏                 | 22/30 [01:57<00:42,  5.35s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 18 / 4 / 0 / 22:  77%|███████████████████████████████████████████████████▎               | 23/30 [02:06<00:38,  5.52s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 4 / 0 / 23:  77%|███████████████████████████████████████████████████▎               | 23/30 [02:06<00:38,  5.52s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 19 / 4 / 0 / 23:  80%|█████████████████████████████████████████████████████▌             | 24/30 [02:16<00:34,  5.67s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 4 / 0 / 24:  80%|█████████████████████████████████████████████████████▌             | 24/30 [02:16<00:34,  5.67s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 20 / 4 / 0 / 24:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:35<00:31,  6.22s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 4 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:35<00:31,  6.23s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513666641.ta.chkpt\" at 2024-04-19 08:01:06 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 4 / 0 / 25:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:48<00:25,  6.48s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 4 / 0 / 26:  87%|██████████████████████████████████████████████████████████         | 26/30 [02:48<00:25,  6.48s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 4 / 0 / 26:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [03:01<00:20,  6.74s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 4 / 0 / 27:  90%|████████████████████████████████████████████████████████████▎      | 27/30 [03:01<00:20,  6.74s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 4 / 0 / 27:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [03:13<00:13,  6.90s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 5 / 0 / 28:  93%|██████████████████████████████████████████████████████████████▌    | 28/30 [03:13<00:13,  6.90s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 5 / 0 / 28:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [03:16<00:06,  6.76s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 24 / 5 / 0 / 29:  97%|████████████████████████████████████████████████████████████████▊  | 29/30 [03:16<00:06,  6.76s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 24 / 5 / 0 / 29: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:19<00:00,  6.66s/it]\u001b[A\u001b[A\n",
      "\n",
      "[Succeeded / Failed / Skipped / Total] 25 / 5 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:19<00:00,  6.66s/it]\u001b[A\u001b[Atextattack: Saving checkpoint under \"checkpoints/1713513710786.ta.chkpt\" at 2024-04-19 08:01:50 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 25 / 5 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:19<00:00,  6.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 25     |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 16.67% |\n",
      "| Attack success rate:          | 83.33% |\n",
      "| Average perturbed word %:     | 11.03% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 405.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_150.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:16<01:23,  3.35s/it]textattack: Saving checkpoint under \"checkpoints/1713513728879.ta.chkpt\" at 2024-04-19 08:02:08 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:34<01:08,  3.40s/it]textattack: Saving checkpoint under \"checkpoints/1713513746133.ta.chkpt\" at 2024-04-19 08:02:26 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:10<01:10,  4.71s/it]textattack: Saving checkpoint under \"checkpoints/1713513782841.ta.chkpt\" at 2024-04-19 08:03:02 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:47<00:53,  5.39s/it]textattack: Saving checkpoint under \"checkpoints/1713513819872.ta.chkpt\" at 2024-04-19 08:03:39 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:41<00:32,  6.47s/it]textattack: Saving checkpoint under \"checkpoints/1713513874003.ta.chkpt\" at 2024-04-19 08:04:34 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 24 / 6 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:24<00:00,  6.81s/it]textattack: Saving checkpoint under \"checkpoints/1713513916437.ta.chkpt\" at 2024-04-19 08:05:16 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 24 / 6 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:24<00:00,  6.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 24     |\n",
      "| Number of failed attacks:     | 6      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 20.0%  |\n",
      "| Attack success rate:          | 80.0%  |\n",
      "| Average perturbed word %:     | 10.86% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 413.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_200.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:17<01:25,  3.41s/it]textattack: Saving checkpoint under \"checkpoints/1713513934890.ta.chkpt\" at 2024-04-19 08:05:34 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:33<01:07,  3.39s/it]textattack: Saving checkpoint under \"checkpoints/1713513951762.ta.chkpt\" at 2024-04-19 08:05:51 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:09<01:09,  4.65s/it]textattack: Saving checkpoint under \"checkpoints/1713513987631.ta.chkpt\" at 2024-04-19 08:06:27 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:46<00:53,  5.33s/it]textattack: Saving checkpoint under \"checkpoints/1713514024448.ta.chkpt\" at 2024-04-19 08:07:04 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:49<00:33,  6.76s/it]textattack: Saving checkpoint under \"checkpoints/1713514086919.ta.chkpt\" at 2024-04-19 08:08:06 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 24 / 6 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:30<00:00,  7.03s/it]textattack: Saving checkpoint under \"checkpoints/1713514128826.ta.chkpt\" at 2024-04-19 08:08:48 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 24 / 6 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:30<00:00,  7.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 24     |\n",
      "| Number of failed attacks:     | 6      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 20.0%  |\n",
      "| Attack success rate:          | 80.0%  |\n",
      "| Average perturbed word %:     | 12.48% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 426.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_250.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:17<01:27,  3.49s/it]textattack: Saving checkpoint under \"checkpoints/1713514147675.ta.chkpt\" at 2024-04-19 08:09:07 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:37<01:15,  3.78s/it]textattack: Saving checkpoint under \"checkpoints/1713514168008.ta.chkpt\" at 2024-04-19 08:09:28 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:09<01:09,  4.65s/it]textattack: Saving checkpoint under \"checkpoints/1713514199962.ta.chkpt\" at 2024-04-19 08:09:59 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:51<00:55,  5.58s/it]textattack: Saving checkpoint under \"checkpoints/1713514241728.ta.chkpt\" at 2024-04-19 08:10:41 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:48<00:33,  6.72s/it]textattack: Saving checkpoint under \"checkpoints/1713514298330.ta.chkpt\" at 2024-04-19 08:11:38 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:33<00:00,  7.13s/it]textattack: Saving checkpoint under \"checkpoints/1713514343983.ta.chkpt\" at 2024-04-19 08:12:23 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:33<00:00,  7.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 23     |\n",
      "| Number of failed attacks:     | 7      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 23.33% |\n",
      "| Attack success rate:          | 76.67% |\n",
      "| Average perturbed word %:     | 10.43% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 434.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_300.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:19<01:39,  3.96s/it]textattack: Saving checkpoint under \"checkpoints/1713514365234.ta.chkpt\" at 2024-04-19 08:12:45 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:36<01:13,  3.67s/it]textattack: Saving checkpoint under \"checkpoints/1713514382123.ta.chkpt\" at 2024-04-19 08:13:02 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:06<01:06,  4.41s/it]textattack: Saving checkpoint under \"checkpoints/1713514411571.ta.chkpt\" at 2024-04-19 08:13:31 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [01:48<00:54,  5.41s/it]textattack: Saving checkpoint under \"checkpoints/1713514453579.ta.chkpt\" at 2024-04-19 08:14:13 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 21 / 4 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:41<00:32,  6.45s/it]textattack: Saving checkpoint under \"checkpoints/1713514506773.ta.chkpt\" at 2024-04-19 08:15:06 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 25 / 5 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:27<00:00,  6.93s/it]textattack: Saving checkpoint under \"checkpoints/1713514553300.ta.chkpt\" at 2024-04-19 08:15:53 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 25 / 5 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:27<00:00,  6.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 25     |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 16.67% |\n",
      "| Attack success rate:          | 83.33% |\n",
      "| Average perturbed word %:     | 12.26% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 428.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_350.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:55,  4.62s/it]textattack: Saving checkpoint under \"checkpoints/1713514577761.ta.chkpt\" at 2024-04-19 08:16:17 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:45<01:31,  4.56s/it]textattack: Saving checkpoint under \"checkpoints/1713514600298.ta.chkpt\" at 2024-04-19 08:16:40 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:23<01:23,  5.57s/it]textattack: Saving checkpoint under \"checkpoints/1713514638170.ta.chkpt\" at 2024-04-19 08:17:18 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:07<01:03,  6.37s/it]textattack: Saving checkpoint under \"checkpoints/1713514682028.ta.chkpt\" at 2024-04-19 08:18:02 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:05<00:37,  7.42s/it]textattack: Saving checkpoint under \"checkpoints/1713514740277.ta.chkpt\" at 2024-04-19 08:19:00 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:59,  4.77s/it]textattack: Saving checkpoint under \"checkpoints/1713514807915.ta.chkpt\" at 2024-04-19 08:20:07 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:45<01:31,  4.55s/it]textattack: Saving checkpoint under \"checkpoints/1713514829593.ta.chkpt\" at 2024-04-19 08:20:29 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:45<01:31,  4.57s/it]textattack: Saving checkpoint under \"checkpoints/1713515055119.ta.chkpt\" at 2024-04-19 08:24:15 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:22<01:22,  5.50s/it]textattack: Saving checkpoint under \"checkpoints/1713515091919.ta.chkpt\" at 2024-04-19 08:24:51 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:03<01:01,  6.19s/it]textattack: Saving checkpoint under \"checkpoints/1713515133085.ta.chkpt\" at 2024-04-19 08:25:33 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 18 / 7 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:00<00:36,  7.21s/it]textattack: Saving checkpoint under \"checkpoints/1713515189649.ta.chkpt\" at 2024-04-19 08:26:29 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 21 / 9 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:53<00:00,  7.80s/it]textattack: Saving checkpoint under \"checkpoints/1713515243273.ta.chkpt\" at 2024-04-19 08:27:23 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 9 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:53<00:00,  7.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 21     |\n",
      "| Number of failed attacks:     | 9      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 30.0%  |\n",
      "| Attack success rate:          | 70.0%  |\n",
      "| Average perturbed word %:     | 10.83% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 476.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_500.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:56,  4.68s/it]textattack: Saving checkpoint under \"checkpoints/1713515268088.ta.chkpt\" at 2024-04-19 08:27:48 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:45<01:30,  4.54s/it]textattack: Saving checkpoint under \"checkpoints/1713515290119.ta.chkpt\" at 2024-04-19 08:28:10 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:21<01:21,  5.46s/it]textattack: Saving checkpoint under \"checkpoints/1713515326572.ta.chkpt\" at 2024-04-19 08:28:46 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:02<01:01,  6.14s/it]textattack: Saving checkpoint under \"checkpoints/1713515367488.ta.chkpt\" at 2024-04-19 08:29:27 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:51<00:34,  6.87s/it]textattack: Saving checkpoint under \"checkpoints/1713515416394.ta.chkpt\" at 2024-04-19 08:30:16 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:43<00:00,  7.44s/it]textattack: Saving checkpoint under \"checkpoints/1713515467966.ta.chkpt\" at 2024-04-19 08:31:07 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:43<00:00,  7.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 23     |\n",
      "| Number of failed attacks:     | 7      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 23.33% |\n",
      "| Attack success rate:          | 76.67% |\n",
      "| Average perturbed word %:     | 12.54% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 458.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_550.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:55,  4.61s/it]textattack: Saving checkpoint under \"checkpoints/1713515492420.ta.chkpt\" at 2024-04-19 08:31:32 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:43<01:26,  4.34s/it]textattack: Saving checkpoint under \"checkpoints/1713515512781.ta.chkpt\" at 2024-04-19 08:31:52 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:21<01:21,  5.43s/it]textattack: Saving checkpoint under \"checkpoints/1713515550858.ta.chkpt\" at 2024-04-19 08:32:30 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 17 / 3 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:01<01:00,  6.10s/it]textattack: Saving checkpoint under \"checkpoints/1713515591356.ta.chkpt\" at 2024-04-19 08:33:11 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 22 / 3 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [02:58<00:35,  7.15s/it]textattack: Saving checkpoint under \"checkpoints/1713515648021.ta.chkpt\" at 2024-04-19 08:34:08 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 25 / 5 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:48<00:00,  7.61s/it]textattack: Saving checkpoint under \"checkpoints/1713515697626.ta.chkpt\" at 2024-04-19 08:34:57 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 25 / 5 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [03:48<00:00,  7.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 25     |\n",
      "| Number of failed attacks:     | 5      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 16.67% |\n",
      "| Attack success rate:          | 83.33% |\n",
      "| Average perturbed word %:     | 13.37% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 465.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_600.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:57,  4.70s/it]textattack: Saving checkpoint under \"checkpoints/1713515722457.ta.chkpt\" at 2024-04-19 08:35:22 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:43<01:27,  4.37s/it]textattack: Saving checkpoint under \"checkpoints/1713515742663.ta.chkpt\" at 2024-04-19 08:35:42 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:21<01:21,  5.45s/it]textattack: Saving checkpoint under \"checkpoints/1713515780799.ta.chkpt\" at 2024-04-19 08:36:20 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:21<01:10,  7.09s/it]textattack: Saving checkpoint under \"checkpoints/1713515840683.ta.chkpt\" at 2024-04-19 08:37:20 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 20 / 5 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:26<00:41,  8.28s/it]textattack: Saving checkpoint under \"checkpoints/1713515905905.ta.chkpt\" at 2024-04-19 08:38:25 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:23<00:00,  8.80s/it]textattack: Saving checkpoint under \"checkpoints/1713515962977.ta.chkpt\" at 2024-04-19 08:39:22 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:24<00:00,  8.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 23     |\n",
      "| Number of failed attacks:     | 7      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 23.33% |\n",
      "| Attack success rate:          | 76.67% |\n",
      "| Average perturbed word %:     | 15.83% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 539.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_650.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:56,  4.64s/it]textattack: Saving checkpoint under \"checkpoints/1713515987594.ta.chkpt\" at 2024-04-19 08:39:47 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:43<01:27,  4.38s/it]textattack: Saving checkpoint under \"checkpoints/1713516008224.ta.chkpt\" at 2024-04-19 08:40:08 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 12 / 3 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:23<01:23,  5.55s/it]textattack: Saving checkpoint under \"checkpoints/1713516047558.ta.chkpt\" at 2024-04-19 08:40:47 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 16 / 4 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:09<01:04,  6.49s/it]textattack: Saving checkpoint under \"checkpoints/1713516094101.ta.chkpt\" at 2024-04-19 08:41:34 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:13<00:38,  7.73s/it]textattack: Saving checkpoint under \"checkpoints/1713516157525.ta.chkpt\" at 2024-04-19 08:42:37 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 22 / 8 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:08<00:00,  8.27s/it]textattack: Saving checkpoint under \"checkpoints/1713516212524.ta.chkpt\" at 2024-04-19 08:43:32 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 22 / 8 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:08<00:00,  8.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 22     |\n",
      "| Number of failed attacks:     | 8      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 26.67% |\n",
      "| Attack success rate:          | 73.33% |\n",
      "| Average perturbed word %:     | 13.26% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 513.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_700.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:22<01:54,  4.56s/it]textattack: Saving checkpoint under \"checkpoints/1713516236743.ta.chkpt\" at 2024-04-19 08:43:56 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:46<01:33,  4.69s/it]textattack: Saving checkpoint under \"checkpoints/1713516260820.ta.chkpt\" at 2024-04-19 08:44:20 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:27<01:27,  5.81s/it]textattack: Saving checkpoint under \"checkpoints/1713516301125.ta.chkpt\" at 2024-04-19 08:45:01 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:26<01:13,  7.34s/it]textattack: Saving checkpoint under \"checkpoints/1713516360770.ta.chkpt\" at 2024-04-19 08:46:00 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 19 / 6 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:29<00:41,  8.37s/it]textattack: Saving checkpoint under \"checkpoints/1713516423293.ta.chkpt\" at 2024-04-19 08:47:03 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:26<00:00,  8.90s/it]textattack: Saving checkpoint under \"checkpoints/1713516480923.ta.chkpt\" at 2024-04-19 08:48:00 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 23 / 7 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:26<00:00,  8.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 23     |\n",
      "| Number of failed attacks:     | 7      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 23.33% |\n",
      "| Attack success rate:          | 76.67% |\n",
      "| Average perturbed word %:     | 16.18% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 547.0  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_750.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:22<01:54,  4.59s/it]textattack: Saving checkpoint under \"checkpoints/1713516506127.ta.chkpt\" at 2024-04-19 08:48:26 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:46<01:33,  4.68s/it]textattack: Saving checkpoint under \"checkpoints/1713516529981.ta.chkpt\" at 2024-04-19 08:48:49 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:27<01:27,  5.86s/it]textattack: Saving checkpoint under \"checkpoints/1713516571083.ta.chkpt\" at 2024-04-19 08:49:31 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:26<01:13,  7.34s/it]textattack: Saving checkpoint under \"checkpoints/1713516629905.ta.chkpt\" at 2024-04-19 08:50:29 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 18 / 7 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:35<00:43,  8.64s/it]textattack: Saving checkpoint under \"checkpoints/1713516699132.ta.chkpt\" at 2024-04-19 08:51:39 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 21 / 9 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:35<00:00,  9.20s/it]textattack: Saving checkpoint under \"checkpoints/1713516759084.ta.chkpt\" at 2024-04-19 08:52:39 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 9 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:35<00:00,  9.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 21     |\n",
      "| Number of failed attacks:     | 9      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 30.0%  |\n",
      "| Attack success rate:          | 70.0%  |\n",
      "| Average perturbed word %:     | 15.83% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 570.5  |\n",
      "+-------------------------------+--------+"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "textattack: Unknown if model of class <class 'models.model_1.ad_transformer'> compatible with goal function <class 'textattack.goal_functions.classification.targeted_classification.TargetedClassification'>.\n",
      "textattack: Logging to CSV at path rank/log_800.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack(\n",
      "  (search_method): GreedySearch\n",
      "  (goal_function):  TargetedClassification(\n",
      "    (target_class):  1\n",
      "  )\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  15\n",
      "    (embedding):  WordEmbedding\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): RepeatModification\n",
      "    (1): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 5 / 0 / 0 / 5:  17%|███████████▋                                                          | 5/30 [00:23<01:55,  4.63s/it]textattack: Saving checkpoint under \"checkpoints/1713516783666.ta.chkpt\" at 2024-04-19 08:53:03 after 5 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 9 / 1 / 0 / 10:  33%|██████████████████████▋                                             | 10/30 [00:47<01:34,  4.73s/it]textattack: Saving checkpoint under \"checkpoints/1713516807826.ta.chkpt\" at 2024-04-19 08:53:27 after 10 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 11 / 4 / 0 / 15:  50%|█████████████████████████████████▌                                 | 15/30 [01:28<01:28,  5.91s/it]textattack: Saving checkpoint under \"checkpoints/1713516849227.ta.chkpt\" at 2024-04-19 08:54:09 after 15 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 15 / 5 / 0 / 20:  67%|████████████████████████████████████████████▋                      | 20/30 [02:28<01:14,  7.40s/it]textattack: Saving checkpoint under \"checkpoints/1713516908557.ta.chkpt\" at 2024-04-19 08:55:08 after 20 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 18 / 7 / 0 / 25:  83%|███████████████████████████████████████████████████████▊           | 25/30 [03:37<00:43,  8.69s/it]textattack: Saving checkpoint under \"checkpoints/1713516977821.ta.chkpt\" at 2024-04-19 08:56:17 after 25 attacks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Succeeded / Failed / Skipped / Total] 21 / 9 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:38<00:00,  9.28s/it]textattack: Saving checkpoint under \"checkpoints/1713517039043.ta.chkpt\" at 2024-04-19 08:57:19 after 30 attacks.\n",
      "[Succeeded / Failed / Skipped / Total] 21 / 9 / 0 / 30: 100%|███████████████████████████████████████████████████████████████████| 30/30 [04:38<00:00,  9.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "=============================================================================================================================\n",
      "=============================================================================================================================\n",
      "\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 21     |\n",
      "| Number of failed attacks:     | 9      |\n",
      "| Number of skipped attacks:    | 0      |\n",
      "| Original accuracy:            | 100.0% |\n",
      "| Accuracy under attack:        | 30.0%  |\n",
      "| Attack success rate:          | 70.0%  |\n",
      "| Average perturbed word %:     | 15.83% |\n",
      "| Average num. words per input: | 22.7   |\n",
      "| Avg num queries:              | 570.5  |\n",
      "+-------------------------------+--------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for rank in range(50, 850, 50):\n",
    "    model.load_state_dict(torch.load(\"weight_trained_1.pth\"))\n",
    "    \n",
    "    all_model_weight = svd_sa(model, rank)\n",
    "    model.load_state_dict(all_model_weight)\n",
    "    \n",
    "    model_wrapper = PyTorchModelWrapper(model, tokenizer)\n",
    "    goal_function = TargetedClassification(model_wrapper, target_class = 1)\n",
    "    attack = Attack(goal_function, constraints, transformation, search_method)\n",
    "    \n",
    "    attack_args = textattack.AttackArgs(\n",
    "    num_examples=30,\n",
    "    log_to_csv=\"rank/log_\"+str(rank)+\".csv\",\n",
    "    checkpoint_interval=5,\n",
    "    checkpoint_dir=\"checkpoints\",\n",
    "    disable_stdout=True\n",
    "    )\n",
    "    \n",
    "    attacker = Attacker(attack, dataset, attack_args)\n",
    "    attacker.attack_dataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "785b8b40-7250-4898-b64a-4e2e3b2062ad",
   "metadata": {},
   "source": [
    "### Result Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "862bd78a-444a-4c6e-8938-a33c176a9965",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9a2d92cd-44b9-4f29-8785-8aeaf8de8356",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_df = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "97502de0-1dc6-4da2-ac37-309e614f0465",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for i in range(50, 850, 50):\n",
    "    df = pd.read_csv(\"rank/log_\" + str(i) + \".csv\")\n",
    "    all_df.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e60e283c-b27b-4b23-90ef-898f8643f489",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        Skipped\n",
       "1     Successful\n",
       "2     Successful\n",
       "3     Successful\n",
       "4     Successful\n",
       "5     Successful\n",
       "6     Successful\n",
       "7         Failed\n",
       "8     Successful\n",
       "9     Successful\n",
       "10    Successful\n",
       "11        Failed\n",
       "12        Failed\n",
       "13    Successful\n",
       "14        Failed\n",
       "15        Failed\n",
       "16    Successful\n",
       "17    Successful\n",
       "18    Successful\n",
       "19    Successful\n",
       "20    Successful\n",
       "21    Successful\n",
       "22    Successful\n",
       "23    Successful\n",
       "24        Failed\n",
       "25    Successful\n",
       "26    Successful\n",
       "27        Failed\n",
       "28    Successful\n",
       "29    Successful\n",
       "Name: result_type, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df[0][\"result_type\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "36614343-8945-40e3-acd2-d5c0dcca1774",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "88c27e1e-a9c1-4979-af69-bfd362996d34",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "success = []\n",
    "failed = []\n",
    "skipped = []\n",
    "for i in range(len(all_df)):\n",
    "    success.append(all_df[i][\"result_type\"].value_counts()[\"Successful\"])\n",
    "    failed.append(all_df[i][\"result_type\"].value_counts()[\"Failed\"])\n",
    "    try:\n",
    "        skipped.append(all_df[i][\"result_type\"].value_counts()[\"Skipped\"])\n",
    "    except:\n",
    "        skipped.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f70ea510-0fbc-41ab-accb-18a64bdd02ba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[22, 25, 24, 24, 23, 25, 23, 24, 21, 23, 25, 23, 22, 23, 21, 21]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d685db2c-728b-4893-8851-a75bb2217348",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "e6077336-e23d-4163-8084-ffdb805ad5ff",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAu0AAAHHCAYAAAALYWaqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/OQEPoAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWXUlEQVR4nO3deVxUZf//8ffIrgi4IVAoLiQuaO6Zqai4oqVi7omm9jWXMpfMut3atHK5b8usu1DScC0xNbPcwCVzSzT3JVxyTRMQUVE4vz/8OXcji0vKHPH1fDzm8XCu65pzPtecGXxzuOaMxTAMQwAAAABMK5+9CwAAAACQM0I7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7kMuOHDkii8WiCRMm2LuUXBUVFSWLxaIjR47YtH/00UcqXbq0HBwc9OSTT9qlttwWGxsri8Wi2NhYe5diGgEBAerRo4e9ywAA0yK0I8/59NNPZbFYVLt27Sz79+zZozFjxmQKjzcfGxUV9WALfEj8+eefevXVVxUUFCQ3Nzd5e3urVq1aGj58uFJSUu7LPn766Se9/vrrqlu3rmbMmKH333//jh7XoUMHWSwWDR8+PMv+ZcuWacyYMZnaU1NTNWbMmFwLy2Z8PYWEhMhisSgwMDDL/hUrVshischiseibb7656+3n9P4CANw7QjvynOjoaAUEBGjz5s06dOhQpv49e/Zo7NixhPYc/PXXX6pRo4ZmzpypsLAwTZkyRYMHD1bZsmU1bdo0nTt37r7sZ/Xq1cqXL58iIyPVvXt3tWzZ8raPSU5O1pIlSxQQEKA5c+bIMIxMY5YtW6axY8dmak9NTdXYsWPtHtrr16+vy5cvq379+rlSx61cXV116NAhbd68OVNfdHS0XF1d73nbOb2/crJ//3598cUX97xfAMjrHO1dAHA/JSQk6Oeff9bChQv1f//3f4qOjtbo0aPtXdZDJzIyUseOHdOGDRv09NNP2/QlJyfL2dn5vuzn7NmzcnNzu6vtffvtt0pPT9f06dPVqFEjrV27Vg0aNLgv9eSWfPny/aNg/E+VKVNG169f15w5c1SrVi1r+5UrVxQTE6OwsDB9++23D7wOwzB05coVubm5ycXF5YHvDwAeZpxpR54SHR2tQoUKKSwsTO3bt1d0dLRNf1RUlJ5//nlJUsOGDa3LAGJjYxUQEKDdu3crLi7O2h4SEiLpxpnnoUOHKjg4WO7u7vLw8FCLFi20Y8eOTDVcuXJFY8aM0RNPPCFXV1f5+vqqXbt2Onz4cLZ1G4ahl156Sc7Ozlq4cGGOc7x06ZKGDBkif39/ubi4qFy5cpowYUKmM84Wi0UDBgzQokWLVKlSJbm4uKhixYpavnz5bZ/Hw4cPy8HBQU899VSmPg8Pj0yBc9OmTWrevLk8PT2VP39+NWjQQBs2bMhxHxaLRTNmzNClS5esz/ed/JUjOjpaTZo0UcOGDVW+fPlMx7hHjx6aOnWqdR83b0eOHFGxYsUkSWPHjrW2/30Zzb59+9S+fXsVLlxYrq6uqlGjhhYvXmyz/Ztr8zds2KDBgwerWLFiKlCggNq2bas///zTOi6n11N2a9oXLFig6tWry83NTUWLFlW3bt104sSJTPNzd3fXiRMn1KZNG7m7u6tYsWIaOnSo0tPTb/v83dS5c2fNmzdPGRkZ1rYlS5YoNTVVHTp0yDT+6NGj6tevn8qVKyc3NzcVKVJEzz//vM0Z9ZzeXzefk1atWunHH39UjRo15Obmps8//9zad3NNu2EYatiwoYoVK6azZ89at5+Wlqbg4GCVKVNGly5duuO5AkBeQGhHnhIdHa127drJ2dlZnTt31sGDB7VlyxZrf/369fXKK69Ikt58803NmjVLs2bNUvny5fXvf/9bjz/+uIKCgqztb731liTp999/16JFi9SqVStNmjRJw4YN02+//aYGDRro5MmT1u2np6erVatWGjt2rKpXr66JEyfq1VdfVVJSknbt2pVlzenp6erRo4dmzpypmJgYtWvXLtv5GYahZ599VpMnT1bz5s01adIklStXTsOGDdPgwYMzjV+/fr369eunTp066cMPP9SVK1cUHh6u8+fP5/g8lixZUunp6Zo1a1aO46QbS1zq16+v5ORkjR49Wu+//74SExPVqFGjLJdf3DRr1izVq1dPLi4u1uf7dstFTp48qTVr1qhz586SbgTPb775RmlpadYx//d//6cmTZpY93HzVqxYMU2bNk2S1LZtW2v7zed79+7deuqpp7R371698cYbmjhxogoUKKA2bdooJiYmUy0DBw7Ujh07NHr0aL388stasmSJBgwYYO3P6fWUlaioKHXo0EEODg4aN26c+vTpo4ULF+qZZ55RYmKizdj09HQ1a9ZMRYoU0YQJE9SgQQNNnDhR//3vf3N8/v6uS5cuOnXqlM0vDrNnz1bjxo3l7e2dafyWLVv0888/q1OnTpoyZYr69u2rVatWKSQkRKmpqZJyfn/dtH//fnXu3FlNmjTRf/7znyw/fGyxWDR9+nRduXJFffv2tbaPHj1au3fv1owZM1SgQIE7nisA5AkGkEds3brVkGSsWLHCMAzDyMjIMB5//HHj1VdftRm3YMECQ5KxZs2aTNuoWLGi0aBBg0ztV65cMdLT023aEhISDBcXF+Ptt9+2tk2fPt2QZEyaNCnTNjIyMqyPk2R89NFHxrVr14yOHTsabm5uxo8//njbOS5atMiQZLz77rs27e3btzcsFotx6NAha5skw9nZ2aZtx44dhiTj448/znE/p0+fNooVK2ZIMoKCgoy+ffsas2fPNhITEzPNKTAw0GjWrJl1foZhGKmpqUapUqWMJk2aWNtmzJhhSDISEhKsbREREUaBAgVuO++bJkyYYLi5uRnJycmGYRjGgQMHDElGTEyMzbj+/fsbWf14+/PPPw1JxujRozP1NW7c2AgODjauXLliM7+nn37aCAwMzDSP0NBQmzm/9tprhoODg81zlN3rac2aNTavwbS0NMPb29uoVKmScfnyZeu4pUuXGpKMUaNGWdsiIiIMSTavO8MwjKpVqxrVq1fPtK9bNWjQwKhYsaJhGIZRo0YNo1evXoZhGMaFCxcMZ2dn46uvvrLWt2DBAuvjUlNTM21r48aNhiRj5syZ1rac3l8lS5Y0JBnLly/Psi8iIsKm7fPPPzckGV9//bXxyy+/GA4ODsagQYNuO0cAyIs40448Izo6WsWLF1fDhg0l3Thb17FjR82dO/eulg1kxcXFRfny3Xi7pKen6/z583J3d1e5cuX066+/Wsd9++23Klq0qAYOHJhpGxaLxeZ+Wlqann/+eS1dulTLli1T06ZNb1vHsmXL5ODgYD2bedOQIUNkGIZ++OEHm/bQ0FCVKVPGer9y5cry8PDQ77//nuN+ihcvrh07dqhv3766cOGCPvvsM3Xp0kXe3t565513rEtx4uPjdfDgQXXp0kXnz5/XuXPndO7cOV26dEmNGzfW2rVrbZZf/FPR0dEKCwtTwYIFJUmBgYGqXr16piUyd+uvv/7S6tWr1aFDB128eNE6j/Pnz6tZs2Y6ePBgpmUqL730ks0xrVevntLT03X06NG73v/WrVt19uxZ9evXz2bpUVhYmIKCgvT9999neszfz0Df3P/tjuutunTpooULFyotLU3ffPONHBwc1LZt2yzHurm5Wf997do1nT9/XmXLlpWXl5fNe+B2SpUqpWbNmt3R2JdeeknNmjXTwIED9cILL6hMmTJ3fIUhAMhrCO3IE9LT0zV37lw1bNhQCQkJOnTokA4dOqTatWvrzJkzWrVq1T/afkZGhiZPnqzAwEC5uLioaNGiKlasmHbu3KmkpCTruMOHD6tcuXJydLz9Z7zHjRunRYsW6ZtvvrGudb6do0ePys/Pzxpab7q5/ODWwFiiRIlM2yhUqJAuXLhw2335+vpq2rRpOnXqlPbv368pU6aoWLFiGjVqlCIjIyVJBw8elCRFRESoWLFiNrcvv/xSV69etXl+/om9e/dq+/btqlu3rvX4Hjp0SCEhIVq6dKmSk5PveduHDh2SYRgaOXJkpnnc/CDz39dWS5mf20KFCknSHT23t7p53MqVK5epLygoKNNxdXV1ta7P//v+73bfnTp1UlJSkn744QdFR0erVatWmV5bN12+fFmjRo2yfpbi5nsgMTHxro5xqVKl7qrGyMhIpaam6uDBg4qKirL55QEAHiVcPQZ5wurVq3Xq1CnNnTtXc+fOzdQfHR19R2eys/P+++9r5MiRevHFF/XOO++ocOHCypcvnwYNGnTPZ5KbNWum5cuX68MPP1RISMgDuZqIg4NDlu1GFpdJzI7FYtETTzyhJ554QmFhYQoMDFR0dLR69+5tnftHH32U7Rcjubu733XdWfn6668lSa+99ppee+21TP3ffvutevbseU/bvjmPoUOHZnsWuGzZsjb378dze6+y2/fd8vX1VUhIiCZOnKgNGzbkeMWYgQMHasaMGRo0aJDq1KkjT09PWSwWderU6a7eA3cbumNjY3X16lVJ0m+//aY6derc1eMBIK8gtCNPiI6Olre3t/WqIX+3cOFCxcTE6LPPPpObm1umZSp/l13fN998o4YNG1rPMN+UmJiookWLWu+XKVNGmzZt0rVr1+Tk5JRjzU899ZT69u2rVq1a6fnnn1dMTMxtz9CXLFlSK1eu1MWLF23OiO7bt8/a/yCVLl1ahQoV0qlTpyTJuvTGw8NDoaGhD2y/hmFo9uzZatiwofr165ep/5133lF0dLQ1tGd3HLNrL126tCTJycnpvs4jp9fa3908bvv371ejRo1s+vbv3/9Aj2uXLl3Uu3dveXl55Xid/G+++UYRERGaOHGite3KlSuZPiR7p3O+E6dOndLAgQPVtGlTOTs7W3+petCvcwAwI5bH4KF3+fJlLVy4UK1atVL79u0z3QYMGKCLFy9aL91386oTt4aNm31ZtTs4OGQ6g7pgwYJM65zDw8N17tw5ffLJJ5m2kdUZ2NDQUM2dO1fLly/XCy+8cNszli1btlR6enqm7U+ePFkWi0UtWrTI8fF3atOmTVleUm/z5s06f/68dRlH9erVVaZMGU2YMCHLb0n9+yUQ/4kNGzboyJEj6tmzZ5bHuGPHjlqzZo31Sj7ZHeP8+fNn2e7t7a2QkBB9/vnn1l9I7sc8sns93apGjRry9vbWZ599Zj2rLEk//PCD9u7dq7CwsHva/51o3769Ro8erU8//TTH6+Vn9R74+OOPM31eJKf3193q06ePMjIyFBkZqf/+979ydHRUr169cuWvGQBgNpxpx0Nv8eLFunjxop599tks+5966ikVK1ZM0dHR6tixo5588kk5ODjogw8+UFJSklxcXNSoUSN5e3urevXqmjZtmt59912VLVtW3t7eatSokVq1aqW3335bPXv21NNPP63ffvtN0dHR1jO0N3Xv3l0zZ87U4MGDtXnzZtWrV0+XLl3SypUr1a9fPz333HOZ6mvTpo1mzJih7t27y8PDw3rd6qy0bt1aDRs21FtvvaUjR46oSpUq+umnn/Tdd99p0KBBNh86/SdmzZql6OhotW3bVtWrV5ezs7P27t2r6dOny9XVVW+++aakG18S9OWXX6pFixaqWLGievbsqccee0wnTpzQmjVr5OHhoSVLlvzjeqKjo+Xg4JBteH322Wf11ltvae7cuRo8eLCqV68uSXrllVfUrFkzOTg4qFOnTnJzc1OFChU0b948PfHEEypcuLAqVaqkSpUqaerUqXrmmWcUHBysPn36qHTp0jpz5ow2btyoP/74I8tr8t9Odq+nWzk5OemDDz5Qz5491aBBA3Xu3FlnzpzRf/7zHwUEBGS5HOh+8fT0tLlWfXZatWqlWbNmydPTUxUqVNDGjRu1cuVKFSlSxGZcTu+vuzFjxgx9//33ioqK0uOPPy7pxi8J3bp107Rp07L8iwsA5Gn2umwNcL+0bt3acHV1NS5dupTtmB49ehhOTk7GuXPnDMMwjC+++MIoXbq04eDgYHN5utOnTxthYWFGwYIFDUnWy/VduXLFGDJkiOHr62u4ubkZdevWNTZu3Gg0aNAg0yX9UlNTjbfeessoVaqU4eTkZPj4+Bjt27c3Dh8+bBiG7SUf/+7TTz81JBlDhw7Ncb4XL140XnvtNcPPz89wcnIyAgMDjY8++sjm8oOGceOSj/3798/0+KwurXernTt3GsOGDTOqVatmFC5c2HB0dDR8fX2N559/3vj1118zjd++fbvRrl07o0iRIoaLi4tRsmRJo0OHDsaqVausY+71ko9paWlGkSJFjHr16uU4rlSpUkbVqlUNwzCM69evGwMHDjSKFStmWCwWm8s//vzzz0b16tUNZ2fnTJd/PHz4sNG9e3fDx8fHcHJyMh577DGjVatWxjfffJNpHlu2bLHZ/62XcTSM7F9PWY01DMOYN2+eUbVqVcPFxcUoXLiw0bVrV+OPP/6wGZPdczZ69OgsL3N5q79f8jE7WV3y8cKFC0bPnj2NokWLGu7u7kazZs2Mffv2Zfl6yu79VbJkSSMsLCzLff59O8ePHzc8PT2N1q1bZxrXtm1bo0CBAsbvv/9+27kCQF5iMQz+zggAAACYGWvaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHJ5/suVMjIydPLkSRUsWPC+fr02AAB4cAzD0MWLF+Xn56d8+TjHCOT50H7y5En5+/vbuwwAAHAPjh8/bv1WXOBRludDe8GCBSXdeNN7eHjYuRoAAHAnkpOT5e/vb/1/HHjU5fnQfnNJjIeHB6EdAICHDEtbgRtYJAYAAACYHKEdAAAAMDlCOwAAAGByeX5NOwAAwL3KyMhQWlqavctAHuXk5CQHB4c7GktoBwAAyEJaWpoSEhKUkZFh71KQh3l5ecnHx+e2H7omtAMAANzCMAydOnVKDg4O8vf35wuecN8ZhqHU1FSdPXtWkuTr65vjeEI7AADALa5fv67U1FT5+fkpf/789i4HeZSbm5sk6ezZs/L29s5xqQy/NgIAANwiPT1dkuTs7GznSpDX3fyl8Nq1azmOI7QDAABkgy93woN2p68xQjsAAABgcnYN7ePGjVPNmjVVsGBBeXt7q02bNtq/f7/NmJCQEFksFptb37597VQxAAAA7lRsbKwsFosSExPtXcpDz64fRI2Li1P//v1Vs2ZNXb9+XW+++aaaNm2qPXv2qECBAtZxffr00dtvv229zwdCAACAPQS88X2u7u/I+LB7etzGjRv1zDPPqHnz5vr+e9uax4wZo0WLFik+Pt6m3WKxKCYmRm3atLnHau+PP//8U6NGjdL333+vM2fOqFChQqpSpYpGjRqlunXr2rU2e7JraF++fLnN/aioKHl7e2vbtm2qX7++tT1//vzy8fHJ7fIAAAAeSpGRkRo4cKAiIyN18uRJ+fn52bukOxYeHq60tDR99dVXKl26tM6cOaNVq1bp/Pnz9i7Nrky1pj0pKUmSVLhwYZv26OhoFS1aVJUqVdKIESOUmppqj/IAAABMLyUlRfPmzdPLL7+ssLAwRUVFWfuioqI0duxY7dixw7rsOCoqSgEBAZKktm3bymKxWO8fPnxYzz33nIoXLy53d3fVrFlTK1eutNnf1atXNXz4cPn7+8vFxUVly5ZVZGRklrWlpqaqRYsWqlu3bpZLZhITE7Vu3Tp98MEHatiwoUqWLKlatWppxIgRevbZZyVJR44ckcVisflLQWJioiwWi2JjY61tu3fvVqtWreTh4aGCBQuqXr16Onz4sLV/+vTpqlixolxcXOTr66sBAwbYbK93794qVqyYPDw81KhRI+3YscPav2PHDjVs2FAFCxaUh4eHqlevrq1bt0qSjh49qtatW6tQoUIqUKCAKlasqGXLlmV7vO6Uaa7TnpGRoUGDBqlu3bqqVKmStb1Lly4qWbKk/Pz8tHPnTg0fPlz79+/XwoULs9zO1atXdfXqVev95OTkB147AACAWcyfP19BQUEqV66cunXrpkGDBmnEiBGyWCzq2LGjdu3apeXLl1vDt6enp8LCwuTt7a0ZM2aoefPm1uuFp6SkqGXLlnrvvffk4uKimTNnqnXr1tq/f79KlCghSerevbs2btyoKVOmqEqVKkpISNC5c+cy1ZWYmKiwsDC5u7trxYoVWS53dnd3l7u7uxYtWqSnnnpKLi4u9/QcnDhxQvXr11dISIhWr14tDw8PbdiwQdevX5ckTZs2TYMHD9b48ePVokULJSUlacOGDdbHP//883Jzc9MPP/wgT09Pff7552rcuLEOHDigwoULq2vXrqpataqmTZsmBwcHxcfHy8nJSZLUv39/paWlae3atSpQoID27Nkjd3f3e5rH35kmtPfv31+7du3S+vXrbdpfeukl67+Dg4Pl6+urxo0b6/DhwypTpkym7YwbN05jx4594PXaS05r6e513Rvsg2OJe8VrB0BOIiMj1a1bN0lS8+bNlZSUpLi4OIWEhMjNzU3u7u5ydHS0WXp880t+vLy8bNqrVKmiKlWqWO+/8847iomJ0eLFizVgwAAdOHBA8+fP14oVKxQaGipJKl26dKaaTp8+rY4dOyowMFCzZ8/O9vr3jo6OioqKUp8+ffTZZ5+pWrVqatCggTp16qTKlSvf8XMwdepUeXp6au7cudYw/cQTT1j73333XQ0ZMkSvvvqqta1mzZqSpPXr12vz5s06e/as9ZeGCRMmaNGiRfrmm2/00ksv6dixYxo2bJiCgoIkSYGBgdbtHDt2TOHh4QoODs72+bgXplgeM2DAAC1dulRr1qzR448/nuPY2rVrS5IOHTqUZf+IESOUlJRkvR0/fvy+1wsAAGBG+/fv1+bNm9W5c2dJN0Jwx44ds12ucjspKSkaOnSoypcvLy8vL7m7u2vv3r06duyYJCk+Pl4ODg5q0KBBjttp0qSJypYtq3nz5t32C6vCw8N18uRJLV68WM2bN1dsbKyqVatms8znduLj41WvXj1rYP+7s2fP6uTJk2rcuHGWj92xY4dSUlJUpEgR65l/d3d3JSQkWJfXDB48WL1791ZoaKjGjx9vs+zmlVde0bvvvqu6detq9OjR2rlz5x3XnRO7hnbDMDRgwADFxMRo9erVKlWq1G0fc3P9kq+vb5b9Li4u8vDwsLkBAAA8CiIjI3X9+nX5+fnJ0dFRjo6OmjZtmr799lvrZwfvxtChQxUTE6P3339f69atU3x8vIKDg5WWlibpf2fobycsLExr167Vnj177mi8q6urmjRpopEjR+rnn39Wjx49NHr0aElSvnw34qthGNbxt36baE513a7mlJQU+fr6Kj4+3ua2f/9+DRs2TNKNK/Ds3r1bYWFhWr16tSpUqKCYmBhJUu/evfX777/rhRde0G+//aYaNWro448/vqN558Suob1///76+uuvNXv2bBUsWFCnT5/W6dOndfnyZUk3PvzwzjvvaNu2bTpy5IgWL16s7t27q379+nf1JxIAAIC87vr165o5c6YmTpxoEzZ37NghPz8/zZkzR5Lk7Oys9PT0TI93cnLK1L5hwwb16NFDbdu2VXBwsHx8fHTkyBFrf3BwsDIyMhQXF5djbePHj1dERIQaN258x8H97ypUqKBLly5JkooVKyZJOnXqlLX/1stXVq5cWevWrcsU5iWpYMGCCggI0KpVq7LcV7Vq1XT69Gk5OjqqbNmyNreiRYtaxz3xxBN67bXX9NNPP6ldu3aaMWOGtc/f3199+/bVwoULNWTIEH3xxRd3Pedb2TW0T5s2TUlJSQoJCZGvr6/1Nm/ePEk3XlQrV65U06ZNFRQUpCFDhig8PFxLliyxZ9kAAACms3TpUl24cEG9evVSpUqVbG7h4eHWJTIBAQFKSEhQfHy8zp07Z72Ax80ge/r0aV24cEHSjbXaCxcutIb/Ll26KCMjw7rPgIAARURE6MUXX9SiRYuUkJCg2NhYzZ8/P1N9EyZMUNeuXdWoUSPt27cvyzmcP39ejRo10tdff62dO3cqISFBCxYs0IcffqjnnntO0o0z5U899ZTGjx+vvXv3Ki4uTv/6179stjNgwAAlJyerU6dO2rp1qw4ePKhZs2ZZv8RzzJgxmjhxoqZMmaKDBw/q119/tZ4NDw0NVZ06ddSmTRv99NNPOnLkiH7++We99dZb2rp1qy5fvqwBAwYoNjZWR48e1YYNG7RlyxaVL19ekjRo0CD9+OOPSkhI0K+//qo1a9ZY+/4Ju34Q9e9/1siKv7//bX9zAwAAwI2lMaGhofL09MzUFx4erg8//FA7d+5UeHi4Fi5cqIYNGyoxMVEzZsxQjx49NHHiRA0ePFhffPGFHnvsMR05ckSTJk3Siy++qKefflpFixbV8OHDM12Zb9q0aXrzzTfVr18/nT9/XiVKlNCbb76ZZY2TJ09Wenq6GjVqpNjYWJsPh0o3rh5Tu3ZtTZ48WYcPH9a1a9fk7++vPn362Gxz+vTp6tWrl6pXr65y5crpww8/VNOmTa39RYoU0erVqzVs2DA1aNBADg4OevLJJ61fzhQREaErV65o8uTJGjp0qIoWLar27dtLuvElU8uWLdNbb72lnj176s8//5SPj4/q16+v4sWLy8HBQefPn1f37t115swZFS1aVO3atbNeCCU9PV39+/fXH3/8IQ8PDzVv3lyTJ0++hyNqy2LcLjk/5JKTk+Xp6amkpKQ8sb6dq0bkHRxL3CteO3gU2Pv/7ytXrighIUGlSpWSq6trru8fj447fa2Z4uoxAAAAALJHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJ2fXLlfBwyO1rQnMN6ryDYwkAwP3BmXYAAADA5AjtAAAAeCBiY2NlsViUmJiYK/uLioqSl5eX9f6YMWP05JNP/qNtHjlyRBaLRfHx8f9oO/8Uy2MAAADu1BjPXN5f0j09bOPGjXrmmWfUvHlzff+97VLFMWPGaNGiRZlCqMViUUxMjNq0aXOPxd4fPXr00FdffZWp/eDBgypbtmyOj+3YsaNatmz5oEqzK860AwAA5DGRkZEaOHCg1q5dq5MnT9q7nLvWvHlznTp1yuZWqlSp2z7Ozc1N3t7euVBh7iO0AwAA5CEpKSmaN2+eXn75ZYWFhSkqKsraFxUVpbFjx2rHjh2yWCyyWCyKiopSQECAJKlt27ayWCzW+4cPH9Zzzz2n4sWLy93dXTVr1tTKlStt9nf16lUNHz5c/v7+cnFxUdmyZRUZGZllbampqWrRooXq1q2b45IZFxcX+fj42NwcHBw0adIkBQcHq0CBAvL391e/fv2UkpJiM7+/L4/Jypdffqny5cvL1dVVQUFB+vTTT236N2/erKpVq8rV1VU1atTQ9u3bc9xebiG0AwAA5CHz589XUFCQypUrp27dumn69OkyDEPSjeUjQ4YMUcWKFa1nsDt27KgtW7ZIkmbMmKFTp05Z76ekpKhly5ZatWqVtm/frubNm6t169Y6duyYdX/du3fXnDlzNGXKFO3du1eff/653N3dM9WVmJioJk2aKCMjQytWrLhtuM5Kvnz5NGXKFO3evVtfffWVVq9erddff/2OHx8dHa1Ro0bpvffe0969e/X+++9r5MiR1uU4KSkpatWqlSpUqKBt27ZpzJgxGjp06F3X+SCwph0AACAPiYyMVLdu3STdWGaSlJSkuLg4hYSEyM3NTe7u7nJ0dJSPj4/1MW5ubpIkLy8vm/YqVaqoSpUq1vvvvPOOYmJitHjxYg0YMEAHDhzQ/PnztWLFCoWGhkqSSpcunamm06dPq2PHjgoMDNTs2bPl7Oyc4xyWLl1qE/xbtGihBQsWaNCgQda2gIAAvfvuu+rbt2+ms+XZGT16tCZOnKh27dpJkkqVKqU9e/bo888/V0REhGbPnq2MjAxFRkbK1dVVFStW1B9//KGXX375jrb/IBHaAQAA8oj9+/dr8+bNiomJkSQ5OjqqY8eOioyMVEhIyF1vLyUlRWPGjNH333+vU6dO6fr167p8+bL1THt8fLwcHBzUoEGDHLfTpEkT1apVS/PmzZODg8Nt99uwYUNNmzbNer9AgQKSpJUrV2rcuHHat2+fkpOTdf36dV25ckWpqanKnz9/jtu8dOmSDh8+rF69eqlPnz7W9uvXr8vT88YHjPfu3avKlSvL1dXV2l+nTp3b1psbCO0AAAB5RGRkpK5fvy4/Pz9rm2EYcnFx0SeffGINp3dq6NChWrFihSZMmKCyZcvKzc1N7du3V1pamqT/naG/nbCwMH377bfas2ePgoODbzu+QIECma4Uc+TIEbVq1Uovv/yy3nvvPRUuXFjr169Xr169lJaWdtvQfnPt+xdffKHatWvb9N3JLxL2RmgHAADIA65fv66ZM2dq4sSJatq0qU1fmzZtNGfOHPXt21fOzs5KT0/P9HgnJ6dM7Rs2bFCPHj3Utm1bSTeC75EjR6z9wcHBysjIUFxcnHV5TFbGjx8vd3d3NW7cWLGxsapQocJdz2/btm3KyMjQxIkTlS/fjY9lzp8//44fX7x4cfn5+en3339X165dsxxTvnx5zZo1S1euXLGebf/ll1/uutYHgQ+iAgAA5AFLly7VhQsX1KtXL1WqVMnmFh4ebr2iS0BAgBISEhQfH69z587p6tWr1vZVq1bp9OnTunDhgiQpMDBQCxcuVHx8vHbs2KEuXbooIyPDus+AgABFREToxRdf1KJFi5SQkKDY2Ngsw/SECRPUtWtXNWrUSPv27bvr+ZUtW1bXrl3Txx9/rN9//12zZs3SZ599dlfbGDt2rMaNG6cpU6bowIED+u233zRjxgxNmjRJktSlSxdZLBb16dNHe/bs0bJlyzRhwoS7rvVBILQDAADkAZGRkQoNDc1yCUx4eLi2bt2qnTt3Kjw8XM2bN1fDhg1VrFgxzZkzR5I0ceJErVixQv7+/qpataokadKkSSpUqJCefvpptW7dWs2aNVO1atVstj1t2jS1b99e/fr1U1BQkPr06aNLly5lWePkyZPVoUMHNWrUSAcOHLir+VWpUkWTJk3SBx98oEqVKik6Olrjxo27q2307t1bX375pWbMmKHg4GA1aNBAUVFR1mvAu7u7a8mSJfrtt99UtWpVvfXWW/rggw/uah8PisW4eQ2gPCo5OVmenp5KSkqSh4eHvcv5xwLe+D7bviPjw/LEPu0xR3t4FOb5KMzRHnhe8Siw9//fV65cUUJCgkqVKmXzoUTgfrvT1xpr2gE9OiEou3nmpTnaw6PwvD4K75FHYY4AHl4sjwEAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAMADERsbK4vFosTExPu6XYvFokWLFmXbHxAQoH//+9/3dZ/3KiQkRIMGDfrH2+EbUQEAAO5Q8FfBubq/3yJ+u6fHbdy4Uc8884yaN2+u77+3/bbfMWPGaNGiRYqPj7dpt1gsiomJUZs2be6x2vvjzz//1KhRo/T999/rzJkzKlSokKpUqaJRo0apbt26d7SNLVu2qECBAg+40txFaAcAAMhjIiMjNXDgQEVGRurkyZPy8/Ozd0l3LDw8XGlpafrqq69UunRpnTlzRqtWrdL58+fveBvFihV7gBXaB8tjAAAA8pCUlBTNmzdPL7/8ssLCwhQVFWXti4qK0tixY7Vjxw5ZLBZZLBZFRUUpICBAktS2bVtZLBbr/cOHD+u5555T8eLF5e7urpo1a2rlypU2+7t69aqGDx8uf39/ubi4qGzZsoqMjMyyttTUVLVo0UJ169bNcslMYmKi1q1bpw8++EANGzZUyZIlVatWLY0YMULPPvtstnMePXq0fH19tXPnTkmZl8dYLBZNmzZNLVq0kJubm0qXLq1vvvnG2n/kyBFZLBbNnTtXTz/9tFxdXVWpUiXFxcXZ7GfXrl1q0aKF3N3dVbx4cb3wwgs6d+6ctf/SpUvq3r273N3d5evrq4kTJ2Zb890itAMAAOQh8+fPV1BQkMqVK6du3bpp+vTpMgxDktSxY0cNGTJEFStW1KlTp3Tq1Cl17NhRW7ZskSTNmDFDp06dst5PSUlRy5YttWrVKm3fvl3NmzdX69atdezYMev+unfvrjlz5mjKlCnau3evPv/8c7m7u2eqKzExUU2aNFFGRoZWrFghLy+vTGPc3d3l7u6uRYsW6erVq7edq2EYGjhwoGbOnKl169apcuXK2Y4dOXKkwsPDtWPHDnXt2lWdOnXS3r17bcYMGzZMQ4YM0fbt21WnTh21bt3aeoY/MTFRjRo1UtWqVbV161YtX75cZ86cUYcOHWweHxcXp++++04//fSTYmNj9euvv952HneC5TH/QMAb32fbd2R8WC5WAgB5Dz9jgXsTGRmpbt26SZKaN2+upKQkxcXFKSQkRG5ubnJ3d5ejo6N8fHysj3Fzc5MkeXl52bRXqVJFVapUsd5/5513FBMTo8WLF2vAgAE6cOCA5s+frxUrVig0NFSSVLp06Uw1nT59Wh07dlRgYKBmz54tZ2fnLGt3dHRUVFSU+vTpo88++0zVqlVTgwYN1KlTp0yB/Pr16+rWrZu2b9+u9evX67HHHsvxeXn++efVu3dv6zxWrFihjz/+WJ9++ql1zIABAxQeHi5JmjZtmpYvX67IyEi9/vrr+uSTT1S1alW9//771vHTp0+Xv7+/Dhw4ID8/P0VGRurrr79W48aNJUlfffWVHn/88RzrulOcaQcAAMgj9u/fr82bN6tz586SboTgjh07Zrtc5XZSUlI0dOhQlS9fXl5eXnJ3d9fevXutZ9rj4+Pl4OCgBg0a5LidJk2aqGzZspo3b162gf2m8PBwnTx5UosXL1bz5s0VGxuratWq2SzzkaTXXntNmzZt0tq1a28b2CWpTp06me7feqb972McHR1Vo0YN65gdO3ZozZo11r8GuLu7KygoSNKNZUSHDx9WWlqaateubd1G4cKFVa5cudvWdicI7QAAAHlEZGSkrl+/Lj8/Pzk6OsrR0VHTpk3Tt99+q6SkpLve3tChQxUTE6P3339f69atU3x8vIKDg5WWlibpf2fobycsLExr167Vnj177mi8q6urmjRpopEjR+rnn39Wjx49NHr0aJsxTZo00YkTJ/Tjjz/e3aTuUUpKilq3bq34+Hib28GDB1W/fv0Hvn9COwAAQB5w/fp1zZw5UxMnTrQJlTt27JCfn5/mzJkjSXJ2dlZ6enqmxzs5OWVq37Bhg3r06KG2bdsqODhYPj4+OnLkiLU/ODhYGRkZmT6weavx48crIiJCjRs3vuPg/ncVKlTQpUuXbNqeffZZzZ49W71799bcuXNvu41ffvkl0/3y5ctnO+b69evatm2bdUy1atW0e/duBQQEqGzZsja3AgUKqEyZMnJyctKmTZus27hw4YIOHDhw1/PNCqEdAAAgD1i6dKkuXLigXr16qVKlSja38PBw6xKZgIAAJSQkKD4+XufOnbN+4DMgIECrVq3S6dOndeHCBUlSYGCgFi5caA3/Xbp0UUZGhnWfAQEBioiI0IsvvqhFixYpISFBsbGxmj9/fqb6JkyYoK5du6pRo0bat29flnM4f/68GjVqpK+//lo7d+5UQkKCFixYoA8//FDPPfdcpvFt27bVrFmz1LNnT5urwWRlwYIFmj59ug4cOKDRo0dr8+bNGjBggM2YqVOnKiYmRvv27VP//v114cIFvfjii5Kk/v3766+//lLnzp21ZcsWHT58WD/++KN69uyp9PR0ubu7q1evXho2bJhWr16tXbt2qUePHsqX7/7EbUI7AABAHhAZGanQ0FB5enpm6gsPD9fWrVu1c+dOhYeHq3nz5mrYsKGKFStmPQM/ceJErVixQv7+/qpataokadKkSSpUqJCefvpptW7dWs2aNVO1atVstj1t2jS1b99e/fr1U1BQkPr06ZPprPhNkydPVocOHdSoUaMsz0C7u7urdu3amjx5surXr69KlSpp5MiR6tOnjz755JMst9m+fXt99dVXeuGFF7Rw4cJsn5+xY8dq7ty5qly5smbOnKk5c+aoQoUKNmPGjx+v8ePHq0qVKlq/fr0WL16sokWLSpL8/Py0YcMGpaenq2nTpgoODtagQYPk5eVlDeYfffSR6tWrp9atWys0NFTPPPOMqlevnm1Nd4OrxwAAANyhe/2G0tywZMmSbPtq1aplveyjpCzPSrdu3VqtW7e2aQsICNDq1att2vr3729z39XVVZMmTdKkSZMybTMkJMRmv5I0ZcoUTZkyJcs6XVxcNG7cOI0bNy7buUjKtM0OHTrYXHrx70t4bvLz89NPP/2U43bLly9vs7zlVjf/8pAdd3d3zZo1S7NmzbK2DRs2LMd93inOtAMAAAAmR2gHAAAATI7lMQAAAMjTbl1Oc6uAgIDbjrE3zrQDAAAAJkdoBwAAyIbZz77i4XenrzFCOwAAwC0cHBwkyfrNn8CDkpqaKunGl1vlhDXtAAAAt3B0dFT+/Pn1559/ysnJ6b59QQ5wk2EYSk1N1dmzZ+Xl5WX9RTE7hHYAeUrAG99n23dkfFguVgLcnj1er7xH7ozFYpGvr68SEhJ09OhRe5eDPMzLy0s+Pj63HUdoBwAAyIKzs7MCAwNZIoMHxsnJ6bZn2G8itAMAAGQjX758cnV1tXcZAB9EBQAAAMyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTs2toHzdunGrWrKmCBQvK29tbbdq00f79+23GXLlyRf3791eRIkXk7u6u8PBwnTlzxk4VAwAAALnPrqE9Li5O/fv31y+//KIVK1bo2rVratq0qS5dumQd89prr2nJkiVasGCB4uLidPLkSbVr186OVQMAAAC5y67fiLp8+XKb+1FRUfL29ta2bdtUv359JSUlKTIyUrNnz1ajRo0kSTNmzFD58uX1yy+/6KmnnrJH2QAAAECuMtWa9qSkJElS4cKFJUnbtm3TtWvXFBoaah0TFBSkEiVKaOPGjVlu4+rVq0pOTra5AQAAAA8z04T2jIwMDRo0SHXr1lWlSpUkSadPn5azs7O8vLxsxhYvXlynT5/Ocjvjxo2Tp6en9ebv7/+gSwcAAAAeKNOE9v79+2vXrl2aO3fuP9rOiBEjlJSUZL0dP378PlUIAAAA2Idd17TfNGDAAC1dulRr167V448/bm338fFRWlqaEhMTbc62nzlzRj4+Plluy8XFRS4uLg+6ZAAAACDX2PVMu2EYGjBggGJiYrR69WqVKlXKpr969epycnLSqlWrrG379+/XsWPHVKdOndwuFwAAALALu55p79+/v2bPnq3vvvtOBQsWtK5T9/T0lJubmzw9PdWrVy8NHjxYhQsXloeHhwYOHKg6depw5RgAAAA8Muwa2qdNmyZJCgkJsWmfMWOGevToIUmaPHmy8uXLp/DwcF29elXNmjXTp59+msuVAgAAAPZj19BuGMZtx7i6umrq1KmaOnVqLlQEAAAAmI9prh4DAAAAIGuEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAydk1tK9du1atW7eWn5+fLBaLFi1aZNPfo0cPWSwWm1vz5s3tUywAAABgJ3YN7ZcuXVKVKlU0derUbMc0b95cp06dst7mzJmTixUCAAAA9udoz523aNFCLVq0yHGMi4uLfHx8cqkiAAAAwHxMv6Y9NjZW3t7eKleunF5++WWdP3/e3iUBAAAAucquZ9pvp3nz5mrXrp1KlSqlw4cP680331SLFi20ceNGOTg4ZPmYq1ev6urVq9b7ycnJuVUuAAAA8ECYOrR36tTJ+u/g4GBVrlxZZcqUUWxsrBo3bpzlY8aNG6exY8fmVokAAADAA2f65TF/V7p0aRUtWlSHDh3KdsyIESOUlJRkvR0/fjwXKwQAAADuP1Ofab/VH3/8ofPnz8vX1zfbMS4uLnJxccnFqgAAAIAHy66hPSUlxeaseUJCguLj41W4cGEVLlxYY8eOVXh4uHx8fHT48GG9/vrrKlu2rJo1a2bHqgEAAIDcZdfQvnXrVjVs2NB6f/DgwZKkiIgITZs2TTt37tRXX32lxMRE+fn5qWnTpnrnnXc4kw4AAIBHil1De0hIiAzDyLb/xx9/zMVqAAAAAHN6qD6ICgAAADyKCO0AAACAyd1TaC9dunSW30yamJio0qVL/+OiAAAAAPzPPYX2I0eOKD09PVP71atXdeLEiX9cFAAAAID/uasPoi5evNj67x9//FGenp7W++np6Vq1apUCAgLuW3EAAAAA7jK0t2nTRpJksVgUERFh0+fk5KSAgABNnDjxvhUHAAAA4C5De0ZGhiSpVKlS2rJli4oWLfpAigIAAADwP/d0nfaEhIT7XQcAAACAbNzzlyutWrVKq1at0tmzZ61n4G+aPn36Py4MAAAAwA33FNrHjh2rt99+WzVq1JCvr68sFsv9rgsAAADA/3dPof2zzz5TVFSUXnjhhftdDwAAAIBb3NN12tPS0vT000/f71oAAAAAZOGeQnvv3r01e/bs+10LAAAAgCzc0/KYK1eu6L///a9WrlypypUry8nJyaZ/0qRJ96U4AAAAAPcY2nfu3Kknn3xSkrRr1y6bPj6UCgAAANxf9xTa16xZc7/rAAAAAJCNe1rTDgAAACD33NOZ9oYNG+a4DGb16tX3XBAAAAAAW/cU2m+uZ7/p2rVrio+P165duxQREXE/6gIAAADw/91TaJ88eXKW7WPGjFFKSso/KggAAACArfu6pr1bt26aPn36/dwkAAAA8Mi7r6F948aNcnV1vZ+bBAAAAB5597Q8pl27djb3DcPQqVOntHXrVo0cOfK+FAYAAADghnsK7Z6enjb38+XLp3Llyuntt99W06ZN70thAAAAAG64p9A+Y8aM+10HAAAAgGzcU2i/adu2bdq7d68kqWLFiqpatep9KQoAAADA/9xTaD979qw6deqk2NhYeXl5SZISExPVsGFDzZ07V8WKFbufNQIAAACPtHu6eszAgQN18eJF7d69W3/99Zf++usv7dq1S8nJyXrllVfud40AAADAI+2ezrQvX75cK1euVPny5a1tFSpU0NSpU/kgKgAAAHCf3dOZ9oyMDDk5OWVqd3JyUkZGxj8uCgAAAMD/3FNob9SokV599VWdPHnS2nbixAm99tpraty48X0rDgAAAMA9hvZPPvlEycnJCggIUJkyZVSmTBmVKlVKycnJ+vjjj+93jQAAAMAj7Z7WtPv7++vXX3/VypUrtW/fPklS+fLlFRoael+LAwAAAHCXZ9pXr16tChUqKDk5WRaLRU2aNNHAgQM1cOBA1axZUxUrVtS6deseVK0AAADAI+muQvu///1v9enTRx4eHpn6PD099X//93+aNGnSfSsOAAAAwF2G9h07dqh58+bZ9jdt2lTbtm37x0UBAAAA+J+7Cu1nzpzJ8lKPNzk6OurPP//8x0UBAAAA+J+7Cu2PPfaYdu3alW3/zp075evr+4+LAgAAAPA/dxXaW7ZsqZEjR+rKlSuZ+i5fvqzRo0erVatW9604AAAAAHd5ycd//etfWrhwoZ544gkNGDBA5cqVkyTt27dPU6dOVXp6ut56660HUigAAADwqLqr0F68eHH9/PPPevnllzVixAgZhiFJslgsatasmaZOnarixYs/kEIBAACAR9Vdf7lSyZIltWzZMl24cEGHDh2SYRgKDAxUoUKFHkR9AAAAwCPvnr4RVZIKFSqkmjVr3s9aAAAAAGThrj6ICgAAACD3EdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJ2TW0r127Vq1bt5afn58sFosWLVpk028YhkaNGiVfX1+5ubkpNDRUBw8etE+xAAAAgJ3YNbRfunRJVapU0dSpU7Ps//DDDzVlyhR99tln2rRpkwoUKKBmzZrpypUruVwpAAAAYD+O9tx5ixYt1KJFiyz7DMPQv//9b/3rX//Sc889J0maOXOmihcvrkWLFqlTp065WSoAAABgN6Zd056QkKDTp08rNDTU2ubp6anatWtr48aN2T7u6tWrSk5OtrkBAAAADzPThvbTp09LkooXL27TXrx4cWtfVsaNGydPT0/rzd/f/4HWCQAAADxopg3t92rEiBFKSkqy3o4fP27vkgAAAIB/xLSh3cfHR5J05swZm/YzZ85Y+7Li4uIiDw8PmxsAAADwMDNtaC9VqpR8fHy0atUqa1tycrI2bdqkOnXq2LEyAAAAIHfZ9eoxKSkpOnTokPV+QkKC4uPjVbhwYZUoUUKDBg3Su+++q8DAQJUqVUojR46Un5+f2rRpY7+iAQAAgFxm19C+detWNWzY0Hp/8ODBkqSIiAhFRUXp9ddf16VLl/TSSy8pMTFRzzzzjJYvXy5XV1d7lQwAAADkOruG9pCQEBmGkW2/xWLR22+/rbfffjsXqwIAAADMxbRr2gEAAADcQGgHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAydn1Ou0AkKvGeObQl5R7dQB3wh6v1+z2yfsDsDvOtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKO9i4AAIC7NsYzm/ak3K0DAHIJZ9oBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5R3sXkGeN8cyhLyn36sA/9ygcy0dhjvbwqDyvj8I8H4U5AjA1zrQDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDAAAAJkdoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByjvYuAA+5MZ459CU9/PvDg8OxBADgjnGmHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABMjtAOAAAAmByhHQAAADA5U4f2MWPGyGKx2NyCgoLsXRYAAACQqxztXcDtVKxYUStXrrTed3Q0fckAAADAfWX6BOzo6CgfHx97lwEAAADYjamXx0jSwYMH5efnp9KlS6tr1646duxYjuOvXr2q5ORkmxsAAADwMDP1mfbatWsrKipK5cqV06lTpzR27FjVq1dPu3btUsGCBbN8zLhx4zR27NhcrtQkxnjm0JeUe3Xgn+NY4l7x2gGAPMnUZ9pbtGih559/XpUrV1azZs20bNkyJSYmav78+dk+ZsSIEUpKSrLejh8/nosVAwAAAPefqc+038rLy0tPPPGEDh06lO0YFxcXubi45GJVAAAAwINl6jPtt0pJSdHhw4fl6+tr71IAAACAXGPq0D506FDFxcXpyJEj+vnnn9W2bVs5ODioc+fO9i4NAAAAyDWmXh7zxx9/qHPnzjp//ryKFSumZ555Rr/88ouKFStm79IAAACAXGPq0D537lx7lwAAAADYnamXxwAAAAAgtAMAAACmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAmR2gHAAAATI7QDgAAAJgcoR0AAAAwOUI7AAAAYHKEdgAAAMDkCO0AAACAyRHaAQAAAJMjtAMAAAAm52jvAh5FwV8FZ9v3W8RvuVgJAAAAHgacaQcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACb3UIT2qVOnKiAgQK6urqpdu7Y2b95s75IAAACAXGP60D5v3jwNHjxYo0eP1q+//qoqVaqoWbNmOnv2rL1LAwAAAHKF6UP7pEmT1KdPH/Xs2VMVKlTQZ599pvz582v69On2Lg0AAADIFaYO7Wlpadq2bZtCQ0Otbfny5VNoaKg2btxox8oAAACA3ONo7wJycu7cOaWnp6t48eI27cWLF9e+ffuyfMzVq1d19epV6/2kpCRJUnJy8n2vL+NqarZ9yRYj2770y+nZP+42dd7rPvUP5p/b+3wU5mi2fT4Kc7THPh+FOZpun4/CHO2xzwfwf+jt3Pz/0DByeB6AR4jFMPG74eTJk3rsscf0888/q06dOtb2119/XXFxcdq0aVOmx4wZM0Zjx47NzTIBAMADcvz4cT3++OP2LgOwO1OfaS9atKgcHBx05swZm/YzZ87Ix8cny8eMGDFCgwcPtt7PyMjQ0aNH9eSTT+r48ePy8PB4oDXbS3Jysvz9/ZljHvAozJM55h2PwjwfhTlK5punYRi6ePGi/Pz87F0KYAqmDu3Ozs6qXr26Vq1apTZt2ki6EcJXrVqlAQMGZPkYFxcXubi42LTly3dj6b6Hh4cpfhA9SMwx73gU5skc845HYZ6Pwhwlc83T09PT3iUApmHq0C5JgwcPVkREhGrUqKFatWrp3//+ty5duqSePXvauzQAAAAgV5g+tHfs2FF//vmnRo0apdOnT+vJJ5/U8uXLM304FQAAAMirTB/aJWnAgAHZLoe5Ey4uLho9enSmZTN5CXPMOx6FeTLHvONRmOejMEfp0Zkn8LAy9dVjAAAAAJj8y5UAAAAAENoBAAAA0yO0AwAAACZHaAcAAABMLs+H9qlTpyogIECurq6qXbu2Nm/ebO+S7tjatWvVunVr+fn5yWKxaNGiRTb9hmFo1KhR8vX1lZubm0JDQ3Xw4EGbMX/99Ze6du0qDw8PeXl5qVevXkpJScnFWeRs3LhxqlmzpgoWLChvb2+1adNG+/fvtxlz5coV9e/fX0WKFJG7u7vCw8MzfUvusWPHFBYWpvz588vb21vDhg3T9evXc3MqOZo2bZoqV65s/dKSOnXq6IcffrD254U53mr8+PGyWCwaNGiQte1hn+eYMWNksVhsbkFBQdb+h31+f3fixAl169ZNRYoUkZubm4KDg7V161Zr/8P+8ycgICDTsbRYLOrfv7+kvHEs09PTNXLkSJUqVUpubm4qU6aM3nnnHf39+hMP+3EEHilGHjZ37lzD2dnZmD59urF7926jT58+hpeXl3HmzBl7l3ZHli1bZrz11lvGwoULDUlGTEyMTf/48eMNT09PY9GiRcaOHTuMZ5991ihVqpRx+fJl65jmzZsbVapUMX755Rdj3bp1RtmyZY3OnTvn8kyy16xZM2PGjBnGrl27jPj4eKNly5ZGiRIljJSUFOuYvn37Gv7+/saqVauMrVu3Gk899ZTx9NNPW/uvX79uVKpUyQgNDTW2b99uLFu2zChatKgxYsQIe0wpS4sXLza+//5748CBA8b+/fuNN99803BycjJ27dplGEbemOPfbd682QgICDAqV65svPrqq9b2h32eo0ePNipWrGicOnXKevvzzz+t/Q/7/G7666+/jJIlSxo9evQwNm3aZPz+++/Gjz/+aBw6dMg65mH/+XP27Fmb47hixQpDkrFmzRrDMPLGsXzvvfeMIkWKGEuXLjUSEhKMBQsWGO7u7sZ//vMf65iH/TgCj5I8Hdpr1apl9O/f33o/PT3d8PPzM8aNG2fHqu7NraE9IyPD8PHxMT766CNrW2JiouHi4mLMmTPHMAzD2LNnjyHJ2LJli3XMDz/8YFgsFuPEiRO5VvvdOHv2rCHJiIuLMwzjxpycnJyMBQsWWMfs3bvXkGRs3LjRMIwbv9zky5fPOH36tHXMtGnTDA8PD+Pq1au5O4G7UKhQIePLL7/Mc3O8ePGiERgYaKxYscJo0KCBNbTnhXmOHj3aqFKlSpZ9eWF+Nw0fPtx45plnsu3Piz9/Xn31VaNMmTJGRkZGnjmWYWFhxosvvmjT1q5dO6Nr166GYeTN4wjkZXl2eUxaWpq2bdum0NBQa1u+fPkUGhqqjRs32rGy+yMhIUGnT5+2mZ+np6dq165tnd/GjRvl5eWlGjVqWMeEhoYqX7582rRpU67XfCeSkpIkSYULF5Ykbdu2TdeuXbOZZ1BQkEqUKGEzz+DgYJtvyW3WrJmSk5O1e/fuXKz+zqSnp2vu3Lm6dOmS6tSpk+fm2L9/f4WFhdnMR8o7x/LgwYPy8/NT6dKl1bVrVx07dkxS3pmfJC1evFg1atTQ888/L29vb1WtWlVffPGFtT+v/fxJS0vT119/rRdffFEWiyXPHMunn35aq1at0oEDByRJO3bs0Pr169WiRQtJee84AnndQ/GNqPfi3LlzSk9Pt/mBKknFixfXvn377FTV/XP69GlJynJ+N/tOnz4tb29vm35HR0cVLlzYOsZMMjIyNGjQINWtW1eVKlWSdGMOzs7O8vLyshl76zyzeh5u9pnFb7/9pjp16ujKlStyd3dXTEyMKlSooPj4+Dwzx7lz5+rXX3/Vli1bMvXlhWNZu3ZtRUVFqVy5cjp16pTGjh2revXqadeuXXlifjf9/vvvmjZtmgYPHqw333xTW7Zs0SuvvCJnZ2dFRETkuZ8/ixYtUmJionr06CEpb7xWJemNN95QcnKygoKC5ODgoPT0dL333nvq2rWrpLz5/wiQl+XZ0I6HT//+/bVr1y6tX7/e3qU8EOXKlVN8fLySkpL0zTffKCIiQnFxcfYu6745fvy4Xn31Va1YsUKurq72LueBuHmGUpIqV66s2rVrq2TJkpo/f77c3NzsWNn9lZGRoRo1auj999+XJFWtWlW7du3SZ599poiICDtXd/9FRkaqRYsW8vPzs3cp99X8+fMVHR2t2bNnq2LFioqPj9egQYPk5+eXJ48jkNfl2eUxRYsWlYODQ6ZP+585c0Y+Pj52qur+uTmHnObn4+Ojs2fP2vRfv35df/31l+megwEDBmjp0qVas2aNHn/8cWu7j4+P0tLSlJiYaDP+1nlm9Tzc7DMLZ2dnlS1bVtWrV9e4ceNUpUoV/ec//8kzc9y2bZvOnj2ratWqydHRUY6OjoqLi9OUKVPk6Oio4sWL54l5/p2Xl5eeeOIJHTp0KM8cR0ny9fVVhQoVbNrKly9vXQqUl37+HD16VCtXrlTv3r2tbXnlWA4bNkxvvPGGOnXqpODgYL3wwgt67bXXNG7cOEl56zgCj4I8G9qdnZ1VvXp1rVq1ytqWkZGhVatWqU6dOnas7P4oVaqUfHx8bOaXnJysTZs2WedXp04dJSYmatu2bdYxq1evVkZGhmrXrp3rNWfFMAwNGDBAMTExWr16tUqVKmXTX716dTk5OdnMc//+/Tp27JjNPH/77Teb/1hWrFghDw+PTMHDTDIyMnT16tU8M8fGjRvrt99+U3x8vPVWo0YNde3a1frvvDDPv0tJSdHhw4fl6+ubZ46jJNWtWzfTpVcPHDigkiVLSso7P38kacaMGfL29lZYWJi1La8cy9TUVOXLZ/vfvIODgzIyMiTlreMIPBLs/UnYB2nu3LmGi4uLERUVZezZs8d46aWXDC8vL5tP+5vZxYsXje3btxvbt283JBmTJk0ytm/fbhw9etQwjBuX6vLy8jK+++47Y+fOncZzzz2X5aW6qlatamzatMlYv369ERgYaKpLdb388suGp6enERsba3P5tdTUVOuYvn37GiVKlDBWr15tbN261ahTp45Rp04da//NS681bdrUiI+PN5YvX24UK1bMVJdee+ONN4y4uDgjISHB2Llzp/HGG28YFovF+OmnnwzDyBtzzMrfrx5jGA//PIcMGWLExsYaCQkJxoYNG4zQ0FCjaNGixtmzZw3DePjnd9PmzZsNR0dH47333jMOHjxoREdHG/nz5ze+/vpr65i88PMnPT3dKFGihDF8+PBMfXnhWEZERBiPPfaY9ZKPCxcuNIoWLWq8/vrr1jF54TgCj4o8HdoNwzA+/vhjo0SJEoazs7NRq1Yt45dffrF3SXdszZo1hqRMt4iICMMwblyua+TIkUbx4sUNFxcXo3Hjxsb+/ftttnH+/Hmjc+fOhru7u+Hh4WH07NnTuHjxoh1mk7Ws5ifJmDFjhnXM5cuXjX79+hmFChUy8ufPb7Rt29Y4deqUzXaOHDlitGjRwnBzczOKFi1qDBkyxLh27VouzyZ7L774olGyZEnD2dnZKFasmNG4cWNrYDeMvDHHrNwa2h/2eXbs2NHw9fU1nJ2djccee8zo2LGjzbXLH/b5/d2SJUuMSpUqGS4uLkZQUJDx3//+16Y/L/z8+fHHHw1Jmeo2jLxxLJOTk41XX33VKFGihOHq6mqULl3aeOutt2wuSZkXjiPwqLAYxt++Gg0AAACA6eTZNe0AAABAXkFoBwAAAEyO0A4AAACYHKEdAAAAMDlCOwAAAGByhHYAAADA5AjtAAAAgMkR2gEAAACTI7QDeCTFxsbKYrEoMTHxrh53/vx5eXt768iRI3e0nXPnzsnb21t//PHHPysYAPBII7QDsNGjRw+1adPGbvsPCAiQxWKRxWJR/vz5FRwcrC+//NJu9dzqvffe03PPPaeAgIA7Gl+0aFF1795do0ePfrCFAQDyNEI7ANN5++23derUKe3atUvdunVTnz599MMPP9i7LKWmpioyMlK9evW6q8f17NlT0dHR+uuvvx5QZQCAvI7QDuCuxMXFqVatWnJxcZGvr6/eeOMNXb9+XZK0dOlSeXl5KT09XZIUHx8vi8WiN954w/r43r17q1u3bjnuo2DBgvLx8VHp0qU1fPhwFS5cWCtWrLD2b9myRU2aNFHRokXl6empBg0a6Ndff7XZhsVi0Zdffqm2bdsqf/78CgwM1OLFi7PdZ2pqqlq0aKG6detmu9Rl2bJlcnFx0VNPPXVX26lYsaL8/PwUExOT47wBAMgOoR3AHTtx4oRatmypmjVraseOHZo2bZoiIyP17rvvSpLq1aunixcvavv27ZJuBPyiRYsqNjbWuo24uDiFhITc0f4yMjL07bff6sKFC3J2dra2X7x4UREREVq/fr1++eUXBQYGqmXLlrp48aLN48eOHasOHTpo586datmypbp27Zrl2e7ExEQ1adJEGRkZWrFihby8vLKsZ926dapevXq29ea0nVq1amndunV3NG8AAG5FaAdwxz799FP5+/vrk08+UVBQkNq0aaOxY8dq4sSJysjIkKenp5588klrSI+NjdVrr72m7du3KyUlRSdOnNChQ4fUoEGDHPczfPhwubu7y8XFRe3bt1ehQoXUu3dva3+jRo3UrVs3BQUFqXz58vrvf/+r1NRUxcXF2WynR48e6ty5s8qWLav3339fKSkp2rx5s82Y06dPq0GDBvL19dWSJUuUP3/+bOs6evSo/Pz8suy73Xb8/Px09OjRHOcNAEB2CO0A7tjevXtVp04dWSwWa1vdunWVkpJivTpKgwYNFBsbK8MwtG7dOrVr107ly5fX+vXrFRcXJz8/PwUGBua4n2HDhik+Pl6rV69W7dq1NXnyZJUtW9baf+bMGfXp00eBgYHy9PSUh4eHUlJSdOzYMZvtVK5c2frvAgUKyMPDQ2fPnrUZ06RJE5UtW1bz5s2zOZuflcuXL8vV1TXLvtttx83NTampqTluHwCA7BDaAdxXISEhWr9+vXbs2CEnJycFBQUpJCREsbGxiouLu+1ZdunGFVfKli2revXqacGCBXrllVe0Z88ea39ERITi4+P1n//8Rz///LPi4+NVpEgRpaWl2WzHycnJ5r7FYlFGRoZNW1hYmNauXWuz/ZzqunDhQpZ9t9vOX3/9pWLFit12HwAAZIXQDuCOlS9fXhs3bpRhGNa2DRs2qGDBgnr88ccl/W9d++TJk60B/WZoj42NveP17Df5+/urY8eOGjFihM0+X3nlFbVs2VIVK1aUi4uLzp07d09zGj9+vCIiItS4cePbBveqVatmO+Z229m1a5eqVq16TzUCAEBoB5BJUlKS4uPjbW7Hjx9Xv379dPz4cQ0cOFD79u3Td999p9GjR2vw4MHKl+/Gj5NChQqpcuXKio6Otgb0+vXr69dff9WBAwfu6Ez7rV599VUtWbJEW7dulSQFBgZq1qxZ2rt3rzZt2qSuXbvKzc3tnuc7YcIEde3aVY0aNdK+ffuyHdesWTPt3r0727Pt2W0nNTVV27ZtU9OmTe+5RgDAo43QDiCT2NhYVa1a1eY2duxYPfbYY1q2bJk2b96sKlWqqG/fvurVq5f+9a9/2Ty+QYMGSk9Pt4b2woULq0KFCvLx8VG5cuXuup4KFSqoadOmGjVqlCQpMjJSFy5cULVq1fTCCy/olVdekbe39z+a8+TJk9WhQwc1atRIBw4cyHJMcHCwqlWrpvnz59/Vdr777juVKFFC9erV+0c1AgAeXRbj73/nBgDk6Pvvv9ewYcO0a9cu618Xbuepp57SK6+8oi5dujzg6gAAeZWjvQsAgIdJWFiYDh48qBMnTsjf3/+248+dO6d27dqpc+fOuVAdACCv4kw7AAAAYHKsaQcAAABMjtAOAAAAmByhHQAAADA5QjsAAABgcoR2AAAAwOQI7QAAAIDJEdoBAAAAkyO0AwAAACZHaAcAAABM7v8BFKgcggE2YG4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bar_width = 12\n",
    "plt.bar(np.arange(50, 801, 50) - bar_width/2, success, width = bar_width, label=\"Attack Success\")\n",
    "plt.bar(np.arange(50, 801, 50) + bar_width/2, failed, width = bar_width, label=\"Attack Failed\")\n",
    "plt.bar(np.arange(50, 801, 50) + bar_width + bar_width/2, skipped, width = bar_width, label = \"Attack Skipped\")\n",
    "plt.title(\"Attack on Self Attention Matrix\")\n",
    "plt.xlabel(\"Low Rank (k)\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.legend(bbox_to_anchor=(1.05, 1.0), loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd07f097-8b01-4314-b803-2801e0e543ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b55d48f-e993-4063-9179-42adf2be051b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Local)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}