docs/tutorials/pytorch/text-classification/bert-base-uncased-MRPC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "17ec3098",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Introduction "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69e00856",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "This tutorial demonstrates how to quantize a BERT model with both static and dynamic post training quantization based on [Intel® Neural Compressor](https://github.com/intel/neural-compressor) and benchmark the quantized models. For better int8 performance benefit, multi-instance benchmarking with 4 cores/instance is recommended."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cbd8bd4",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Prerequisite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2172da4c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "86b20e2b",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Install packages"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1816be1",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "* Follow [installation](https://github.com/intel/intel-extension-for-transformers#installation) to install **intel-extension-for-transformers**. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef103e44",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# install model dependency\n",
    "! pip install accelerate datasets >= 1.1.3 sentencepiece != 0.1.92 protobuf torch >= 1.10 transformers >= 4.12.0 wandb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e35d9563",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b424903",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import transformers\n",
    "from dataclasses import dataclass, field\n",
    "from datasets import load_dataset, load_metric\n",
    "from intel_extension_for_transformers.transformers import metrics, objectives, OptimizedModel, QuantizationConfig\n",
    "from intel_extension_for_transformers.transformers.trainer import NLPTrainer\n",
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoModelForSequenceClassification,\n",
    "    AutoTokenizer,\n",
    "    DataCollatorWithPadding,\n",
    "    EvalPrediction,\n",
    "    HfArgumentParser,\n",
    "    PretrainedConfig,\n",
    "    TrainingArguments,\n",
    "    default_data_collator,\n",
    "    set_seed,\n",
    ")\n",
    "from transformers.trainer_utils import get_last_checkpoint\n",
    "from transformers.utils import check_min_version\n",
    "from typing import Optional\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "# Will error if the minimal version of Transformers is not installed. Remove at your own risks.\n",
    "check_min_version(\"4.12.0\")\n",
    "\n",
    "task_to_keys = {\n",
    "    \"cola\": (\"sentence\", None),\n",
    "    \"mnli\": (\"premise\", \"hypothesis\"),\n",
    "    \"mrpc\": (\"sentence1\", \"sentence2\"),\n",
    "    \"qnli\": (\"question\", \"sentence\"),\n",
    "    \"qqp\": (\"question1\", \"question2\"),\n",
    "    \"rte\": (\"sentence1\", \"sentence2\"),\n",
    "    \"sst2\": (\"sentence\", None),\n",
    "    \"stsb\": (\"sentence1\", \"sentence2\"),\n",
    "    \"wnli\": (\"sentence1\", \"sentence2\"),\n",
    "}\n",
    "\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "048665d4",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Define arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aead749f",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# ========== Define arguments =========\n",
    "@dataclass\n",
    "class DataTrainingArguments:\n",
    "    \"\"\"\n",
    "    Arguments pertaining to what data we are going to input our model for training and eval.\n",
    "\n",
    "    Using `HfArgumentParser` we can turn this class\n",
    "    into argparse arguments to be able to specify them on\n",
    "    the command line.\n",
    "    \"\"\"\n",
    "    task_name: Optional[str] = field(\n",
    "        default=None,\n",
    "        metadata={\"help\": \"The name of the task to train on: \" + \", \".join(task_to_keys.keys())},\n",
    "    )\n",
    "    dataset_name: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"The name of the dataset to use (via the datasets library).\"}\n",
    "    )\n",
    "    max_seq_length: int = field(\n",
    "        default=128,\n",
    "        metadata={\n",
    "            \"help\": \"The maximum total input sequence length after tokenization. Sequences longer \"\n",
    "            \"than this will be truncated, sequences shorter will be padded.\"\n",
    "        },\n",
    "    )\n",
    "    overwrite_cache: bool = field(\n",
    "        default=False, metadata={\"help\": \"Overwrite the cached preprocessed datasets or not.\"}\n",
    "    )\n",
    "    train_file: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"A csv or a json file containing the training data.\"}\n",
    "    )\n",
    "    validation_file: Optional[str] = field(\n",
    "        default=None, metadata={\"help\": \"A csv or a json file containing the validation data.\"}\n",
    "    )\n",
    "\n",
    "    def __post_init__(self):\n",
    "        if self.task_name is not None:\n",
    "            self.task_name = self.task_name.lower()\n",
    "            if self.task_name not in task_to_keys.keys():\n",
    "                raise ValueError(\"Unknown task, you should pick one in \" + \",\".join(task_to_keys.keys()))\n",
    "        elif self.dataset_name is not None:\n",
    "            pass\n",
    "        elif self.train_file is None or self.validation_file is None:\n",
    "            raise ValueError(\"Need either a GLUE task, a training/validation file or a dataset name.\")\n",
    "        else:\n",
    "            train_extension = self.train_file.split(\".\")[-1]\n",
    "            assert train_extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n",
    "            validation_extension = self.validation_file.split(\".\")[-1]\n",
    "            assert (\n",
    "                validation_extension == train_extension\n",
    "            ), \"`validation_file` should have the same extension (csv or json) as `train_file`.\"\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class ModelArguments:\n",
    "    \"\"\"\n",
    "    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n",
    "    \"\"\"\n",
    "    model_name_or_path: str = field(\n",
    "        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n",
    "    )\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class OptimizationArguments:\n",
    "    \"\"\"\n",
    "    Arguments pertaining to what type of optimization we are going to apply on the model.\n",
    "    \"\"\"\n",
    "    tune: bool = field(\n",
    "        default=False,\n",
    "        metadata={\"help\": \"Whether or not to apply quantization.\"},\n",
    "    )\n",
    "    quantization_approach: Optional[str] = field(\n",
    "        default=\"PostTrainingStatic\",\n",
    "        metadata={\"help\": \"Quantization approach. Supported approach are PostTrainingStatic, \"\n",
    "                  \"PostTrainingDynamic and QuantizationAwareTraining.\"},\n",
    "    )\n",
    "    is_relative: Optional[bool] = field(\n",
    "        default=True,\n",
    "        metadata={\"help\": \"Metric tolerance model, expected to be relative or absolute.\"},\n",
    "    )\n",
    "    perf_tol: Optional[float] = field(\n",
    "        default=0.01,\n",
    "        metadata={\"help\": \"Performance tolerance when optimizing the model.\"},\n",
    "    )\n",
    "    benchmark: bool = field(\n",
    "        default=False,\n",
    "        metadata={\"help\": \"run benchmark.\"})\n",
    "    int8: bool = field(\n",
    "        default=False,\n",
    "        metadata={\"help\":\"run benchmark.\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11b035ee",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model_args = ModelArguments(\n",
    "    model_name_or_path=\"textattack/bert-base-uncased-MRPC\",\n",
    ")\n",
    "data_args = DataTrainingArguments(\n",
    "    task_name=\"mrpc\",\n",
    "    max_seq_length=128,\n",
    "    overwrite_cache=True\n",
    ")\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./saved_result_static\",\n",
    "    do_eval=True,\n",
    "    do_train=True,\n",
    "    no_cuda=True,\n",
    "    overwrite_output_dir=True,\n",
    "    per_device_train_batch_size=8,\n",
    ")\n",
    "optim_args = OptimizationArguments(\n",
    "    tune=True,\n",
    "    quantization_approach=\"PostTrainingStatic\"\n",
    ")\n",
    "log_level = training_args.get_process_log_level()\n",
    "logger.setLevel(log_level)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "522f4153",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Download dataset from the hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22bcb0fb",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# download the dataset.\n",
    "raw_datasets = load_dataset(\"glue\", data_args.task_name)\n",
    "# Labels\n",
    "label_list = raw_datasets[\"train\"].features[\"label\"].names\n",
    "num_labels = len(label_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d162019c",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Download fp32 model from the hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2620a5f",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Load pretrained model and tokenizer\n",
    "config = AutoConfig.from_pretrained(\n",
    "    model_args.model_name_or_path,\n",
    "    num_labels=num_labels,\n",
    "    finetuning_task=data_args.task_name,\n",
    "    revision=\"main\"\n",
    ")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    model_args.model_name_or_path,\n",
    "    use_fast=True,\n",
    "    revision=\"main\"\n",
    ")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    model_args.model_name_or_path,\n",
    "    from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n",
    "    config=config,\n",
    "    revision=\"main\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2634a186",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Preprocessing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a873a324",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Preprocessing the raw_datasets\n",
    "sentence1_key, sentence2_key = task_to_keys[data_args.task_name]\n",
    "# Padding strategy\n",
    "padding = False\n",
    "# Some models have set the order of the labels to use, so let's make sure we do use it.\n",
    "label_to_id = None\n",
    "if model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id:\n",
    "    # Some have all caps in their config, some don't.\n",
    "    label_name_to_id = {k.lower(): v for k, v in model.config.label2id.items()}\n",
    "    if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)):\n",
    "        label_to_id = {i: int(label_name_to_id[label_list[i]]) for i in range(num_labels)}\n",
    "    else:\n",
    "        logger.warning(\n",
    "            f\"Your model seems to have been trained with labels, but they don't match the dataset: \"\n",
    "            f\"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}.\\n\"\n",
    "            f\"Ignoring the model labels as a result.\"\n",
    "        )\n",
    "if label_to_id is not None:\n",
    "    model.config.label2id = label_to_id\n",
    "    model.config.id2label = {id: label for label, id in config.label2id.items()}\n",
    "max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)\n",
    "\n",
    "def preprocess_function(examples):\n",
    "    # Tokenize the texts\n",
    "    args = (\n",
    "        (examples[sentence1_key],) if sentence2_key is None else (examples[sentence1_key], examples[sentence2_key])\n",
    "    )\n",
    "    result = tokenizer(*args, padding=padding, max_length=max_seq_length, truncation=True)\n",
    "\n",
    "    # Map labels to IDs (not necessary for GLUE tasks)\n",
    "    if label_to_id is not None and \"label\" in examples:\n",
    "        result[\"label\"] = [(label_to_id[l] if l != -1 else -1) for l in examples[\"label\"]]\n",
    "    return result\n",
    "\n",
    "with training_args.main_process_first(desc=\"dataset map pre-processing\"):\n",
    "    raw_datasets = raw_datasets.map(\n",
    "        preprocess_function, batched=True, load_from_cache_file=not data_args.overwrite_cache\n",
    "    )\n",
    "\n",
    "if training_args.do_train:\n",
    "    if \"train\" not in raw_datasets:\n",
    "        raise ValueError(\"--do_train requires a train dataset\")\n",
    "    train_dataset = raw_datasets[\"train\"]\n",
    "\n",
    "if training_args.do_eval:\n",
    "    if \"validation\" not in raw_datasets and \"validation_matched\" not in raw_datasets:\n",
    "        raise ValueError(\"--do_eval requires a validation dataset\")\n",
    "    eval_dataset = raw_datasets[\"validation_matched\" if data_args.task_name == \"mnli\" else \"validation\"]\n",
    "\n",
    "# Log a few random samples from the training set:\n",
    "if training_args.do_train:\n",
    "    for index in random.sample(range(len(train_dataset)), 3):\n",
    "        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n",
    "\n",
    "# Get the metric function\n",
    "metric = load_metric(\"glue\", data_args.task_name)\n",
    "\n",
    "metric_name = \"eval_accuracy\"\n",
    "\n",
    "# You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a\n",
    "# predictions and label_ids field) and has to return a dictionary string to float.\n",
    "def compute_metrics(p: EvalPrediction):\n",
    "    preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions\n",
    "    preds =  np.argmax(preds, axis=1)\n",
    "    if data_args.task_name is not None:\n",
    "        result = metric.compute(predictions=preds, references=p.label_ids)\n",
    "        if len(result) > 1:\n",
    "            result[\"combined_score\"] = np.mean(list(result.values())).item()\n",
    "        return result\n",
    "    else:\n",
    "        return {\"accuracy\": (preds == p.label_ids).astype(np.float32).mean().item()}\n",
    "\n",
    "# Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.\n",
    "data_collator = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "942d2967",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Quantization & Benchmark"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a07a8735",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Static Post Training Quantization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df04531d",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Set seed before initializing model.\n",
    "set_seed(training_args.seed)\n",
    "# Initialize our Trainer\n",
    "trainer_static = NLPTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset if training_args.do_train else None,\n",
    "    eval_dataset=eval_dataset if training_args.do_eval else None,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "# tuning\n",
    "if not training_args.do_eval:\n",
    "    raise ValueError(\"do_eval must be set to True for quantization.\")\n",
    "\n",
    "model.config.save_pretrained(\"./saved_results_static\")\n",
    "trainer_static.save_model(\"./saved_results_static\")\n",
    "\n",
    "tune_metric = metrics.Metric(\n",
    "    name=metric_name, is_relative=True, criterion=0.25\n",
    ")\n",
    "objective = objectives.performance\n",
    "quantization_config = QuantizationConfig(\n",
    "    approach=\"PostTrainingStatic\",\n",
    "    max_trials=600,\n",
    "    metrics=[tune_metric],\n",
    "    objectives=[objective]\n",
    ")\n",
    "trainer_static.quantize(quant_config=quantization_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a7e93de",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Run Benchmark after Static Post Training Quantization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf54c034",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = trainer_static.evaluate()\n",
    "throughput = results.get(\"eval_samples_per_second\")\n",
    "eval_acc = results.get(\"eval_accuracy\")\n",
    "print('Batch size = {}'.format(training_args.per_device_eval_batch_size))\n",
    "print(\"Finally Eval eval_accuracy Accuracy: {:.5f}\".format(eval_acc))\n",
    "print(\"Latency: {:.5f} ms\".format(1000 / throughput))\n",
    "print(\"Throughput: {:.5f} samples/sec\".format(throughput))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc69524d",
   "metadata": {},
   "source": [
    "## Run Benchmark after Static Post Training Quantization with Multi-Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "860e0503",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.system('numactl --hardware')\n",
    "results = os.system('bash ../multi_instance.sh --model=saved_results_static --core_per_instance=4 --data_type=int8')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a51f6ca",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Dynamic Post Training Quantization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a57aa0d6",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Set seed before initializing model.\n",
    "set_seed(training_args.seed)\n",
    "training_args.output_dir = \"saved_results_dynamic\"\n",
    "# Initialize our Trainer\n",
    "trainer_dynamic = NLPTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset if training_args.do_train else None,\n",
    "    eval_dataset=eval_dataset if training_args.do_eval else None,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "\n",
    "# tuning\n",
    "if not training_args.do_eval:\n",
    "    raise ValueError(\"do_eval must be set to True for quantization.\")\n",
    "\n",
    "model.config.save_pretrained(\"./saved_results_dynamic\")\n",
    "trainer_dynamic.save_model(\"./saved_results_dynamic\")\n",
    "\n",
    "tune_metric = metrics.Metric(\n",
    "    name=metric_name, is_relative=True, criterion=0.25\n",
    ")\n",
    "objective = objectives.performance\n",
    "quantization_config = QuantizationConfig(\n",
    "    approach=\"PostTrainingDynamic\",\n",
    "    max_trials=600,\n",
    "    metrics=[tune_metric],\n",
    "    objectives=[objective]\n",
    ")\n",
    "trainer_dynamic.quantize(quant_config=quantization_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9dd5b041",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Run Benchmark after Dynamic Post Training Quantization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "689e288d",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results = trainer_dynamic.evaluate()\n",
    "throughput = results.get(\"eval_samples_per_second\")\n",
    "eval_acc = results.get(\"eval_accuracy\")\n",
    "print('Batch size = {}'.format(training_args.per_device_eval_batch_size))\n",
    "print(\"Finally Eval eval_accuracy Accuracy: {:.5f}\".format(eval_acc))\n",
    "print(\"Latency: {:.5f} ms\".format(1000 / throughput))\n",
    "print(\"Throughput: {:.5f} samples/sec\".format(throughput))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b98f5dc",
   "metadata": {},
   "source": [
    "## Run Benchmark after Dynamic Post Training Quantization with Multi-Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36575ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "results = os.system('bash ../multi_instance.sh --model=saved_results_dynamic --core_per_instance=4 --data_type=int8')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44cca2a1",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Run Benchmark for FP32 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fda9f985",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "set_seed(training_args.seed)\n",
    "trainer_fp32 = NLPTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset if training_args.do_train else None,\n",
    "    eval_dataset=eval_dataset if training_args.do_eval else None,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "results = trainer_fp32.evaluate()\n",
    "\n",
    "throughput = results.get(\"eval_samples_per_second\")\n",
    "eval_acc = results.get(\"eval_accuracy\")\n",
    "print('Batch size = {}'.format(training_args.per_device_eval_batch_size))\n",
    "print(\"Finally Eval eval_accuracy Accuracy: {:.5f}\".format(eval_acc))\n",
    "print(\"Latency: {:.5f} ms\".format(1000 / throughput))\n",
    "print(\"Throughput: {:.5f} samples/sec\".format(throughput))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f7c85bc",
   "metadata": {},
   "source": [
    "## Run Benchmark for FP32 Model with Multi-Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36dec093",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "results = os.system('bash ../multi_instance.sh --model=lvwerra/pegasus-samsum --core_per_instance=4 --data_type=fp32')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "f54fd8d6160ddfbc370985ee3ad2925997e28943a671b1747496a6859c59cd26"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}