notebook_default_fio_reports.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "from tabulate import tabulate\n",
    "\n",
    "REPORTS_DIR = './reports'\n",
    "#The 1st level - environments\n",
    "#The 2nd level - experiments\n",
    "#The 3rd level - fio reports (files)\n",
    "\n",
    "#\n",
    "#reports\n",
    "# |-prod\n",
    "# | |-test-001\n",
    "# | |-test-002\n",
    "# |-staging\n",
    "# | |-test-001\n",
    "# | |-test-002\n",
    "\n",
    "\n",
    "REPORTS_MAKS = \"*\\.fio\\.report\"\n",
    "\n",
    "def load_envs(reports_dir = './reports', envs = []):\n",
    "        return {o:os.path.join(reports_dir,o) for o in os.listdir(reports_dir) if \n",
    "                (envs is None or len(envs) == 0 or o in envs )\n",
    "                and os.path.isdir(os.path.join(reports_dir,o))}\n",
    "\n",
    "def load_fio_repors(exp_dir, mask):\n",
    "    return [{'name':o, 'content':Path(os.path.join(exp_dir,o)).read_text()} for o in os.listdir(exp_dir) if os.path.isfile(os.path.join(exp_dir,o))]\n",
    "    \n",
    "def load_experiments(envs, mask = \".*\"):\n",
    "    exps = []\n",
    "    for env_k, env_path in envs.items():\n",
    "        exps_dirs = [{'exp':o, 'path':os.path.join(env_path,o)} for o in os.listdir(env_path) if os.path.isdir(os.path.join(env_path,o))]\n",
    "        for d in exps_dirs:\n",
    "            reports = load_fio_repors(d['path'], mask)\n",
    "            for r in reports:\n",
    "                exp = {'env':env_k, 'exp':d['exp'], 'report':r['name'], 'content':r['content']}\n",
    "                exps.append(exp)\n",
    "    return exps\n",
    "\n",
    "def to_mbs(val):\n",
    "    if \"MB/s\" in val:\n",
    "        return val.rstrip(\"MB/s\")\n",
    "    if \"KB/s\" in val:\n",
    "        return float(val.rstrip(\"KB/s\"))/1000\n",
    "    if \"MiB/s\"  in val:\n",
    "        return float(val.rstrip(\"MiB/s\"))*1024*1024/1000/1000\n",
    "    if \"KiB/s\"  in val:\n",
    "        return float(val.rstrip(\"KiB/s\"))*1024/1000/1000\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Fio2Parser(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.Version = 2\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_report_summary_info_line(line):\n",
    "        pattern = \"[a-z]+=[0-9\\.]+[a-zA-Z/\\.]+\"\n",
    "        vals = re.findall(pattern, line)\n",
    "        info = {}\n",
    "        for v in vals:\n",
    "            kv = v.split('=')\n",
    "            if (len(kv)!=2):\n",
    "                continue\n",
    "            info[kv[0]] = kv[1]\n",
    "        return info\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_file_info_line(line):\n",
    "        pattern = \"[a-z]+=[0-9]+\"\n",
    "        vals = re.findall(pattern, line)\n",
    "        info = {}\n",
    "        for v in vals:\n",
    "            kv = v.split('=')\n",
    "            if (len(kv)!=2):\n",
    "                continue\n",
    "            info[kv[0]] = kv[1]\n",
    "        return info\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def get_report_summary_info(fname, c):\n",
    "        lines = c.splitlines()\n",
    "        g = (i for i, e in enumerate(lines) if 'Run status group 0' in e)\n",
    "        try:\n",
    "            idx = next(g)\n",
    "            read_line = lines[idx+1]\n",
    "            write_line = lines[idx+2]\n",
    "            return {'read':Fio2Parser.get_report_summary_info_line(read_line),\n",
    "                'write':Fio2Parser.get_report_summary_info_line(write_line)}\n",
    "\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print((\"Invalid (summary) report: {}\".format(fname)))\n",
    "            return {}\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_iops_avg(fname, c):\n",
    "        lines = c.splitlines()\n",
    "        tmp = []\n",
    "        for i, e in enumerate(lines):\n",
    "            if not ('-file' in e and '(groupid=' in e and 'pid=' in e):\n",
    "                continue\n",
    "            read_line = lines[i+1]\n",
    "            write_line = lines[i+12]\n",
    "            tmp.append({'read':Fio2Parser.get_file_info_line(read_line)['iops'],\n",
    "                'write':Fio2Parser.get_file_info_line(write_line)['iops']})\n",
    "        read_iops = sum(int(r['read']) for r in tmp)/len(tmp)\n",
    "        write_iops = sum(int(r['write']) for r in tmp)/len(tmp)\n",
    "        return {'read':read_iops, 'write':write_iops}\n",
    "\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_summary(e, info):\n",
    "        info = Fio2Parser.get_report_summary_info(e['report'], e['content'])\n",
    "        iops = Fio2Parser.get_iops_avg(e['report'], e['content'])\n",
    "        if(len(info.keys()) > 0):\n",
    "            return [e['env'],e['exp'], e['report'], to_mbs(info['read']['aggrb']), to_mbs(info['write']['aggrb']), \n",
    "                   iops['read'], iops['write']]\n",
    "        else:\n",
    "            return []\n",
    "\n",
    "\n",
    "class Fio3Parser(object):\n",
    "\n",
    "    def __init__(self):\n",
    "        self.Version = 3\n",
    "\n",
    "    @staticmethod\n",
    "    def get_report_summary_info_line(line):\n",
    "        pattern = \"[a-z]+=[0-9\\.]+([a-zA-Z/\\.]+)?\"\n",
    "        vals = re.findall(pattern, line)\n",
    "        info = {}\n",
    "        for v in vals:\n",
    "            kv = v.split('=')\n",
    "            info[kv[0]] = kv[1]\n",
    "        return info\n",
    "\n",
    "    @staticmethod\n",
    "    def get_report_summary_info(fname, c):\n",
    "        lines = c.splitlines()\n",
    "        g = (i for i, e in enumerate(lines) if 'Run status group 0' in e)\n",
    "        try:\n",
    "            idx = next(g)\n",
    "            read_line = lines[idx+1]\n",
    "            write_line = lines[idx+2]\n",
    "            return {'read':get_report_summary_info_line(read_line),\n",
    "                'write':get_report_summary_info_line(write_line)}\n",
    "\n",
    "        except Exception as e:\n",
    "            print((\"Invalid report: {}\".format(fname)))\n",
    "            return {} \n",
    "\n",
    "    @staticmethod\n",
    "    def get_file_info_line(line):\n",
    "        pattern = \"[a-zA-Z]+=[0-9]+\"\n",
    "        vals = re.findall(pattern, line)\n",
    "        info = {}\n",
    "        for v in vals:\n",
    "            kv = v.split('=')\n",
    "            if (len(kv)!=2):\n",
    "                continue\n",
    "            info[kv[0]] = kv[1]\n",
    "        return info\n",
    "\n",
    "    @staticmethod\n",
    "    def get_iops_avg(fname, c):\n",
    "        lines = c.splitlines()\n",
    "        tmp = []\n",
    "        for i, e in enumerate(lines):\n",
    "            if not ('-file' in e and '(groupid=' in e and 'pid=' in e):\n",
    "                continue\n",
    "            read_line = lines[i+1]\n",
    "            write_line = lines[i+13]\n",
    "            if not \"IOPS\" in write_line:\n",
    "                write_line = lines[i+14]\n",
    "            else:\n",
    "                raise Exception(\"Invalid fio report - {}\".format(fname))\n",
    "\n",
    "            tmp.append({'read':Fio3Parser.get_file_info_line(read_line)['IOPS'],\n",
    "                'write':Fio3Parser.get_file_info_line(write_line)['IOPS']})\n",
    "        read_iops = sum(int(r['read']) for r in tmp)/len(tmp)\n",
    "        write_iops = sum(int(r['write']) for r in tmp)/len(tmp)\n",
    "        return {'read':read_iops, 'write':write_iops}\n",
    "\n",
    "    @staticmethod\n",
    "    def get_summary(e, info):\n",
    "            info = get_report_summary_info(e['report'], e['content'])\n",
    "            iops = Fio3Parser.get_iops_avg(e['report'], e['content'])\n",
    "            if(len(info.keys()) > 0):\n",
    "                return [e['env'],e['exp'], e['report'], to_mbs(info['read']['bw']), to_mbs(info['write']['bw']),\n",
    "                       iops['read'], iops['write']]\n",
    "            else:\n",
    "                return []\n",
    "\n",
    "def get_fio_parser(fname, content):\n",
    "    lines =  content.splitlines()\n",
    "    for l in lines:\n",
    "        if l.startswith(\"fio-2.\"):\n",
    "            return Fio2Parser()\n",
    "        if l.startswith(\"fio-3.\"):\n",
    "            return Fio3Parser()\n",
    "    raise Exception(\"Invalid fio report\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 1000)\n",
    "\n",
    "exps = load_experiments(load_envs())\n",
    "\n",
    "summary = []\n",
    "for e in exps:\n",
    "    parser = None\n",
    "    try:\n",
    "        parser = get_fio_parser(e['report'], e['content'])\n",
    "    except Exception as ex:\n",
    "        print(\"Skipping {}\".format(e['report']))\n",
    "        continue\n",
    "\n",
    "    s = parser.get_summary(e, info)\n",
    "    summary.append(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(summary, columns=[\"env\",\"exp\", \"report\", \"read_mbps\", \"write_mbps\", \"iops_read\", \"iops_write\"])\n",
    "df = df.sort_values(by=[\"read_mbps\", \"write_mbps\"], ascending=False)\n",
    "#df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tabulate(df, tablefmt=\"pipe\", headers=\"keys\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}