example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import rustxes\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ocel_xml_path = \"test_data/ContainerLogistics.xml\"\n",
    "\n",
    "\n",
    "ocel_rs = rustxes.import_ocel_xml_pm4py(ocel_xml_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pm4py\n",
    "\n",
    "ocel_py = pm4py.read_ocel2_xml(ocel_xml_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13910\n",
      "13882\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print(len(ocel_rs.objects))\n",
    "print(len(ocel_py.objects))\n",
    "print(len(ocel_rs.objects) == len(ocel_py.objects))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ocel_rs.events.equals(ocel_py.events.convert_dtypes().reset_index(drop=True))\n",
    "assert ocel_rs.objects.equals(ocel_py.objects.convert_dtypes().reset_index(drop=True))\n",
    "assert ocel_rs.relations.equals(ocel_py.relations.convert_dtypes().reset_index(drop=True))\n",
    "assert ocel_rs.o2o.equals(ocel_py.o2o.convert_dtypes().reset_index(drop=True))\n",
    "assert ocel_rs.get_summary() == ocel_py.get_summary()\n",
    "\n",
    "\n",
    "# This will still fail! (see below)\n",
    "assert ocel_rs.object_changes.equals(ocel_py.object_changes.convert_dtypes().reset_index(drop=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ocel_json_path = \"/home/aarkue/doc/projects/rust-bridge-process-mining/process_mining/src/event_log/tests/test_data/order-management.json\"\n",
    "ocel_json_rs = rustxes.import_ocel_json_pm4py(ocel_json_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pm4py\n",
    "\n",
    "ocel_json_py = pm4py.read_ocel2_json(ocel_json_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_df_columns(df):\n",
    "  return df.reindex(sorted(df.columns), axis=1)\n",
    "\n",
    "def equals_with_sorted_columns(df1,df2):\n",
    "  df1 = sort_df_columns(df1)\n",
    "  df2 = sort_df_columns(df2)\n",
    "  if not df1.equals(df2):\n",
    "    from pandas.testing import assert_frame_equal\n",
    "    try:\n",
    "      assert_frame_equal(df1,df2)\n",
    "    except Exception as e:\n",
    "      print(e)\n",
    "    finally:\n",
    "      return False\n",
    "  else:\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\nEvents:\\n\" + str(equals_with_sorted_columns(ocel_json_rs.events,ocel_json_py.events.convert_dtypes().reset_index(drop=True))))\n",
    "print(\"\\nObjects:\\n\" + str(equals_with_sorted_columns(ocel_json_rs.objects,ocel_json_py.objects.convert_dtypes().reset_index(drop=True))))\n",
    "print(\"\\nRelations:\\n\" + str(equals_with_sorted_columns(ocel_json_rs.relations,ocel_json_py.relations.convert_dtypes().reset_index(drop=True))))\n",
    "print(\"\\nO2O:\\n\" + str(equals_with_sorted_columns(ocel_json_rs.o2o,ocel_json_py.o2o.convert_dtypes().reset_index(drop=True))))\n",
    "print(\"\\nSummary:\\n\" + str(ocel_json_rs.get_summary() == ocel_json_py.get_summary()))\n",
    "\n",
    "\n",
    "# This will still fail! (see below)\n",
    "print(\"\\nObject Changes:\\n\" + str(equals_with_sorted_columns(ocel_json_rs.object_changes,ocel_json_py.object_changes.convert_dtypes().reset_index(drop=True))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pm4py\n",
    "\n",
    "ocel_pm4py = pm4py.ocel.OCEL(\n",
    "    events=ocel_rs[\"events\"].to_pandas().convert_dtypes(),\n",
    "    objects=ocel_rs[\"objects\"].to_pandas().convert_dtypes(),\n",
    "    relations=ocel_rs[\"relations\"].to_pandas().convert_dtypes(),\n",
    "    globals={},\n",
    "    o2o=ocel_rs[\"o2o\"].to_pandas().convert_dtypes(),\n",
    "    parameters={\n",
    "        pm4py.objects.ocel.obj.Parameters.EVENT_ID: \"ocel:eid\",\n",
    "        pm4py.objects.ocel.obj.Parameters.EVENT_ACTIVITY: \"ocel:activity\",\n",
    "        pm4py.objects.ocel.obj.Parameters.EVENT_TIMESTAMP: \"ocel:timestamp\",\n",
    "        pm4py.objects.ocel.obj.Parameters.OBJECT_ID: \"ocel:oid\",\n",
    "        pm4py.objects.ocel.obj.Parameters.OBJECT_TYPE: \"ocel:type\",\n",
    "        pm4py.objects.ocel.obj.Parameters.QUALIFIER: \"ocel:qualifier\",\n",
    "        pm4py.objects.ocel.obj.Parameters.CHANGED_FIELD: \"ocel:field\",\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ocel_py.get_summary() == ocel_pm4py.get_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ocel_rs.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o2o_rs = ocel_rs[\"o2o\"].to_pandas().convert_dtypes()\n",
    "o2o_rs.equals(ocel_py.o2o.convert_dtypes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objects_rs = ocel_rs[\"objects\"].to_pandas().convert_dtypes()\n",
    "objects_rs.equals(ocel_py.objects.convert_dtypes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "events_rs = ocel_rs[\"events\"].to_pandas().convert_dtypes()\n",
    "events_rs.equals(ocel_py.events.convert_dtypes().reset_index(drop=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "relations_rs = ocel_rs[\"relations\"].to_pandas().convert_dtypes()\n",
    "relations_rs.equals(ocel_py.relations.convert_dtypes().reset_index(drop=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas.testing import assert_frame_equal\n",
    "object_changes_rs = sort_df_columns(ocel_rs[\"object_changes\"].to_pandas().convert_dtypes())\n",
    "object_changes_py = sort_df_columns(ocel_py.object_changes.convert_dtypes().reset_index(drop=False))\n",
    "object_changes_py[\"ocel:timestamp\"] = object_changes_py[\"ocel:timestamp\"].astype(\"datetime64[ns, UTC]\")\n",
    "object_changes_py = object_changes_py.sort_values([\"ocel:timestamp\",\"index\"]).drop(\"index\",axis=1).reset_index(drop=True)\n",
    "# Note: Timestamps don't match!\n",
    "# I think the Rust implementation parses them correctly (Wed May 22 2024 01:00:00 GMT+0200 (Mitteleuropäische Sommerzeit) => 2024-05-21T23:00:00.000Z),\n",
    "# BUT: this is a very weird format and should be fixed upstream in the OCEL logs\n",
    "# Also: pm4py implementation currently doesn't sort wrt. to timestamp for object changes (but it does for events, ...)\n",
    "print(object_changes_rs.drop(\"ocel:timestamp\",axis=1).equals(object_changes_py.drop(\"ocel:timestamp\", axis=1)))\n",
    "if not object_changes_rs.equals(object_changes_py): \n",
    "  assert_frame_equal(object_changes_rs,object_changes_py)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set1 = set(ocel_rs[\"objects\"][\"ocel:oid\"].unique())\n",
    "set2 = set(ocel_py.objects[\"ocel:oid\"].unique())\n",
    "print(\"Is rust object set a superset of py? \" + str(set1.issuperset(set2)))\n",
    "res = set1 - set2\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"material:983\" in set2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "\n",
    "# Rough comparison checks of two DataFrames\n",
    "# Ignores certain differences (e.g., different datatypes) on purpose\n",
    "def compare_log_dfs(df1: DataFrame,df2: DataFrame):\n",
    "  col_diff =  set(df1.columns).symmetric_difference(set(df2.columns));\n",
    "  if len(col_diff) > 0:\n",
    "      print(f\"❌ Column are different: {col_diff}\")\n",
    "      return False\n",
    "\n",
    "  all_columns = set(df1.columns).union(set(df2.columns))\n",
    "  all_equal = True\n",
    "  for col in all_columns:\n",
    "    # or all(...) to handle some weird datatype stuff; if all values are equal, we also consider the column as equal\n",
    "    if  df1[col].equals(df2[col]) or all([v1 == v2 or (pd.isnull(v1) and pd.isnull(v2)) or str(v1) == str(v2) for (v1,v2) in zip(df1[col],df2[col])]):\n",
    "      pass\n",
    "    else:\n",
    "      all_equal = False\n",
    "      print(f\"❌ For column {col} not all entries are equal!\")\n",
    "  if all_equal:\n",
    "    print(f\"✅✅✅ All values in all column are equal!\")\n",
    "    return True\n",
    "  else:\n",
    "    print(f\"❌❌❌ NOT all values in all column are equal!\")\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_import_res = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pm4py\n",
    "import numpy as np\n",
    "import glob\n",
    "import pandas as pd\n",
    "\n",
    "for log_name in glob.glob(\"/home/aarkue/dow/event_data/BPI_Challenge_2017*.xes\"):\n",
    "  print(log_name)\n",
    "  log_name_postfix = log_name.split(\"/\")[-1]\n",
    "\n",
    "  # Measure execution times\n",
    "  start = time.time()\n",
    "  (log_rs,other_data_str) = rustxes.import_xes(log_name)\n",
    "  log_rs = log_rs.to_pandas() if log_rs.shape != (0,0) else pd.DataFrame()\n",
    "  log_rs = log_rs.fillna(np.nan)\n",
    "  rs_dur = time.time() - start\n",
    "  start = time.time()\n",
    "  log_py_lbl = pm4py.read_xes(log_name,variant=\"line_by_line\")\n",
    "  py_lbl_dur = time.time() - start\n",
    "  start = time.time()\n",
    "  try:\n",
    "    log_py_iter = pm4py.read_xes(log_name,variant=\"iterparse\")\n",
    "  except Exception as e:\n",
    "    print(e)\n",
    "  py_iter_dur = time.time() - start\n",
    "\n",
    "  # Print & save execution times in log_import_res for plotting\n",
    "  print(rs_dur,py_lbl_dur,py_iter_dur)\n",
    "  log_import_res[log_name_postfix] = {\"Rust\": rs_dur, \"PM4Py (line_by_line)\": py_lbl_dur, \"PM4Py (iterparse)\": py_iter_dur }\n",
    "  # Check if DataFrames are _roughtly_ equal\n",
    "  if not compare_log_dfs(log_rs,log_py_iter):\n",
    "    print(\"WARNING: Logs do not match for \" + log_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(log_import_res)\n",
    "import plotly.express as px\n",
    "fig = px.bar(df.transpose(), barmode=\"group\", labels={\"index\": \"Event Log\", \"value\": \"Parse Duration [s]\", \"variable\": \"Parsing Implementation\"})\n",
    "fig.show()\n",
    "for log_name in log_import_res: \n",
    "  x = [ \"Rust\", \"PM4Py (line_by_line)\",\"PM4Py (iterparse)\"]\n",
    "  fig = px.bar(labels={\"x\": \"XES Parser\", \"y\": \"Parse Duration [s]\"}, title=f\"XES Parsing Performance on <i>{log_name}</i>\",x=x, y=[log_import_res[log_name][n] for n in x],color=x)\n",
    "  fig.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}