No public description

PiperOrigin-RevId: 645204893

package.json

@@ -5,8 +5,9 @@
     "main": "client/index.ts",
     "scripts": {
         "build": "node build.mjs",
-        "publish": "npm run build && npm run publish:website",
-        "publish:website": "cp -r ./dist/* ./docs",
+        "publish": "npm run build && npm run publish:python && npm run publish:website",
+        "publish:python": "cp -R ./dist/* ./python/src/llm_comparator/data",
+        "publish:website": "cp -R ./dist/* ./docs",
         "serve": "web-dev-server --open --root-dir ./dist",
         "test": "echo \"Error: no test specified\" && exit 1"
     },

python/notebooks/run_scripts_for_llm_comparator.ipynb

@@ -112,54 +112,14 @@
     },
     {
       "metadata": {
-        "id": "3_kgtTlOtI8s"
+        "id": "7W4B1uzqvJZ-"
       },
       "cell_type": "code",
       "source": [
-        "! git clone https://github.com/PAIR-code/llm-comparator"
+        "comparison.show_in_colab(file_path)"
       ],
       "outputs": [],
       "execution_count": null
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "CiKOappAwtkv"
-      },
-      "outputs": [],
-      "source": [
-        "#@title For displaying LLM Comparator.\n",
-        "import IPython\n",
-        "\n",
-        "# TODO: Move to the pip package.\n",
-        "def show_llm_comparator(json_path, height=800, port=4321):\n",
-        "  IPython.get_ipython().system_raw(f'python3 -m http.server {port} \u0026')\n",
-        "  IPython.display.display(IPython.display.Javascript(\"\"\"\n",
-        "  (async ()=\u003e{\n",
-        "    fm = document.createElement('iframe')\n",
-        "    fm.src = await google.colab.kernel.proxyPort(%s)\n",
-        "    results_path = fm.src + '%s'\n",
-        "    fm.src += 'llm-comparator/docs/'\n",
-        "    fm.src += '?results_path=' + results_path\n",
-        "    fm.width = '100%%'\n",
-        "    fm.height = '%d'\n",
-        "    fm.frameBorder = 0\n",
-        "    document.body.append(fm)\n",
-        "  })();\n",
-        "  \"\"\" % (port, json_path, height) ))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "U-o0bDW4wvnR"
-      },
-      "outputs": [],
-      "source": [
-        "show_llm_comparator(file_path, port=7676)"
-      ]
     }
   ],
   "metadata": {

python/pyproject.toml

@@ -39,4 +39,11 @@ keywords = [
 "Homepage" = "https://github.com/PAIR-code/llm-comparator"
 "Repository" = "https://github.com/PAIR-code/llm-comparator"
 "Bug Tracker" = "https://github.com/PAIR-code/llm-comparator/issues"
-"Demo" = "https://pair-code.github.io/llm-comparator/"
+"Demo" = "https://pair-code.github.io/llm-comparator/"
+
+[tool.setuptools.packages.find]
+namespaces = true
+where = ["src"]
+
+[tool.setuptools.package-data]
+"*" = ["*.html", "*.js", "*.svg"]

python/src/llm_comparator/comparison.py

@@ -2,6 +2,9 @@
 
 from collections.abc import Sequence
 import json
+import os
+import pathlib
+from typing import Optional
 
 from llm_comparator import llm_judge_runner
 from llm_comparator import rationale_bullet_generator
@@ -17,6 +20,9 @@ def run(
     bulletizer: rationale_bullet_generator.RationaleBulletGenerator,
     clusterer: rationale_cluster_generator.RationaleClusterGenerator,
     model_names: Sequence[str] = ('A', 'B'),
+    judge_opts: Optional[types.JsonDict] = None,
+    bulletizer_opts: Optional[types.JsonDict] = None,
+    clusterer_opts: Optional[types.JsonDict] = None,
 ) -> types.JsonDict:
   """Runs a comparison with LLM Comparator.
 
@@ -35,15 +41,26 @@ def run(
     clusterer: The Rationale Cluster Generator to use.
     model_names: The names of the models as you would like them to appear in the
       LLM Comparator web application.
+    judge_opts: keyword arguments passed to judge.run(). See the
+      llm_comparator.llm_judge_runner.LLMJudgeRunner.run() documentation for
+      details.
+    bulletizer_opts: keyword arguments passed to bulletizer.run(). See the
+      llm_comparator.rationale_bullet_generator.RationaleBulletGenerator.run()
+      documentation for details.
+    clusterer_opts: keyword arguments passed to clusterer.run(). See the
+      llm_comparator.rationale_cluster_generator.RationaleClusterGenerator.run()
+      documentation for details.
 
   Returns:
     The evaluation results as a JSON object, or the value of output_path if
     provided and writing to that file was successful.
   """
 
-  judgements = judge.run(inputs)
-  bullets = bulletizer.run(judgements)
-  clusters, cluster_similarities = clusterer.run(bullets)
+  judgements = judge.run(inputs, **(judge_opts or {}))
+  bullets = bulletizer.run(judgements, **(bulletizer_opts or {}))
+  clusters, cluster_similarities = clusterer.run(
+      bullets, **(clusterer_opts or {})
+  )
 
   per_example_generator = zip(inputs, judgements, cluster_similarities)
 
@@ -66,7 +83,41 @@ def run(
   }
 
 
-def write(comparison_result: types.JsonDict, output_path: str) -> str:
-  with open(output_path, 'w') as f:
+def write(comparison_result: types.JsonDict, file_path: str) -> str:
+  with open(file_path, 'w') as f:
     json.dump(comparison_result, f)
-  return output_path
+  return file_path
+
+
+def show_in_colab(file_path: str, height: int = 800, port: int = 8888) -> None:
+  """Serves the LLM Comparator app from the Colab content directory."""
+  import IPython  # pylint: disable=g-import-not-at-top #pytype: disable=import-error
+
+  if (ishell := IPython.get_ipython()) is None:
+    raise RuntimeError('Not running in an IPython context.')
+
+  # Copy the website files from the data directory to the Colab content
+  # directory if they don't already exist.
+  if not os.path.isdir('/content/llm_comparator'):
+    website_root = pathlib.Path(__file__).parent / 'data'
+    ishell.system_raw(f'cp -R {website_root} /content/llm_comparator')
+
+  # Serve the website from the Colab content directory.
+  # TODO(llm-comparator): Check if a server is already running before trying to
+  # start a new one.
+  ishell.system_raw(f'python3 -m http.server {port} &')
+
+  # Display the served website in an iframe.
+  IPython.display.display(IPython.display.Javascript("""
+  (async () => {
+    const serverAddress = await google.colab.kernel.proxyPort(%s);
+    const results_path = serverAddress + '%s';
+
+    const fm = document.createElement('iframe');
+    fm.frameBorder = 0
+    fm.height = '%d'
+    fm.width = '100%%'
+    fm.src = serverAddress + 'llm_comparator/?results_path=' + results_path;
+    document.body.append(fm)
+  })();
+  """ % (port, file_path, height)))