util_detection_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-09T17:20:19.111441Z",
     "iopub.status.busy": "2021-02-09T17:20:19.111133Z",
     "iopub.status.idle": "2021-02-09T17:20:19.189648Z",
     "shell.execute_reply": "2021-02-09T17:20:19.188687Z",
     "shell.execute_reply.started": "2021-02-09T17:20:19.111413Z"
    }
   },
   "outputs": [],
   "source": [
    "import kfp\n",
    "import yaml\n",
    "import re\n",
    "\n",
    "c = kfp.Client()\n",
    "# Get all the pipelines (100 is max call in api)\n",
    "tmp_ls_pipelines = c.list_pipelines('',100,'')\n",
    "ls_pipelines = tmp_ls_pipelines.pipelines\n",
    "while tmp_ls_pipelines.next_page_token is not None:\n",
    "    tmp_ls_pipelines = c.list_pipelines(tmp_ls_pipelines.next_page_token,100,'')\n",
    "    ls_pipelines = ls_pipelines + tmp_ls_pipelines.pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-09T17:20:20.897058Z",
     "iopub.status.busy": "2021-02-09T17:20:20.896624Z",
     "iopub.status.idle": "2021-02-09T17:20:20.921817Z",
     "shell.execute_reply": "2021-02-09T17:20:20.920858Z",
     "shell.execute_reply.started": "2021-02-09T17:20:20.897013Z"
    }
   },
   "outputs": [],
   "source": [
    "def seq_iter(obj):\n",
    "    return obj if isinstance(obj, dict) else range(len(obj))\n",
    "\n",
    "# Recursive function to check against regex\n",
    "# yaml_file : the part of the yaml still in need of validation\n",
    "# parent_keys : to keep track of where the hits are\n",
    "# excluded_keys : to ignore anything under that key\n",
    "def recursive_secret_check(yaml_file, parent_keys=None, excluded_keys=None):\n",
    "    positive_tests = []\n",
    "    if parent_keys is None:\n",
    "        parent_keys = []\n",
    "    if excluded_keys is None:\n",
    "        excluded_keys = ['secretKeyRef']\n",
    "    if isinstance(yaml_file, str):\n",
    "        if re.search(r\"^secret$|^password$\", yaml_file.lower()):\n",
    "            positive_tests.append(parent_keys)  # Record the flagged key\n",
    "    else:\n",
    "        for k in seq_iter(yaml_file):\n",
    "            if k in excluded_keys:\n",
    "                continue  # (next step in for loop)\n",
    "            full_key_context = list(parent_keys)  # make a copy of the parent_keys\n",
    "            full_key_context.append(k)  # appends a second list of just this key k\n",
    "            # Skip the check if it's not string\n",
    "            if isinstance(k, str) and re.search(r\"^secret$|^password$\", k.lower()):\n",
    "                positive_tests.append(full_key_context)  # Record the flagged key\n",
    "            #Recursive call to test deeper\n",
    "            deeper_positives = recursive_secret_check(yaml_file[k], full_key_context)\n",
    "            #Concatenate result\n",
    "            positive_tests = positive_tests + deeper_positives\n",
    "    return positive_tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-09T17:20:22.933174Z",
     "iopub.status.busy": "2021-02-09T17:20:22.932870Z",
     "iopub.status.idle": "2021-02-09T17:20:30.790253Z",
     "shell.execute_reply": "2021-02-09T17:20:30.787838Z",
     "shell.execute_reply.started": "2021-02-09T17:20:22.933147Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compute-pi-demo\n",
      "prov-scraper-kfp\n",
      "testpipe-447708894\n",
      "testpipe-448130251\n",
      "testpipe-448145217\n",
      "testpipe-449187266\n",
      "testpipe-449224450\n",
      "testpipe-449386903\n",
      "testpipe-449429067\n",
      "testpipe-449471008\n",
      "testpipe-449623593\n",
      "testpipe-449691596\n",
      "testpipe-449926804\n",
      "testpipe-449949127\n",
      "testpipe-449951297\n",
      "testpipe-450011510\n",
      "testpipe-450036305\n",
      "testpipe-451339070\n",
      "testpipe-451515172\n",
      "testpipe-451905763\n",
      "testpipe-454718103\n",
      "test-lvib7\n",
      "testing-2020-xqsmf\n",
      "testing-2020-zbjli\n",
      "titanic-ml-an6ik\n",
      "titanic-ml-xqad4\n",
      "titanic-ml-o34rw\n",
      "titanic-ml-xfm89\n",
      "titanic-ml-lchzs\n",
      "titanic-ml-sud13\n",
      "titanic-ml-7v9kz\n",
      "titanic-ml-g9092\n",
      "titanic-ml-n6mjv\n",
      "titanic-ml-adjnk\n",
      "titanic-ml-ldzmo\n",
      "titanic-ml-4yr5o\n",
      "titanic-ml-5n3re\n",
      "Pipeline with no version : titanic-ml-5n3re 205cb39e-7467-46e8-ac97-62262f35dd66\n",
      "run-ltc-prod-scraper\n",
      "OpenAddressesPipeline\n",
      "titanic-ml-sz37x\n",
      "OpenAddressesPipeline_1CPU\n",
      "Crop Imaging Events\n",
      "pipeline-psw-name\n",
      "==================\n",
      "Warning: ff0e36a2-c8ac-4db3-8df0-0f7b58cdcbc0 version ff0e36a2-c8ac-4db3-8df0-0f7b58cdcbc0\n",
      "[['spec', 'templates', 0, 'inputs', 'parameters', 0, 'name'], ['spec', 'templates', 0, 'inputs', 'parameters', 1, 'name'], ['spec', 'templates', 2, 'inputs', 'parameters', 0, 'name'], ['spec', 'templates', 2, 'inputs', 'parameters', 1, 'name'], ['spec', 'templates', 2, 'dag', 'tasks', 0, 'arguments', 'parameters', 0, 'name'], ['spec', 'templates', 2, 'dag', 'tasks', 0, 'arguments', 'parameters', 1, 'name'], ['spec', 'arguments', 'parameters', 0, 'name'], ['spec', 'arguments', 'parameters', 1, 'name']]\n",
      "Check completed\n"
     ]
    }
   ],
   "source": [
    "# Loop for each of the pipeline\n",
    "for p_pipeline in ls_pipelines:\n",
    "    print(p_pipeline.name)\n",
    "    #For each pipeline, get the versions\n",
    "    p_pipeline_versions = c.pipelines.list_pipeline_versions(resource_key_id = p_pipeline.id, resource_key_type = 'PIPELINE')\n",
    "    #If no version, cannot get templates\n",
    "    if p_pipeline_versions.total_size is not None:\n",
    "        for p_version in p_pipeline_versions.versions:\n",
    "            p_template = c.pipelines.get_pipeline_version_template(p_version.id)\n",
    "            #each template of each pipeline\n",
    "            yaml_template = yaml.load(p_template.template, Loader=yaml.BaseLoader)\n",
    "\n",
    "            res = recursive_secret_check(yaml_template)\n",
    "            if res:\n",
    "                print('==================')\n",
    "                print('Warning: ' + p_pipeline.id + ' version ' + p_version.id)\n",
    "                print(res)\n",
    "    else:\n",
    "        print('Pipeline with no version : ' + p_pipeline.name + ' ' + p_pipeline.id)\n",
    "print('Check completed')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "kubeflow_notebook": {
   "autosnapshot": false,
   "docker_image": "k8scc01covidacr.azurecr.io/jupyterlab-cpu:ee176847",
   "experiment": {
    "id": "",
    "name": ""
   },
   "experiment_name": "",
   "katib_metadata": {
    "algorithm": {
     "algorithmName": "grid"
    },
    "maxFailedTrialCount": 3,
    "maxTrialCount": 12,
    "objective": {
     "objectiveMetricName": "",
     "type": "minimize"
    },
    "parallelTrialCount": 3,
    "parameters": []
   },
   "katib_run": false,
   "pipeline_description": "",
   "pipeline_name": "",
   "snapshot_volumes": false,
   "steps_defaults": [],
   "volume_access_mode": "rwm",
   "volumes": []
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}