MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Download helper from google tutorial\n",
    "from __future__ import print_function\n",
    "import gzip\n",
    "import os\n",
    "import urllib\n",
    "\n",
    "import numpy\n",
    "\n",
    "SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'\n",
    "\n",
    "\n",
    "def maybe_download(filename, work_directory):\n",
    "  \"\"\"Download the data from Yann's website, unless it's already here.\"\"\"\n",
    "  if not os.path.exists(work_directory):\n",
    "    os.mkdir(work_directory)\n",
    "  filepath = os.path.join(work_directory, filename)\n",
    "  if not os.path.exists(filepath):\n",
    "    filepath, _ = urllib.urlretrieve(SOURCE_URL + filename, filepath)\n",
    "    statinfo = os.stat(filepath)\n",
    "    print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')\n",
    "  return filepath\n",
    "\n",
    "\n",
    "def _read32(bytestream):\n",
    "  dt = numpy.dtype(numpy.uint32).newbyteorder('>')\n",
    "  return numpy.frombuffer(bytestream.read(4), dtype=dt)\n",
    "\n",
    "\n",
    "def extract_images(filename):\n",
    "  \"\"\"Extract the images into a 4D uint8 numpy array [index, y, x, depth].\"\"\"\n",
    "  print('Extracting', filename)\n",
    "  with gzip.open(filename) as bytestream:\n",
    "    magic = _read32(bytestream)\n",
    "    if magic != 2051:\n",
    "      raise ValueError(\n",
    "          'Invalid magic number %d in MNIST image file: %s' %\n",
    "          (magic, filename))\n",
    "    num_images = _read32(bytestream)\n",
    "    rows = _read32(bytestream)\n",
    "    cols = _read32(bytestream)\n",
    "    buf = bytestream.read(rows * cols * num_images)\n",
    "    data = numpy.frombuffer(buf, dtype=numpy.uint8)\n",
    "    data = data.reshape(num_images, rows, cols, 1)\n",
    "    return data\n",
    "\n",
    "\n",
    "def dense_to_one_hot(labels_dense, num_classes=10):\n",
    "  \"\"\"Convert class labels from scalars to one-hot vectors.\"\"\"\n",
    "  num_labels = labels_dense.shape[0]\n",
    "  index_offset = numpy.arange(num_labels) * num_classes\n",
    "  labels_one_hot = numpy.zeros((num_labels, num_classes))\n",
    "  labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "  return labels_one_hot\n",
    "\n",
    "\n",
    "def extract_labels(filename, one_hot=False):\n",
    "  \"\"\"Extract the labels into a 1D uint8 numpy array [index].\"\"\"\n",
    "  print('Extracting', filename)\n",
    "  with gzip.open(filename) as bytestream:\n",
    "    magic = _read32(bytestream)\n",
    "    if magic != 2049:\n",
    "      raise ValueError(\n",
    "          'Invalid magic number %d in MNIST label file: %s' %\n",
    "          (magic, filename))\n",
    "    num_items = _read32(bytestream)\n",
    "    buf = bytestream.read(num_items)\n",
    "    labels = numpy.frombuffer(buf, dtype=numpy.uint8)\n",
    "    if one_hot:\n",
    "      return dense_to_one_hot(labels)\n",
    "    return labels\n",
    "\n",
    "\n",
    "class DataSet(object):\n",
    "\n",
    "  def __init__(self, images, labels, fake_data=False):\n",
    "    if fake_data:\n",
    "      self._num_examples = 10000\n",
    "    else:\n",
    "      assert images.shape[0] == labels.shape[0], (\n",
    "          \"images.shape: %s labels.shape: %s\" % (images.shape,\n",
    "                                                 labels.shape))\n",
    "      self._num_examples = images.shape[0]\n",
    "\n",
    "      # Convert shape from [num examples, rows, columns, depth]\n",
    "      # to [num examples, rows*columns] (assuming depth == 1)\n",
    "      assert images.shape[3] == 1\n",
    "      images = images.reshape(images.shape[0],\n",
    "                              images.shape[1] * images.shape[2])\n",
    "      # Convert from [0, 255] -> [0.0, 1.0].\n",
    "      images = images.astype(numpy.float32)\n",
    "      images = numpy.multiply(images, 1.0 / 255.0)\n",
    "    self._images = images\n",
    "    self._labels = labels\n",
    "    self._epochs_completed = 0\n",
    "    self._index_in_epoch = 0\n",
    "\n",
    "  @property\n",
    "  def images(self):\n",
    "    return self._images\n",
    "\n",
    "  @property\n",
    "  def labels(self):\n",
    "    return self._labels\n",
    "\n",
    "  @property\n",
    "  def num_examples(self):\n",
    "    return self._num_examples\n",
    "\n",
    "  @property\n",
    "  def epochs_completed(self):\n",
    "    return self._epochs_completed\n",
    "\n",
    "  def next_batch(self, batch_size, fake_data=False):\n",
    "    \"\"\"Return the next `batch_size` examples from this data set.\"\"\"\n",
    "    if fake_data:\n",
    "      fake_image = [1.0 for _ in xrange(784)]\n",
    "      fake_label = 0\n",
    "      return [fake_image for _ in xrange(batch_size)], [\n",
    "          fake_label for _ in xrange(batch_size)]\n",
    "    start = self._index_in_epoch\n",
    "    self._index_in_epoch += batch_size\n",
    "    if self._index_in_epoch > self._num_examples:\n",
    "      # Finished epoch\n",
    "      self._epochs_completed += 1\n",
    "      # Shuffle the data\n",
    "      perm = numpy.arange(self._num_examples)\n",
    "      numpy.random.shuffle(perm)\n",
    "      self._images = self._images[perm]\n",
    "      self._labels = self._labels[perm]\n",
    "      # Start next epoch\n",
    "      start = 0\n",
    "      self._index_in_epoch = batch_size\n",
    "      assert batch_size <= self._num_examples\n",
    "    end = self._index_in_epoch\n",
    "    return self._images[start:end], self._labels[start:end]\n",
    "\n",
    "\n",
    "def read_data_sets(train_dir, fake_data=False, one_hot=False):\n",
    "  class DataSets(object):\n",
    "    pass\n",
    "  data_sets = DataSets()\n",
    "\n",
    "  if fake_data:\n",
    "    data_sets.train = DataSet([], [], fake_data=True)\n",
    "    data_sets.validation = DataSet([], [], fake_data=True)\n",
    "    data_sets.test = DataSet([], [], fake_data=True)\n",
    "    return data_sets\n",
    "\n",
    "  TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'\n",
    "  TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'\n",
    "  TEST_IMAGES = 't10k-images-idx3-ubyte.gz'\n",
    "  TEST_LABELS = 't10k-labels-idx1-ubyte.gz'\n",
    "  VALIDATION_SIZE = 5000\n",
    "\n",
    "  local_file = maybe_download(TRAIN_IMAGES, train_dir)\n",
    "  train_images = extract_images(local_file)\n",
    "\n",
    "  local_file = maybe_download(TRAIN_LABELS, train_dir)\n",
    "  train_labels = extract_labels(local_file, one_hot=one_hot)\n",
    "\n",
    "  local_file = maybe_download(TEST_IMAGES, train_dir)\n",
    "  test_images = extract_images(local_file)\n",
    "\n",
    "  local_file = maybe_download(TEST_LABELS, train_dir)\n",
    "  test_labels = extract_labels(local_file, one_hot=one_hot)\n",
    "\n",
    "  validation_images = train_images[:VALIDATION_SIZE]\n",
    "  validation_labels = train_labels[:VALIDATION_SIZE]\n",
    "  train_images = train_images[VALIDATION_SIZE:]\n",
    "  train_labels = train_labels[VALIDATION_SIZE:]\n",
    "\n",
    "  data_sets.train = DataSet(train_images, train_labels)\n",
    "  data_sets.validation = DataSet(validation_images, validation_labels)\n",
    "  data_sets.test = DataSet(test_images, test_labels)\n",
    "\n",
    "  return data_sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py:43: DeprecationWarning: converting an array with ndim > 0 to an index will result in an error in the future\n"
     ]
    }
   ],
   "source": [
    "mnist = read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f5b642361d0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnV2IbNlVx/+rq7r7dvXHre57b26GMDp5UBAREoQgREke\ngkSEqC+RAWEQER/8wqdEH4xvasAQ9EGQTCRRiQZDxvHBj1EU44MfkcSMmvgBGUjCzL1z+7s+uquq\na/vQtc79n1V7n6rurqquqrN+sDmnqrurT1fX/6y11157LQkhwHGccrBy2xfgOM7scME7TolwwTtO\niXDBO06JcME7TolwwTtOibi24EXk/SLyNRH5XxH50CQvynGc6SDXWYcXkQqA/wbwPgDfAvCvAJ4P\nIXyVvscX+B3nFgkhiH3uuhb+XQD+L4TwWgihC+CPAfzITS7OcZzpc13Bvw3AN+jxNwfPOY4zx1xX\n8O6uO84Ccl3BfwvAs/T4WVxaecdx5pjrCv6LAL5DRJ4TkTUAPw7g5cldluM406B6nR8KIfRE5OcA\n/BWACoAXOULvOM58cq1lubFe2JflHOdWmeSynOM4C4gL3nFKhAvecUqEC95xSoQL3nFKhAvecUqE\nC95xSoQL3nFKhAvecUqEC95xSoQL3nFKhAvecUqEC95xSoQL3nFKhAvecUqEC95xSoQL3nFKhAve\ncUqEC95xSoQL3nFKhAvecUqEC95xSoQL3nFKhAvecUqEC95xSoQL3nFKhAvecUqEC95xSoQL3nFK\nhAvecUqEC95xSkT1Jj8sIq8BOAFwAaAbQnjXJC7KcZzpcCPBAwgA3htCOJjExTiOM10m4dLLBF7D\ncZwZcFPBBwB/IyJfFJGfnsQFOY4zPW7q0r87hPC6iDwA8IqIfC2E8IVJXJjjOJPnRhY+hPD64Pgm\ngM8D8KCd48wx1xa8iNREZHtwvgngBwG8OqkLcxxn8tzEpX8I4PMioq/zRyGEv57IVTmOMxUkhDCd\nFxaZzgs7jjMWIYShFTTPtHOcEuGCd5wS4YJ3nBJx03X4pWIQgISIjH3OjwEgFhMZN04SQsi+N3U+\nzmvb56YVp3EWDxf8ABHBysoKVlZWcud2VCqV6PnKykpSrHxUYo/7/X7hSL2m/b08Ur/PKScueEJE\nUKlUhka1Ws2OPPi5SqUSFZwVX0qEIQT0ej1cXFwkj0XC1qE3Bj0WeQdO+XDBD1CrriJeXV0dOq6t\nreWOfF6tVnMiGyU8ewPo9/vo9XrodrvR0ev1hl7Pnvf7fYgIQggQkaHnHMcFT6jgWcw81tfXk2N1\ndXXI/baPi1z+fr+PTqeD8/NzdDqdofNOp5N7zdj5xcVFdtTX1tiC4wAu+Bxs4a3A79y5g42NjaGj\nnq+treVEx0Pd8Zh7z4I/OzvLxvn5OVZXV3F2dpbFCOxr2t8jIpnY+fVd9I7igh+gEXcreBZ3rVbD\n5uYmarXa0FhfX89ZWD63Vjcm/IuLC7Tb7dzQoKBem309K34Wtlp9F7vDuOAJdulV8Hfu3MlEvbW1\nha2tLWxubg6d37lzJxN6aqQsvAq+2Wxm04NqtZqJXbGvp2JfWVkZsuzsFbjoHcUFP0CDdhqgU+u+\nubmZE3hqXEXwwLClv7i4GAoI8lhbW0u+rgb8NOgXO/Z6vVt+h6fPqJyEove/aBVlmQKeLvgB6s6v\nrq5mLvzW1ha2t7exs7OD7e3tTPx2jHLp9Tz1YQMurbdOJdbW1oY8i7Ozs6HXtI9Z5DHhl4GixCU7\nJbLnsVUVfm4ZcMETKvj19fVsvr6zs4N6vY6dnZ3cnF3n9HrOgk8F1kZ9GPV337lzJxfA0yBebA7P\nR7uMZx8vO6PyIIo8oG63m7sx8/8NwNKI3gU/gC28uvNbW1vY2dnB3bt3Ua/Xh6LzPFZXV4c+KHbZ\nLJYZp8d+v4/19fXcchwf+QNZJPjUWn5ZBG/fbz52u93osuf5+TlWVlbQ6/VyFl9fc5liIC74ATHB\nb25uYnt7G/V6Hbu7u5n11cGPWfCpBBkgnRabSrzpdDqZYFOegz5OJe2URfCjUpPPz8+HPKd2u51L\np764uBha7VgGy6644AfYObx16ev1em5dntfpNdA2yqUExk+tjaXXFn2YrUsfG8tO7CbIN8d2u41W\nq5UNXglRS843Rl3tULd+GXDBEzGXni28BtRsmu3a2hqq1WpU2KmUWoatfCqpZpxhvYIyCj61dHlx\ncYFWq4VGo5H9v1js6sLzrkdOaFoWXPADrEtvLfzu7m40v17PK5XK0GumXMFRz6fW64vSavv9fk7o\nMdEvOyx09Yz4cbPZzMReqVQysetUysZU9GahiU/L4Nq74K+AXarhD4R+nb/X/ixbiqJzu8dez2NL\nRjxSc/eY4Mf58N6mZRt1fbFr0zhISvTV6tOPOy/TxZYtOanJLfwSom5dt9vF2dkZms0mGo0Gjo+P\nsbGxAQA5666DLfyo+TsXzYgV0tDAkX0udROI/Q3A04xB/r0rKyvZ96SWB0cV/Zg2owKbej32qOf9\nfh/VajXqztsEpdjg91pv6L1eL3vvlgEX/AB17TqdDs7OztBqtXB6epptjAkhJLfNquCLloQ0ABQr\ntFFUcKPo+/kmofCeAH2s+/pTNyN7fUU3nGm87/Y8dcPkvy82YjEQG7xTr8cGRDVYx3P6Xq+Xuf7L\nggueuLi4yATfbDYzsavV4FTX2Bx+VNJNUfUcfayDH/P3pp7Tijt8A9Bze/NJ3ZBG3VwmSSp4WTRt\nSXlCHHwrCmrGVkJ46O9hN583MC0DLvgB1qVvtVq5aG6v14vmuOvQ5ZuiPPqUoFPVdWLVdmzlnViw\nkD+gsUBUaqS8DB2TfK9jj4u8I74+6xGlbmyxGx0LPLXsqWI/Pz/P/lfLggt+gHXpm81mznJ2Op3k\nppbV1VVUKpXCElU6vywScyw2wI+tZ9HvX6bjWqsO5EXPc9xUaq4KKlazb9If+lRw01rjUTekWE3B\nUYPFnvpf6SpHatfiIuOCH6AWvtPpZHvRednm7OyssMSVCt7uWtOhgi8SdpEHocuFWmjDBuRYlKn5\nP3scmlWmx4uLi6jHwaKf5HvNRz0vSiqyN6TYUV8nFZSMRfGti2/F7nP4JcW69Gox9Ll2u51MulHB\nF21c4RhAkbhtWS1+jqPNQF7smiBSFBOwlszuo7eehz2f1PvMRz6PeR325pS6Kdnri00beH4es/Tq\nxvPN3V36JUajuPxPZsufsu6jBN/tdjP3e5Qlt4LncXZ2luXun5+fZ+e6GaRoymA9kFga76jqvJMg\nZX2BvOBjS2vsCXGQMRVYtI9j0zL7d/INctorFLeBC36ATabR+TzPf1Uc+qHTunMatBvl0ne73bHm\n6akiGKkCmprXnwrsxQRvr48FlZp2TOI91mNsrT0V8NShdQJ005IeeUkRGBZ6bDMM30istWcPwy4L\nLjoueILneXyH5+fVA7CiiLnM/GHSKP2owJ2d0/N5rIouPy6KD6jgY5Vw9DyVYzApwet7nBL+qNRY\nW1dwc3Mzs/R60+WNMAByj1nsMcGz8G0AcVlwwRM8Z9REDLYGsXmtPuagWGx5Tq1Q0Rx01A2hKGio\ngrci5fOitFu77BibetyUWGINP07dMPWom5nOz8+zm6iKnTe/jGvh+f/EsRb+ny2T2IExBC8inwTw\nwwAehxC+Z/DcHoA/AfDtAF4D8MEQwtEUr3Pq8AehSOypJBjO9IoFntTCXyfxhm8GRck/sWlATPCp\n3XSp+IQ+ntR7HBN7KqDGj3d2dnJiX1lZwerqalZPMJZ5COR3wMWmbrEgXpld+t8H8DsAPk3PfRjA\nKyGEj4rIhwaPPzyF65sp+qFT+v1+5gqzsGPprzbTK5b8cZ3UWh6pOfY4MQAVvIrdHlnwsRWCaVl4\nFpQVn40ztNvtLB6iYtedjbxWr0cAQy7+uKIvreBDCF8QkefM0x8A8J7B+acA/D2WQPC63MWZXXY+\nn9rkoj+X+jDrh27cYV/fWvzYtMK6+Fb83MWGBa/nRSsE07Dw9v1JxRZ4ycxa9lqtlpX/4jwE/f+x\n2Pl/m5rDL3vg7rpz+IchhEeD80cAHk7oem4NXhrS+bjdNaakzu1r2fNxXyf1O4uSTiqVSs4ix4p1\npFpY6eCIfywwOAlSomfBF1XfZbFrCbJOp5NNuxTOvLO/2wqe5/Bs4ZdJ6MqNg3YhhCAiS/GuxJJC\n5okid98KPmblYyIvsvB2FeCmjGPhi+bw7XY7q+Db6XRy4kz9DvbabEadejjn5+fZSAl/Wbiu4B+J\nyFtDCG+IyDMAHk/yopxiUh9qnYLEotKaPBTLALQrC/oaNi4x6eu2z3HQE7j0anTJs1qt5vIOuHKN\nTZThuIkGU0UkJ24tYKn17ZrNJlqtVnZT0ffK3lAWnesK/mUALwD4zcHxpYldkVNIbE1ZBa7zWI4p\nqJA1aJdKCuKdYnzDYPd3Utdvxa7nFp2Tq6vOBUM5GBnbwmpvLAAywXPFWhY9exB8Q1wmKz/Ostxn\ncBmguy8i3wDwqwB+A8BnReSnMFiWm+ZFOpew2AHk8udVlPZmwHPU2Pq2tfCxaLbeMCZx/amjBuNS\nW2BXVlaGqgTblFj+22ODYxgqbhU8W3idMuh7tixiB8aL0j+f+NL7JnwtzhjEPnxs5fV7dElRlxX5\nphCLRLMVt2LX15jUtceseghPMxHZanPiEbvzbN05n4GDbTYfgt35lEuv35OKESw6nmm3YFiRxJYG\ndUlKXfxYYpAdNsDFWYc6pvU3KLrWr0LXowYeNX+e3fqYS2+9Gx2xOTxb+Ha7PZSfUDqX3pkfrEvP\nz/PX1eLbYQNkMbdXbxaxnIBJ/Q2p59Wt5yBkpVKJNv2wLj3fkFjwHKuIzeG5MYUm9nCcwy28c6uk\nxG7PUzvHYt9vf47n8rPcGqpReWvpdd19lEvP12tFb5fgWPTNZhPNZjPZoXdZrDvggl9YYh/CSXww\nZ/XhTt1INMFI5+za44+Hduu1RSo4FsFZhDonVyvOQ6Py+j2p6c6y4IJ3Zor1HDj+oDsCVeTavXd7\nexvb29vY3d3Fzs4Otra2UKvVsiaeGqFXsbPrztH44+NjnJ6eZvP1WHAuNdVZFtG74J2ZYVOV7dDN\nMJw2q+26tWU3C359fT0TPJDPxz8/P8/Nz5vNJk5OTnKC5wSb2KanZRE544J3Zk5sM5LO39WNr9Vq\n2Nrawt27d7G7u5tZ9+3tbWxtbWFjYyNp4bUuoc7PT09P0Wg0cHJygkajkS3BqeBj+fPLKnoXvDNT\nYtF/PfLyG1v4er2Ovb09bG1tZfN4tfBra2vR+oOaXNNoNHB6eorj4+NCl74M7jzggndmSGobMO/3\nZwu/vb2dWfh79+6hVqtlll0Dd2zh2aXXZiLqyh8fH+dc+iILD2DphK644J2ZE0ub5d1+NminFn5j\nY2NoJ19R0M5aeHbpizbJFOX3LzoueGemxKw7p9Fq0C5m4dfX15MVeQEMWXiewx8dHWXW3QqeXXrG\nBe84N4BFbqv1ViqVnLteq9VyY3NzM7fmzkeN/HPQjptKcCadtezLuCOuCBe8MzOs627r7mlQTtfY\n7b53ju4rsZ2BsTp1ehNY9qq0o3DBOzNDs+hU8LaZhgpeLb0G5dSyqzWP7QwE8nv3rehtJZtlLmNV\nhAvemRmcG69zdXbjY4Lnlt2cL283ERVZ+FiVn2VNnR2FC96ZGdal1+CcDuvSq4WP7YhT2ELHtsRa\n194tvOPMCHbpbUadLsPxWntsDl9UNSdl5VM1/Mpm3QEXvDNDUhZes+q2t7czC8+JNSp43tMPxMVu\n98HHSl4ve7OJIlzwzszg/e48h9eecamgHa+1c3IMR+hjteZjFr7MYgdc8M6EiW1/VTg6z+68ZtTF\nluVs6qzmy7N7rueaMsvFKLkgpbv1LnhngtgWWfack2p0zs5bYFX0XODCWncubGHH/v4+Dg4OcHR0\nlKXRcs58bGnOLbzjXBMVN1eR5fONjY1CwccsvC7JaektTpvl4hbtdhuHh4fY39/PBH96ejok+FjF\n3jLhgncmRix1lodaeN3iqoE6FbwG62KCBy4tfKfTwdnZWZYT32w20Wg00Gg0cHR0hMPDw0ILbwXv\nFt5xrontcsu96avV6kiXXl15zr5jwbOFb7VamRXXnXC65513xlnBc3DP5/COcwNY8NyvXgcn2sQE\nr1F5W4qaLbwKXnfBHR4eZuP09DQ3rOBjGXbLWOSiCBe8MzFigufus6Pm8LwEx0dupGEFf3R0hIOD\nAzx58iSbs7O7z4KP1awrk9gBF7wzQewedxY9r7vzchwLngN8/FrcLNO69EdHR9jf38fjx49z+9xt\nKWp150fV5V92XPDOxFCR6pzdil2HJtbYx7aZBJDf/mr3uGvATufv3BvO9pHv9XqlEnYKF7wzMdil\nj1l2FbjtHBPrjRcrJKndYrh5BA9OsClrUG4ULnhnYhQJnvPjreDZbS9qeMmCj4lfE3BsCq0L/iku\neGdiWJc+ZuFTggfy7rvtX9/r9XKtoVjofB5rBOmCf8rIHsAi8kkReSQir9JzvyYi3xSRLw3G+6d7\nmc4iwGvwuqw2ruBtXTruD8fNH8ex8JxgU8ZsuiLGafr9+wCsoAOAj4UQ3jkYfzn5S3MWjetaeNvO\n2nZ8TYnczuVj+fJu4fOMdOlDCF8QkeciX5pdH2FnIUjN4XX9PTaH59JVwNM5vG3xHBM+W/5R3V9d\n9JeMY+FT/LyI/LuIvCgi9YldkbOwcKvnWNAuFaVnl54tfK/XyzWV0Dl8zJ1PlZ52C5/nuoL/XQBv\nB/AOAK8D+K2JXZGzkKSSbqxLr1l3thIt8HT+rkLXBBqbPcfBu9S2V3fn41wrSh9CeKznIvIJAH8+\nsSty5h5OjOHnRmXYxarY8Bq8uvI2uUaHbRV1fn6eBejKmDV3Ha5l4UXkGXr4YwBeTX2vs1yw2GON\nIccRvC1MCTx15TlQ12q1sky6o6MjHB8fJ7e8pmrdOXlGWngR+QyA9wC4LyLfAPARAO8VkXfgMlr/\ndQA/M9WrdOYCK3Z7HCX4oiU5Xn+PCd52fx1l4WM58854UfrnI09/cgrX4iwI7IZfRfBFFh54ujlG\ng3S6I04tfMyl13l7zMI7w3imnXMlYi69nscCdqk5PBenHMfCa/fXVFGL1NKbiz+PC965Mjxv58fc\nSor7uPPOuFhzCWC8Obzub4+59BYXehwXvHMjxgnaafLNOGm1NkrPglfLrst1dg7vIh+NC94ZG15r\njw112e1Q686uvLXsNrMutgYfqzXv6+1XwwXvjA1Xo9V8eR2VSiWrQjuqCyzP2XUOfnFxUbgxJpZo\n42K/Oi54Z2zUZedsOT7a7jG2MSQ3plDLrmIXkej2V5sr76mzN8MF74yN7e+u7rqe7+zsZBaexa5f\n52WzWGWbIgvP+919N9z1ccE7Y6OCt/3hdCccu/Mxl97Wg2cLb116a93Vwtu+cr4b7mq44J2xsd1f\ntdy0DrXw1qVXwfd6PQDIlYvmds5F83euPFvmRhI3xQXvjA3P4bltlFp2PaaCdjp31wqyaq1T+92t\n8Hu9Xs6iu3W/Oi54Z2zsHF4Fv7W1lXV/tRaeE25U7By0K6pqY+fw6iF4nvz1ccE7Gane7hpZt3N3\nFbs2krBLcjZvnoVuU2g1d17PNXXWRuadm+GCdzK4p3usAwzP2W3XGLXwLPbV1dVcX7jY5hhtJsH7\n3TWjzm6OcW6OC97JsN1fNSNOE21iPeFU8PV6PRedX19fz1l2W6dOs+g0dVZ3xaU2xziTwQXvZNh8\neNvuma27tfD1ej3XXUYtfKVSAZDfHMO58ip0HVbwGqhzJoML3smItXrm1s3WwluXXufsvEmGO7+q\nhWeXvtFoZF1gXfDTxwXvZMS2uLKIU3P4nZ0d1Ov1nDfAOfbA0zk8V6G1FW1i+919Dj9ZXPBOhpaZ\ntnXludR0ag5v2z3zsMtw6tJz91fe7+4Wfnq44B0Aw0UsbAqtXYpT0avY6/V4awK1zDGX3u53t3Xn\nXfCTxwVfUmypaW4iwZl0KvJarTaUK8959BsbG8mOL/pY3fnUfnfeFWf3uzuTwQVfMoqSa7hxRK1W\nG0qXrdfr2RbYWq0WjcTzZhg71ILbLjKx/u5q2T1XfrK44EtErA6dnvNOOLXssbV2FXwsuQbId4/h\njjDdbjfaLso2heSsOt/+Onlc8CXENpBQwbOF57TZ3d3dTOyaK69r7lx9li28Wmq12ppskxK9WvjU\nfncX/WRwwZcMK3I9t/N3denVsu/t7WWu/TguPTeC5KW4os0x3Nfdt79OBxd8ibAuvD1Wq9XcLjh1\n6VXwvMed02dj3WPYwnOl2SILb0tXuXWfPC74kmEtPHd81bV3tvDq0u/t7WUps1y2ahwLzzvibLCO\nz9mFd7FPBxd8yUjN362Fj7n0nGbL59bCxwQ/yrqr4AFvBjlNXPAlwW6M4Q0yat15js6Va/Scf453\n0wHDvd1tMwlda9dMOq0xz/vdfb19+rjgSwRnz9nj+vo6Hjx4gL29vSwir9F42xpKYwGaLqvnHJzj\njTG6I862e+ayVc5scMEvMbbxY6VSyaXKctXZWq2Ge/fuYW9vL1e9hgXPUX3g6ZwdQJZJp646p83G\n2j3HmkE608cFv4TYtFmFE2us6769vY3d3d3cmrstZmGz8zh1VkSiefKnp6c4Pj4e2v7abrc9V/4W\nKBS8iDwL4NMA3gIgAPi9EMJvi8gegD8B8O0AXgPwwRDC0ZSv1bkBauFZ8LrxhUtU6bkm2GgBymr1\n8qPCATUVqgo/5dIfHx/j8PAwey62/dWZDSsjvt4F8EshhO8G8H0AflZEvgvAhwG8EkL4TgB/O3js\nzAGxTTE6rIXXJbf79+/j4cOHyTk8L7/F5vC2KCVvilF3/ujoKJvD80YZt/CzpdDChxDeAPDG4Lwh\nIl8F8DYAHwDwnsG3fQrA38NFP3dY8fMcXi383t4e7t+/j/v372eJNTx4Dq+WmAN13BQiVtyCXXqb\nO+8162bP2HN4EXkOwDsB/DOAhyGER4MvPQLwcOJX5kyUlEu/u7uLBw8e4C1veUuuHh0frUvPrnxs\nOc669CcnJzg8PBzKrffluNkzluBFZAvA5wD8YgjhlC1HCCGIiIdY54DUTjgA2eYYu9edC1loV1ge\nWraK+7lriyi7QcautetjfY53z3lDyNthpOBFZBWXYv+DEMJLg6cfichbQwhviMgzAB5P8yKd8eGl\nMzt/5+KUKn5eh9fnbWKN3jRU6Ladsx51nq616bihRGzbqwt99hQG7eTyP/0igP8KIXycvvQygBcG\n5y8AeMn+rDN7bLosZ9NxyWkWu7XmXIjSrrvrxhiuSacVZw8ODrKlN11rb7VayW2vvvX1dhhl4d8N\n4CcAfEVEvjR47pcB/AaAz4rIT2GwLDe1K3TGxubGc1FJFnvMyus83Vr3mIXXve06T9dhBW8tfKzr\nq4t9toyK0v8j0l7A+yZ/Oc5NYevOXWRGiX19fX3oZ7SwhcLdXlXw3ETCLr2p4Hn5LVX3zpkNnmm3\nRMQsPG94sWK3oo+VmLY74TibjptIsHVnl95Wn/V2z7eLC37JsHN4FjvP32Pz+FjAz7r0uvymFp4z\n6TRYZy28uvQqbha5i322uOCXhNj+dhZ8KkLPLr2+Dr+mHm0jCXbpNWinVp2X5jholxK5i352uOAX\nkFT6LFetiQmae7dzw0dNm7UWmEe/34/WkufkGnXhee7u+fLzhQt+gRhVZloz47iLqz7e2NjA/fv3\nsbu7m+XJaxYd93/jZTMevV4Ph4eHubV27gVnRc5NJNyCzw8u+AWjqEQVF6Dk7jB6roK/e/dubq+7\nZtFxPXluCqGdYFTs3NqZM+lsQwkN1Lng5wcX/AIREzkH6DhPXts589D97mzhuQglR+FtFt3Z2RkO\nDw9zhSyshecWUVxy2gU/P7jgF4hYtdlYAcrNzc3c3vZ6vZ7td9fB5at0/s6ZdDpXVzG3Wq1s+U1F\nby287TTDS3HOfOCCXzDsshuvt6uF15ryWm12b28Pu7u7ud7usTk873pLZdJZC69ib7fbQ/N/7+0+\nf7jgF4hUPXmNzvNed64n/+DBA9y7dy8a0OPecDaxRtfZbSYdb5BRK99qtaKZdD6Hny9c8AuCjc7H\n1tmtS1+v13Hv3j08ePAADx48GKonr49tlN5ujuGKNRy0sxbelrzy5Jr5wwW/QHBLKJs9xw0k2HXX\ngN3Ozs5QXXkuPQ08FTxvgdU1d7XoLHDbz51xkc8nLvgFwraE4qGWXctSaWINF7GI7YIzxUyG2j1z\nUwm75OZR+MXDBb9AqHXnyjWcXKOWnZs92qo1McHbOvMseFvwwgo+Nkf3G8D84oJfINjCa1adWnR1\n4a2F59TZ2D75q1p4vQnYTq/6885844JfEHj+bufsWpuOs+vYpbcVbK7r0nNijdejW0xc8AtEUVtn\nbQ1VNIdPbX0tculV8LG0WRf84uGCXyC0GCW79JxoUxS00+o1dnmvyMLbTrBFgnfRLwYu+AUiNYfn\nls52+ytbeCC+tRZAJtyrRundwi8WLvg5IlV8AkDOlbdi17V2dumtdde19iLs/ndOkbV15N2yLyYu\n+DkiNr/WY7VazbV4ZquuqbQ2YHcVsTvlwAU/JxSVmNasOO4YY627ncOz4FPto53y4YKfI2wDCS4X\nzVH5VPosB+yu6s475cAFP0ekOsbwxpiYS6+58twEcn19PftZt/CO4oKfI+wuON7VxstwKQsfaxvl\nFt5hXPBzRMzCq4A5Z56z69jCW69ApwVu4R3FBT8nxJpA8p51rj7LFp6DdrYKDhfKcBzABT9XxATP\n7Zx5jq4Wn89jy3mxbDpGH6d6vtkiFqnnnMXABT9H2GU4m2hjM+e4gEUsNx7AkNitcDnRJlWTnrPw\nvJrNYuOCnyO0e0yRheegXKqgRcyyK7FGjmrVY/3bU1bfs+wWk8LJnYg8KyJ/JyL/KSL/ISK/MHj+\n10TkmyLypcF4/2wud7kpcumLLDxvjCkSvBU4W3EuMR0TvYt9ORhl4bsAfimE8GUR2QLwbyLyCoAA\n4GMhhI/zmaARAAAGPElEQVRN/QpLQipoxxaerbt16TUwN45Lz1ZbRW2FPsrK8+s5i0Oh4EMIbwB4\nY3DeEJGvAnjb4Mu+1jNhYkUqeVmOLbx16e3W19gRyIve1pAvculTTSb1NZ3FYOz1GhF5DsA7AfzT\n4KmfF5F/F5EXRaQ+hWsrHeNE6Yss/FXm73Y3HLv01srHAnYu8sVkLMEP3Pk/BfCLIYQGgN8F8HYA\n7wDwOoDfmtoVlojUOrydw9tKtKmAXUz4MbFb6263wdoAnwt/cRkZpReRVQCfA/CHIYSXACCE8Ji+\n/gkAfz61KywRsWaRNmvOJtTEKs/GXO0QQq4brD12Op1c3flY62db4cYFv3gUCl4uP0UvAvivEMLH\n6flnQgivDx7+GIBXp3eJzrjE1tb5sXaBtUNbS+3v7+Pg4CDrEnt6epq1geZqtV7eanEZZeHfDeAn\nAHxFRL40eO5XADwvIu/AZbT+6wB+ZnqX6IwLz89tNL7f7+c6wqol58faHVYFr62gz87OhppPeNWb\nxWRUlP4fEZ/n/8V0Lse5CTYCb5fduF9co9HIHbVpJB9PT0/RbrczC5/KwHMWB8+0WyJSSTWaWMMd\nYbUxJA/t9c6DLXwskOeiXyxc8EtCqvgk93xnC6+u++HhYTZv1waR9qgWPiZ0F/ti4YJfMqzoOTKv\nFl57vR8cHODJkyd48uQJ9vf3s2h9bHS73WTyjYt+cXDBLxHWpdeIugre9nxXwT969Ahvvvlm7mes\nh2C7xKbOnfnGBT9HxFxybgShbrbdMbeyspKts8fW2LUDbGzefnx8jJOTE5ycnAzl19vhLD4u+DlC\no+lqjbkBJO9Z16+3Wq3MWm9tbWXuuy6h8eNOp4P9/X3s7+/j8PAQJycnaDabWUCuqPiFszy44OeE\nEMJQgI3Fzr3e1DVvNBpZIcuNjY2hNFke3W4Xh4eH2Vq7FXwsZ16vy1keXPBzBFvwSqUC4OmNgOfh\nrVYLtVoNJycnucYTNjpv8+XtWjsvuaXy5fUanOXABT8nWJddLbuKXefxrVYrV3ueN9Vwsk0s8SaW\nZRcTvEfglxcX/BzR7/fR7XYBYEjsXMGW69XzzrmitNp+v5+tqetRz63g9fe74JcPF/wccXFxASAv\ndm43ZYfdQVe0Tq5WngN6fNTfrb8/dnQWH5nWP1NE/FNyRYr2tMc6y9oRWxvn52KZcny0uNAXmxDC\nUBUUt/BzhLvQzrTxliSOUyJc8I5TIlzwjlMiXPCOUyKmFqV3HGf+cAvvOCXCBe84JWImgheR94vI\n10Tkf0XkQ7P4nVdBRF4Tka8MGmP+yxxczydF5JGIvErP7YnIKyLyPyLy17fZ7SdxfXPRYLSgAepc\nvH+33aB16nN4EakA+G8A7wPwLQD/CuD5EMJXp/qLr4CIfB3A94YQDm77WgBARH4AQAPAp0MI3zN4\n7qMAnoQQPjq4ae6GED48R9f3EQCn4ZYbjIrIWwG8NVADVAA/CuAnMQfvX8H1fRAzeP9mYeHfBeD/\nQgivhRC6AP4YwI/M4PdelblpjhlC+AKAQ/P0BwB8anD+KVx+SG6FxPUBc/AehhDeCCF8eXDeAKAN\nUOfi/Su4PmAG798sBP82AN+gx9/E0z9wXggA/kZEvigiP33bF5PgYQjh0eD8EYCHt3kxCeaqwSg1\nQP1nzOH7dxsNWmch+EVY93t3COGdAH4IwM8OXNa5JVzOw+btfZ2rBqMDd/lzuGyAespfm4f377Ya\ntM5C8N8C8Cw9fhaXVn5u0D55IYQ3AXwel9OQeePRYP4HEXkGwOMR3z9TQgiPwwAAn8AtvofUAPUP\nwqABKubo/Us1aJ3F+zcLwX8RwHeIyHMisgbgxwG8PIPfOxYiUhOR7cH5JoAfxHw2x3wZwAuD8xcA\nvFTwvTNnICLl1hqMphqgYk7ev6IGrfRtU3v/ZpJpJyI/BODjACoAXgwh/PrUf+mYiMjbcWnVgcvt\nwn9029cnIp8B8B4A93E53/xVAH8G4LMAvg3AawA+GEI4mpPr+wiA9+LSHc0ajNKceZbX9v0A/gHA\nV/DUbf9lAP+COXj/Etf3KwCexwzeP0+tdZwS4Zl2jlMiXPCOUyJc8I5TIlzwjlMiXPCOUyJc8I5T\nIlzwjlMiXPCOUyL+H5z1w69ZWFRMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5b642eea50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.imshow(mnist.test.images[0].reshape([28,28]), cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55000\n",
      "[ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA6klEQVR4nGNgGDaAEYmd+P+t5vHD\naJJRBokMDAwMDAIMf9m+f7sc/hpJvuf3PxSwVxxJ5yOZS98ZGBiObGJgYGBwjlNg2B+B0KsWxItk\njtLVf/+KcTox5N+/V1AmEz6vYEhmeTIwcBpjKpQsffLkyd9///79+wARYIFLuRinKsHY81B1qe7+\n++/fv/vnznm7XP/3rxRFrvD2v0+PiyPlGRgYGE78+6iJIrny3wY7KNPgOdwrUCBUA2eG/Pu3DLvP\nGBgYuv+9s8Qld+nXvxU4NX769x6nxsjfH8NwybGe+zYXp6Espa445dAAAHuTXz5Q3jMCAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA8klEQVR4nM2SIUzDYBCFHwSHbS2V\nbS3o1SOpnq4G7Bygl6CYxw5sLQkKLkHRbW6zDXND3rsfwcZY/9+ScObe5cu7e+KAf13JjdGaMoTi\nYatU6jzy2cBoXIjMXeNDofI9AnpUj+WrhdT56AhwrDyaRUDFY5TUwFEA5SS5aK1dTwe/SJEtp+lL\n7D5OA7YRjUZrBwg44eDgni/fQgeLWlbGIhgGALKxihc1/lG1nXcXyv1G9njX8U2eNvKwsS3cB4Cz\ndLoe89fUzXaduWv6J0DSf6Bx2M0zVqpIq1Re+WHF0Wj8lJ0v2Ptu0TWqxyVuZ13bX9QXLi9y8Vjn\ngMcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA4UlEQVR4nNWPIU9CURiGn8lVuMlo\nYDL7pYqV5i+g8wdgVIqBTMNsATdNbAQdBgszsJEIjMCGm0EDAnMj3Kv3hQAoO3dns/Km7z3Pnu/b\ngT3NUUfL2amF1aXmRQzgJALL0nUCgOpnyWDeQl8OAOcTmbChILue7uWfGXCgFsRc101/6NY8OVCb\nzJMk6d0zYV7B848kSZXtm7MdUjhZus1kAXoReBPA3VtYhpeHyDc3KUq533JgwBCNbCJ9PVrZ8asu\n/5qxNpP6nlrNoSY7zTDj9O0QQutWxvKvrGZtfii7+v+sANNBUW3AwRsiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA0UlEQVR4nM3OvQ7BYBgF4IPqQgiS\nJiKxugEDNyARDAYRF+EezIYOOpTBz2CTEGNHJovFIgZrJxYD4ucwSFNp+63inb6cJ+d7X+An41dv\nWZEpXbImsKRKLtLeJmmkJguKbVIXLWw92QkKLHeg7hcVDc5TIoPJitBKnPgckb2kiunLkdmYwBEA\nkB8ZceevsRMLAELbKzl0NqUwADSWGRmIWpmF510GkXrv83ZdO+NqT5LkWnFh8f6hR9dtgEmSz3HZ\nTqRvHmz6vHjUAJjNgDf8z7wBKb5K7BwQoLgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAh0lEQVR4nGNgGHAw+5sREo8JVfIB\nhypuyUcMcbglGRh+45YMZFiO0z2GP19z4tTJxvrvO07JYFSDUCUl8Uky4JFkU2S4gVOS24phD05J\nSQaGHTjtWP5vMzNOnc4M7//ikhRnZcRpKMPGf/88cMnJPPm3G1UrkrFiUgwL/+PSafTvEB9uO+kF\nACQaG6H43b9+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAABD0lEQVR4nM3QsSvEcRjH8c+d8wsh\n3Q23G0m6Gyg3it9fYJbBZjBZGCSDYzadom7mSsp6Sec2JXeXUki3iKRTv6L3w3Bu8f2OBs/y1PfV\n83yf55H+eeSKZna2nPRQYvMVwDjwYB4DykZzwKnLf/K+kUl1F7nq/Y3z0BiTpD2OnKYNu0xL6lu4\neXJ/rBNK8WwNO/FhRsoCp/0uXtA6328Rbfd4FhkBDBY9pNyumX1ZyUPDZQOqW49vM47NRRiV1aSy\n1H5bGPFSDgNJ8fWPic5rop3Gg4fZW0lSMNmV6GD8J8cO2zZ4HMbcttHOkJSauodr5+hL8FwqRRiV\nUWfa6VrTAOprgWdPpQuFu+qKT/46vgEAdn7Lusya8wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAnElEQVR4nGNgGHhQd1UJp5zw438h\nCB4TqmS8NDIPTdKBAbekjRUeSSEhPJIMDAwvn2A6EwL2//u3E6dOLQaGDbg0Mrz8908Ol85Yfoav\nf3HoEzjw718TsgCyTgE7hl93cdmo+e/fc5zO2fXv3wJccqKn/+3hQRFBstPRmPH7HxZcDmL47/2t\nDofku08MDH+e4rI05v/5aJyupRcAAOmbKo9/IXMkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAABG0lEQVR4nMXQzysEYRzH8c9aU1JO\nHPxKW5JylKKUi3KQC4UScdHmH7DZmrPEHyAHuTi5OW5hD6tJOWxZxVLacXHxo+Zgp+0dh52nZma5\n8j19vs/reer7faT/rqGDr/3fbNUFV1puayRrxgdcbdTKiw2YAbhNy4a3uFrHQGVcsoGP3Ygls0C+\nXdLwYQ3WIjgGFIJJHgNsCmzlSHImvR+X6H+GvNlg4IX3iRAWgbRp9qAUsoUqXq43aDo9yEmSmiVJ\nKUs3U+bmeqv8HYUH0qkJiaR0dRbFSxOmbek8POwmXNRTR6YKD6kY3vVI6ttyAUYia3Zdw73jOGWA\np+2W6CfMfhKUXxxUvAqvdSvNhw4TJnQvzY1m0Uml4d3f1jf+BYdABkU+8wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAyElEQVR4nGNgGLJAefLZ7/HYpVhj\n/nyv2XAMqxxb578rrgwylrYmmHLsi/9dMmJgYOB/shtTruPfRQkGBgaGlH93udElQ/49lGRgYGBg\nKPl3Fl1O+N5nTwYGBgYGyROYkkoQIebkO/+QJFlgDLnQbwz+YXwPu8peoOtkbPj379+/f49yWZX+\nVSNEYYwwf4YXq08wMDCcYzDC9CgU8D5E2MmELikqexi3pAPDa9ySQkhsDEkGvJKMl/FI/r+NTyc+\nYz99xucMIgEAteA84zwxaYsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAABK0lEQVR4nMWQPUhCYRSGH11sKjGU\ngtKlyWhrKoqWanWoyS0awiD6QYKmGqI5qCGIxpagxrDARqcCvRZEuTSkgg5J9KPvlw2V6b17veN5\neL/znA/+OYHxnYzZX/OAz8Fmc5KRtAQJOwvlpOdCvibNYAHg/oXxUO1wpKs7mqaNR3uzpHMAxhQj\nZm/6XNcA3BWzuOzNY+UBBpOVxHezKcFsbSswdVWRqkMrjlOm9XVKat1v2dloSpJRMuwBO4xkjDGm\nPg9AttU2sN1ffzuqfLwCUG/p9Zb1FO9k15y4AZvtnh4iAKeKOkQpaBiAOd04YdGEAfDdltobwx+h\n+/pyH4CpeieASYDGJ65u8mKdwUJHOfgO1kDzs968jCQZXQIctC7t2UhL0sWi3yn0l/kE4aaA9cFo\nB+cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "width": 200
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from io import BytesIO\n",
    "import cv2\n",
    "import PIL\n",
    "import numpy as np\n",
    "from IPython.display import display, Image\n",
    "\n",
    "def display_img_array(ima, **kwargs):\n",
    "    if ima.dtype == np.float32 or ima.dtype == np.float64:\n",
    "        ima = (ima*255).astype(np.uint8)\n",
    "    im = PIL.Image.fromarray(ima)\n",
    "    bio = BytesIO()\n",
    "    im.save(bio, format='png')\n",
    "    display(Image(bio.getvalue(), format='png', **kwargs))\n",
    "print(mnist.train.num_examples)\n",
    "for i in range(10):\n",
    "    print(mnist.train.labels[i])\n",
    "    display_img_array(mnist.train.images[i].reshape([28,28]), width=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "# Interactive session (aka default session)\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tfdot import tfdot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Softmax regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(\"float\", shape=[None, 784], name=\"x\")\n",
    "y_ = tf.placeholder(\"float\", shape=[None, 10], name=\"y_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.zeros([784, 10]), name='W')\n",
    "b = tf.Variable(tf.zeros([10]), name='b')\n",
    "tf.initialize_all_variables().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"398pt\" height=\"191pt\"\n",
       " viewBox=\"0.00 0.00 398.00 191.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 187)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-187 394,-187 394,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-64 8,-139 94,-139 94,-64 8,-64\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"102,-64 102,-139 188,-139 188,-64 102,-64\"/>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- b/Assign -->\n",
       "<g id=\"node1\" class=\"node\"><title>b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"51\" cy=\"-90\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- b -->\n",
       "<g id=\"node8\" class=\"node\"><title>b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"51\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- b/Assign&#45;&gt;b -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>b/Assign&#45;&gt;b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M51,-71.6966C51,-63.9827 51,-54.7125 51,-46.1124\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"54.5001,-46.1043 51,-36.1043 47.5001,-46.1044 54.5001,-46.1043\"/>\n",
       "</g>\n",
       "<!-- W/Assign -->\n",
       "<g id=\"node2\" class=\"node\"><title>W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"145\" cy=\"-90\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- W -->\n",
       "<g id=\"node6\" class=\"node\"><title>W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"145\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- W/Assign&#45;&gt;W -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>W/Assign&#45;&gt;W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M145,-71.6966C145,-63.9827 145,-54.7125 145,-46.1124\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"148.5,-46.1043 145,-36.1043 141.5,-46.1044 148.5,-46.1043\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node3\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"219\" cy=\"-165\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"219\" y=\"-161.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node4\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"291\" cy=\"-165\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"291\" y=\"-161.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "<!-- zeros -->\n",
       "<g id=\"node5\" class=\"node\"><title>zeros</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"145\" cy=\"-165\" rx=\"29.4969\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-161.3\" font-family=\"Times,serif\" font-size=\"14.00\">zeros</text>\n",
       "</g>\n",
       "<!-- zeros&#45;&gt;W/Assign -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>zeros&#45;&gt;W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M145,-146.7C145,-138.245 145,-127.869 145,-118.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"148.5,-118.176 145,-108.176 141.5,-118.176 148.5,-118.176\"/>\n",
       "</g>\n",
       "<!-- zeros_1 -->\n",
       "<g id=\"node7\" class=\"node\"><title>zeros_1</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"51\" cy=\"-165\" rx=\"38.1938\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"51\" y=\"-161.3\" font-family=\"Times,serif\" font-size=\"14.00\">zeros_1</text>\n",
       "</g>\n",
       "<!-- zeros_1&#45;&gt;b/Assign -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>zeros_1&#45;&gt;b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51,-146.7C51,-138.245 51,-127.869 51,-118.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"54.5001,-118.176 51,-108.176 47.5001,-118.176 54.5001,-118.176\"/>\n",
       "</g>\n",
       "<!-- init -->\n",
       "<g id=\"node9\" class=\"node\"><title>init</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"363\" cy=\"-165\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"363\" y=\"-161.3\" font-family=\"Times,serif\" font-size=\"14.00\">init</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f7092459250>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y = tf.nn.softmax(tf.matmul(x,W)+b, name=\"y_softmax\")\n",
    "cross_entropy = -tf.reduce_sum(y_*tf.log(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"476pt\" height=\"853pt\"\n",
       " viewBox=\"0.00 0.00 476.00 853.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 849)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-849 472,-849 472,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_range</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-208 8,-283 152,-283 152,-208 8,-208\"/>\n",
       "<text text-anchor=\"middle\" x=\"80\" y=\"-267.8\" font-family=\"Times,serif\" font-size=\"14.00\">range</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"170,-643 170,-718 256,-718 256,-643 170,-643\"/>\n",
       "<text text-anchor=\"middle\" x=\"213\" y=\"-702.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"324,-726 324,-801 410,-801 410,-726 324,-726\"/>\n",
       "<text text-anchor=\"middle\" x=\"367\" y=\"-785.8\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- range/start -->\n",
       "<g id=\"node1\" class=\"node\"><title>range/start</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"117\" cy=\"-234\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-230.3\" font-family=\"Times,serif\" font-size=\"14.00\">start</text>\n",
       "</g>\n",
       "<!-- range -->\n",
       "<g id=\"node18\" class=\"node\"><title>range</title>\n",
       "<ellipse fill=\"#ded9ef\" stroke=\"black\" cx=\"153\" cy=\"-162\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"153\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\">range</text>\n",
       "</g>\n",
       "<!-- range/start&#45;&gt;range -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>range/start&#45;&gt;range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M125.35,-216.765C129.649,-208.404 134.992,-198.016 139.831,-188.606\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"142.985,-190.127 144.446,-179.633 136.76,-186.926 142.985,-190.127\"/>\n",
       "</g>\n",
       "<!-- range/delta -->\n",
       "<g id=\"node2\" class=\"node\"><title>range/delta</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"44\" cy=\"-234\" rx=\"27.8951\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-230.3\" font-family=\"Times,serif\" font-size=\"14.00\">delta</text>\n",
       "</g>\n",
       "<!-- range/delta&#45;&gt;range -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>range/delta&#45;&gt;range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M62.3576,-220.408C68.2786,-216.397 74.8901,-211.972 81,-208 94.9578,-198.927 110.613,-189.101 123.752,-180.953\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.729,-183.845 132.394,-175.609 122.048,-177.891 125.729,-183.845\"/>\n",
       "</g>\n",
       "<!-- b/Assign -->\n",
       "<g id=\"node3\" class=\"node\"><title>b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"213\" cy=\"-669\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"213\" y=\"-665.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- b -->\n",
       "<g id=\"node10\" class=\"node\"><title>b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"213\" cy=\"-597\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"213\" y=\"-593.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- b/Assign&#45;&gt;b -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>b/Assign&#45;&gt;b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M213,-650.697C213,-642.983 213,-633.712 213,-625.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"216.5,-625.104 213,-615.104 209.5,-625.104 216.5,-625.104\"/>\n",
       "</g>\n",
       "<!-- W/Assign -->\n",
       "<g id=\"node4\" class=\"node\"><title>W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"367\" cy=\"-752\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"367\" y=\"-748.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- W -->\n",
       "<g id=\"node8\" class=\"node\"><title>W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"367\" cy=\"-669\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"367\" y=\"-665.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- W/Assign&#45;&gt;W -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>W/Assign&#45;&gt;W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M367,-733.822C367,-723.19 367,-709.306 367,-697.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"370.5,-697.153 367,-687.153 363.5,-697.153 370.5,-697.153\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node5\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"295\" cy=\"-669\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"295\" y=\"-665.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- MatMul -->\n",
       "<g id=\"node12\" class=\"node\"><title>MatMul</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"298\" cy=\"-597\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"298\" y=\"-593.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;MatMul -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>x&#45;&gt;MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M295.742,-650.697C296.072,-642.983 296.469,-633.712 296.838,-625.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"300.335,-625.245 297.267,-615.104 293.342,-624.945 300.335,-625.245\"/>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node6\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"183\" cy=\"-381\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-377.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "<!-- mul -->\n",
       "<g id=\"node16\" class=\"node\"><title>mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"219\" cy=\"-309\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"219\" y=\"-305.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- y_&#45;&gt;mul -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>y_&#45;&gt;mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M191.35,-363.765C195.712,-355.283 201.147,-344.714 206.041,-335.197\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"209.235,-336.641 210.696,-326.147 203.01,-333.439 209.235,-336.641\"/>\n",
       "</g>\n",
       "<!-- zeros -->\n",
       "<g id=\"node7\" class=\"node\"><title>zeros</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"367\" cy=\"-827\" rx=\"29.4969\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"367\" y=\"-823.3\" font-family=\"Times,serif\" font-size=\"14.00\">zeros</text>\n",
       "</g>\n",
       "<!-- zeros&#45;&gt;W/Assign -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>zeros&#45;&gt;W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M367,-808.7C367,-800.245 367,-789.869 367,-780.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370.5,-780.176 367,-770.176 363.5,-780.176 370.5,-780.176\"/>\n",
       "</g>\n",
       "<!-- W&#45;&gt;MatMul -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>W&#45;&gt;MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M352.703,-653.496C343.555,-644.215 331.481,-631.966 321.004,-621.337\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"323.235,-618.615 313.722,-613.95 318.249,-623.529 323.235,-618.615\"/>\n",
       "</g>\n",
       "<!-- zeros_1 -->\n",
       "<g id=\"node9\" class=\"node\"><title>zeros_1</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"213\" cy=\"-752\" rx=\"38.1938\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"213\" y=\"-748.3\" font-family=\"Times,serif\" font-size=\"14.00\">zeros_1</text>\n",
       "</g>\n",
       "<!-- zeros_1&#45;&gt;b/Assign -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>zeros_1&#45;&gt;b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M213,-733.822C213,-723.19 213,-709.306 213,-697.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.5,-697.153 213,-687.153 209.5,-697.153 216.5,-697.153\"/>\n",
       "</g>\n",
       "<!-- add -->\n",
       "<g id=\"node13\" class=\"node\"><title>add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"255\" cy=\"-525\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"255\" y=\"-521.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- b&#45;&gt;add -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>b&#45;&gt;add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M222.53,-580.116C227.666,-571.557 234.122,-560.797 239.921,-551.132\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"243.072,-552.683 245.216,-542.307 237.07,-549.081 243.072,-552.683\"/>\n",
       "</g>\n",
       "<!-- init -->\n",
       "<g id=\"node11\" class=\"node\"><title>init</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"441\" cy=\"-827\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"441\" y=\"-823.3\" font-family=\"Times,serif\" font-size=\"14.00\">init</text>\n",
       "</g>\n",
       "<!-- MatMul&#45;&gt;add -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>MatMul&#45;&gt;add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M287.81,-579.411C282.54,-570.832 276.003,-560.191 270.154,-550.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"273.038,-548.677 264.821,-541.988 267.073,-552.341 273.038,-548.677\"/>\n",
       "</g>\n",
       "<!-- y_softmax -->\n",
       "<g id=\"node14\" class=\"node\"><title>y_softmax</title>\n",
       "<ellipse fill=\"#b4d7bb\" stroke=\"black\" cx=\"255\" cy=\"-453\" rx=\"48.1917\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"255\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_softmax</text>\n",
       "</g>\n",
       "<!-- add&#45;&gt;y_softmax -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>add&#45;&gt;y_softmax</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M255,-506.697C255,-498.983 255,-489.712 255,-481.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"258.5,-481.104 255,-471.104 251.5,-481.104 258.5,-481.104\"/>\n",
       "</g>\n",
       "<!-- Log -->\n",
       "<g id=\"node15\" class=\"node\"><title>Log</title>\n",
       "<ellipse fill=\"#cddce7\" stroke=\"black\" cx=\"255\" cy=\"-381\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"255\" y=\"-377.3\" font-family=\"Times,serif\" font-size=\"14.00\">Log</text>\n",
       "</g>\n",
       "<!-- y_softmax&#45;&gt;Log -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>y_softmax&#45;&gt;Log</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M255,-434.697C255,-426.983 255,-417.712 255,-409.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"258.5,-409.104 255,-399.104 251.5,-409.104 258.5,-409.104\"/>\n",
       "</g>\n",
       "<!-- Log&#45;&gt;mul -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>Log&#45;&gt;mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M246.65,-363.765C242.288,-355.283 236.853,-344.714 231.959,-335.197\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"234.99,-333.439 227.304,-326.147 228.765,-336.641 234.99,-333.439\"/>\n",
       "</g>\n",
       "<!-- Rank -->\n",
       "<g id=\"node17\" class=\"node\"><title>Rank</title>\n",
       "<ellipse fill=\"#adc6aa\" stroke=\"black\" cx=\"191\" cy=\"-234\" rx=\"29.4969\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"191\" y=\"-230.3\" font-family=\"Times,serif\" font-size=\"14.00\">Rank</text>\n",
       "</g>\n",
       "<!-- mul&#45;&gt;Rank -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>mul&#45;&gt;Rank</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M212.646,-291.435C209.234,-282.538 204.947,-271.361 201.09,-261.307\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"204.275,-259.835 197.425,-251.752 197.739,-262.342 204.275,-259.835\"/>\n",
       "</g>\n",
       "<!-- Sum -->\n",
       "<g id=\"node19\" class=\"node\"><title>Sum</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"200\" cy=\"-90\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"200\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum</text>\n",
       "</g>\n",
       "<!-- mul&#45;&gt;Sum -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>mul&#45;&gt;Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M226.58,-291.592C227.583,-288.78 228.452,-285.846 229,-283 240.334,-224.148 221.939,-154.438 209.641,-117.335\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"212.916,-116.096 206.364,-107.771 206.294,-118.365 212.916,-116.096\"/>\n",
       "</g>\n",
       "<!-- Rank&#45;&gt;range -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>Rank&#45;&gt;range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M182.187,-216.765C177.648,-208.404 172.009,-198.016 166.9,-188.606\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.877,-186.752 162.03,-179.633 163.725,-190.092 169.877,-186.752\"/>\n",
       "</g>\n",
       "<!-- range&#45;&gt;Sum -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>range&#45;&gt;Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M163.665,-145.116C169.554,-136.345 176.994,-125.264 183.606,-115.416\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"186.709,-117.074 189.377,-106.821 180.897,-113.172 186.709,-117.074\"/>\n",
       "</g>\n",
       "<!-- Neg -->\n",
       "<g id=\"node20\" class=\"node\"><title>Neg</title>\n",
       "<ellipse fill=\"#b0efb3\" stroke=\"black\" cx=\"200\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"200\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">Neg</text>\n",
       "</g>\n",
       "<!-- Sum&#45;&gt;Neg -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>Sum&#45;&gt;Neg</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M200,-71.6966C200,-63.9827 200,-54.7125 200,-46.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.5,-46.1043 200,-36.1043 196.5,-46.1044 203.5,-46.1043\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f7088030450>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"2147pt\" height=\"2499pt\"\n",
       " viewBox=\"0.00 0.00 2146.54 2499.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 2495)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-2495 2142.54,-2495 2142.54,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_range</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"548,-1854 548,-1929 692,-1929 692,-1854 548,-1854\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-1913.8\" font-family=\"Times,serif\" font-size=\"14.00\">range</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"153,-2289 153,-2364 239,-2364 239,-2289 153,-2289\"/>\n",
       "<text text-anchor=\"middle\" x=\"196\" y=\"-2348.8\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_gradients</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"478,-99 478,-1782 1794,-1782 1794,-99 478,-99\"/>\n",
       "<text text-anchor=\"middle\" x=\"1136\" y=\"-1766.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_gradients/Sum_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"641,-1271 641,-1751 1130,-1751 1130,-1271 641,-1271\"/>\n",
       "<text text-anchor=\"middle\" x=\"885.5\" y=\"-1735.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/Sum_grad</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_gradients/Sum_grad/range</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"896,-1645 896,-1720 1056,-1720 1056,-1645 896,-1645\"/>\n",
       "<text text-anchor=\"middle\" x=\"976\" y=\"-1704.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/Sum_grad/range</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_gradients/Sum_grad/Fill</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"739,-1645 739,-1720 888,-1720 888,-1645 739,-1645\"/>\n",
       "<text text-anchor=\"middle\" x=\"813.5\" y=\"-1704.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/Sum_grad/Fill</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_gradients/Sum_grad/tuple</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"820,-1279 820,-1354 1122,-1354 1122,-1279 820,-1279\"/>\n",
       "<text text-anchor=\"middle\" x=\"971\" y=\"-1338.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/Sum_grad/tuple</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_gradients/MatMul_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1119,-107 1119,-265 1637,-265 1637,-107 1119,-107\"/>\n",
       "<text text-anchor=\"middle\" x=\"1378\" y=\"-249.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/MatMul_grad</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_gradients/MatMul_grad/tuple</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1127,-115 1127,-190 1629,-190 1629,-115 1127,-115\"/>\n",
       "<text text-anchor=\"middle\" x=\"1378\" y=\"-174.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/MatMul_grad/tuple</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_gradients/mul_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1138,-980 1138,-1354 1656,-1354 1656,-980 1138,-980\"/>\n",
       "<text text-anchor=\"middle\" x=\"1397\" y=\"-1338.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/mul_grad</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_gradients/mul_grad/tuple</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1146,-988 1146,-1063 1648,-1063 1648,-988 1146,-988\"/>\n",
       "<text text-anchor=\"middle\" x=\"1397\" y=\"-1047.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/mul_grad/tuple</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_gradients/Log_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1664,-916 1664,-1063 1786,-1063 1786,-916 1664,-916\"/>\n",
       "<text text-anchor=\"middle\" x=\"1725\" y=\"-1047.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/Log_grad</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_gradients/y_softmax_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"676,-500 676,-908 962,-908 962,-500 676,-500\"/>\n",
       "<text text-anchor=\"middle\" x=\"819\" y=\"-892.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/y_softmax_grad</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_gradients/y_softmax_grad/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"746,-719 746,-794 954,-794 954,-719 746,-719\"/>\n",
       "<text text-anchor=\"middle\" x=\"850\" y=\"-778.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/y_softmax_grad/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_gradients/y_softmax_grad/Sum</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"747,-802 747,-877 935,-877 935,-802 747,-802\"/>\n",
       "<text text-anchor=\"middle\" x=\"841\" y=\"-861.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/y_softmax_grad/Sum</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_gradients/add_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"970,-273 970,-647 1488,-647 1488,-273 970,-273\"/>\n",
       "<text text-anchor=\"middle\" x=\"1229\" y=\"-631.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/add_grad</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_gradients/add_grad/tuple</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"978,-281 978,-356 1480,-356 1480,-281 978,-281\"/>\n",
       "<text text-anchor=\"middle\" x=\"1229\" y=\"-340.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/add_grad/tuple</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_gradients/Neg_grad</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"510,-1498 510,-1573 633,-1573 633,-1498 510,-1498\"/>\n",
       "<text text-anchor=\"middle\" x=\"571.5\" y=\"-1557.8\" font-family=\"Times,serif\" font-size=\"14.00\">gradients/Neg_grad</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1837,-2372 1837,-2447 1923,-2447 1923,-2372 1837,-2372\"/>\n",
       "<text text-anchor=\"middle\" x=\"1880\" y=\"-2431.8\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_GradientDescent</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-8 8,-190 432,-190 432,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"220\" y=\"-174.8\" font-family=\"Times,serif\" font-size=\"14.00\">GradientDescent</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_GradientDescent/update_W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"224,-16 224,-91 424,-91 424,-16 224,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"324\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">GradientDescent/update_W</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_GradientDescent/update_b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"16,-16 16,-91 216,-91 216,-16 16,-16\"/>\n",
       "<text text-anchor=\"middle\" x=\"116\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">GradientDescent/update_b</text>\n",
       "</g>\n",
       "<!-- range/start -->\n",
       "<g id=\"node1\" class=\"node\"><title>range/start</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"583\" cy=\"-1880\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1876.3\" font-family=\"Times,serif\" font-size=\"14.00\">start</text>\n",
       "</g>\n",
       "<!-- range -->\n",
       "<g id=\"node75\" class=\"node\"><title>range</title>\n",
       "<ellipse fill=\"#ded9ef\" stroke=\"black\" cx=\"619\" cy=\"-1808\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"619\" y=\"-1804.3\" font-family=\"Times,serif\" font-size=\"14.00\">range</text>\n",
       "</g>\n",
       "<!-- range/start&#45;&gt;range -->\n",
       "<g id=\"edge87\" class=\"edge\"><title>range/start&#45;&gt;range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M591.35,-1862.76C595.649,-1854.4 600.992,-1844.02 605.831,-1834.61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"608.985,-1836.13 610.446,-1825.63 602.76,-1832.93 608.985,-1836.13\"/>\n",
       "</g>\n",
       "<!-- range/delta -->\n",
       "<g id=\"node2\" class=\"node\"><title>range/delta</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"656\" cy=\"-1880\" rx=\"27.8951\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"656\" y=\"-1876.3\" font-family=\"Times,serif\" font-size=\"14.00\">delta</text>\n",
       "</g>\n",
       "<!-- range/delta&#45;&gt;range -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>range/delta&#45;&gt;range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M647.418,-1862.76C642.999,-1854.4 637.508,-1844.02 632.535,-1834.61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"635.559,-1832.84 627.792,-1825.63 629.371,-1836.11 635.559,-1832.84\"/>\n",
       "</g>\n",
       "<!-- b/Assign -->\n",
       "<g id=\"node3\" class=\"node\"><title>b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"196\" cy=\"-2315\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"196\" y=\"-2311.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- b -->\n",
       "<g id=\"node67\" class=\"node\"><title>b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"196\" cy=\"-2243\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"196\" y=\"-2239.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- b/Assign&#45;&gt;b -->\n",
       "<g id=\"edge84\" class=\"edge\"><title>b/Assign&#45;&gt;b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M196,-2296.7C196,-2288.98 196,-2279.71 196,-2271.11\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"199.5,-2271.1 196,-2261.1 192.5,-2271.1 199.5,-2271.1\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/range/start -->\n",
       "<g id=\"node4\" class=\"node\"><title>gradients/Sum_grad/range/start</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"931\" cy=\"-1671\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"931\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">start</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/range -->\n",
       "<g id=\"node12\" class=\"node\"><title>gradients/Sum_grad/range</title>\n",
       "<ellipse fill=\"#ded9ef\" stroke=\"black\" cx=\"931\" cy=\"-1599\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"931\" y=\"-1595.3\" font-family=\"Times,serif\" font-size=\"14.00\">range</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/range/start&#45;&gt;gradients/Sum_grad/range -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>gradients/Sum_grad/range/start&#45;&gt;gradients/Sum_grad/range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M931,-1652.7C931,-1644.98 931,-1635.71 931,-1627.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"934.5,-1627.1 931,-1617.1 927.5,-1627.1 934.5,-1627.1\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/range/delta -->\n",
       "<g id=\"node5\" class=\"node\"><title>gradients/Sum_grad/range/delta</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1004\" cy=\"-1671\" rx=\"27.8951\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1004\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">delta</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/range/delta&#45;&gt;gradients/Sum_grad/range -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>gradients/Sum_grad/range/delta&#45;&gt;gradients/Sum_grad/range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M988.874,-1655.5C978.763,-1645.8 965.272,-1632.86 953.859,-1621.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"955.998,-1619.12 946.357,-1614.73 951.153,-1624.17 955.998,-1619.12\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Fill/value -->\n",
       "<g id=\"node6\" class=\"node\"><title>gradients/Sum_grad/Fill/value</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"815\" cy=\"-1671\" rx=\"29.795\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"815\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">value</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Fill -->\n",
       "<g id=\"node13\" class=\"node\"><title>gradients/Sum_grad/Fill</title>\n",
       "<ellipse fill=\"#bbf9cf\" stroke=\"black\" cx=\"815\" cy=\"-1599\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"815\" y=\"-1595.3\" font-family=\"Times,serif\" font-size=\"14.00\">Fill</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Fill/value&#45;&gt;gradients/Sum_grad/Fill -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>gradients/Sum_grad/Fill/value&#45;&gt;gradients/Sum_grad/Fill</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M815,-1652.7C815,-1644.98 815,-1635.71 815,-1627.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"818.5,-1627.1 815,-1617.1 811.5,-1627.1 818.5,-1627.1\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/tuple/group_deps -->\n",
       "<g id=\"node7\" class=\"node\"><title>gradients/Sum_grad/tuple/group_deps</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"1062\" cy=\"-1305\" rx=\"51.9908\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1062\" y=\"-1301.3\" font-family=\"Times,serif\" font-size=\"14.00\">group_deps</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/tuple/control_dependency -->\n",
       "<g id=\"node8\" class=\"node\"><title>gradients/Sum_grad/tuple/control_dependency</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"910\" cy=\"-1305\" rx=\"81.7856\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"910\" y=\"-1301.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/mul -->\n",
       "<g id=\"node29\" class=\"node\"><title>gradients/mul_grad/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"1173\" cy=\"-1233\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1173\" y=\"-1229.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/tuple/control_dependency&#45;&gt;gradients/mul_grad/mul -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>gradients/Sum_grad/tuple/control_dependency&#45;&gt;gradients/mul_grad/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M948.142,-1288.86C964.284,-1282.79 983.395,-1276.06 1001,-1271 1047.29,-1257.69 1101.81,-1246.74 1136.86,-1240.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1137.87,-1243.67 1147.09,-1238.44 1136.62,-1236.78 1137.87,-1243.67\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/mul_1 -->\n",
       "<g id=\"node32\" class=\"node\"><title>gradients/mul_grad/mul_1</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"1477\" cy=\"-1233\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1477\" y=\"-1229.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul_1</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/tuple/control_dependency&#45;&gt;gradients/mul_grad/mul_1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>gradients/Sum_grad/tuple/control_dependency&#45;&gt;gradients/mul_grad/mul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M944.727,-1288.61C961.406,-1281.95 981.883,-1274.81 1001,-1271 1190.36,-1233.21 1246.37,-1292.26 1435,-1251 1437.65,-1250.42 1440.34,-1249.69 1443.02,-1248.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1444.3,-1252.12 1452.57,-1245.5 1441.97,-1245.52 1444.3,-1252.12\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Shape -->\n",
       "<g id=\"node9\" class=\"node\"><title>gradients/Sum_grad/Shape</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"1012\" cy=\"-1599\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1012\" y=\"-1595.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/DynamicStitch -->\n",
       "<g id=\"node14\" class=\"node\"><title>gradients/Sum_grad/DynamicStitch</title>\n",
       "<ellipse fill=\"#a99cf6\" stroke=\"black\" cx=\"916\" cy=\"-1524\" rx=\"63.8893\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"916\" y=\"-1520.3\" font-family=\"Times,serif\" font-size=\"14.00\">DynamicStitch</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Shape&#45;&gt;gradients/Sum_grad/DynamicStitch -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>gradients/Sum_grad/Shape&#45;&gt;gradients/Sum_grad/DynamicStitch</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M993.49,-1583.92C979.883,-1573.58 961.112,-1559.3 945.454,-1547.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"947.206,-1544.33 937.127,-1541.07 942.969,-1549.9 947.206,-1544.33\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/div -->\n",
       "<g id=\"node15\" class=\"node\"><title>gradients/Sum_grad/div</title>\n",
       "<ellipse fill=\"#abafb2\" stroke=\"black\" cx=\"956\" cy=\"-1452\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"956\" y=\"-1448.3\" font-family=\"Times,serif\" font-size=\"14.00\">div</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Shape&#45;&gt;gradients/Sum_grad/div -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>gradients/Sum_grad/Shape&#45;&gt;gradients/Sum_grad/div</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1010.18,-1580.69C1007.54,-1560.19 1001.53,-1525.37 989,-1498 985.483,-1490.32 980.534,-1482.65 975.536,-1475.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"978.187,-1473.59 969.284,-1467.84 972.663,-1477.88 978.187,-1473.59\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Rank -->\n",
       "<g id=\"node10\" class=\"node\"><title>gradients/Sum_grad/Rank</title>\n",
       "<ellipse fill=\"#adc6aa\" stroke=\"black\" cx=\"1093\" cy=\"-1671\" rx=\"29.4969\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1093\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">Rank</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Rank&#45;&gt;gradients/Sum_grad/range -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>gradients/Sum_grad/Rank&#45;&gt;gradients/Sum_grad/range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1076.62,-1655.84C1071.53,-1651.94 1065.75,-1647.97 1060,-1645 1023.15,-1625.98 1009.91,-1631.35 971,-1617 968.79,-1616.18 966.524,-1615.32 964.249,-1614.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"965.543,-1611.17 954.962,-1610.63 962.899,-1617.65 965.543,-1611.17\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Shape_1 -->\n",
       "<g id=\"node11\" class=\"node\"><title>gradients/Sum_grad/Shape_1</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"690\" cy=\"-1671\" rx=\"40.8928\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"690\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape_1</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Shape_1&#45;&gt;gradients/Sum_grad/Fill -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>gradients/Sum_grad/Shape_1&#45;&gt;gradients/Sum_grad/Fill</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M714.398,-1656.34C734.914,-1644.85 764.286,-1628.4 786,-1616.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"787.775,-1619.26 794.79,-1611.32 784.354,-1613.15 787.775,-1619.26\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/range&#45;&gt;gradients/Sum_grad/DynamicStitch -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>gradients/Sum_grad/range&#45;&gt;gradients/Sum_grad/DynamicStitch</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M927.521,-1581.07C925.754,-1572.47 923.564,-1561.81 921.57,-1552.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"924.953,-1551.18 919.512,-1542.09 918.097,-1552.59 924.953,-1551.18\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Fill&#45;&gt;gradients/Sum_grad/DynamicStitch -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>gradients/Sum_grad/Fill&#45;&gt;gradients/Sum_grad/DynamicStitch</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M832.593,-1585.28C847.164,-1574.75 868.209,-1559.54 885.513,-1547.03\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"887.609,-1549.84 893.663,-1541.14 883.508,-1544.17 887.609,-1549.84\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/DynamicStitch&#45;&gt;gradients/Sum_grad/div -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>gradients/Sum_grad/DynamicStitch&#45;&gt;gradients/Sum_grad/div</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M925.683,-1506.05C930.469,-1497.68 936.341,-1487.4 941.638,-1478.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"944.759,-1479.73 946.682,-1469.31 938.681,-1476.25 944.759,-1479.73\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Reshape -->\n",
       "<g id=\"node16\" class=\"node\"><title>gradients/Sum_grad/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"871\" cy=\"-1452\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"871\" y=\"-1448.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/DynamicStitch&#45;&gt;gradients/Sum_grad/Reshape -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>gradients/Sum_grad/DynamicStitch&#45;&gt;gradients/Sum_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M905.107,-1506.05C899.745,-1497.71 893.173,-1487.49 887.233,-1478.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"890.145,-1476.31 881.793,-1469.79 884.256,-1480.09 890.145,-1476.31\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Tile -->\n",
       "<g id=\"node17\" class=\"node\"><title>gradients/Sum_grad/Tile</title>\n",
       "<ellipse fill=\"#aed5d3\" stroke=\"black\" cx=\"910\" cy=\"-1380\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"910\" y=\"-1376.3\" font-family=\"Times,serif\" font-size=\"14.00\">Tile</text>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/div&#45;&gt;gradients/Sum_grad/Tile -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>gradients/Sum_grad/div&#45;&gt;gradients/Sum_grad/Tile</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M945.562,-1435.12C939.798,-1426.34 932.516,-1415.26 926.045,-1405.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"928.814,-1403.26 920.397,-1396.82 922.964,-1407.1 928.814,-1403.26\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Reshape&#45;&gt;gradients/Sum_grad/Tile -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>gradients/Sum_grad/Reshape&#45;&gt;gradients/Sum_grad/Tile</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M880.242,-1434.41C884.905,-1426.04 890.662,-1415.71 895.865,-1406.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"899.014,-1407.91 900.824,-1397.47 892.899,-1404.5 899.014,-1407.91\"/>\n",
       "</g>\n",
       "<!-- gradients/Sum_grad/Tile&#45;&gt;gradients/Sum_grad/tuple/control_dependency -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>gradients/Sum_grad/Tile&#45;&gt;gradients/Sum_grad/tuple/control_dependency</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M910,-1361.7C910,-1353.25 910,-1342.87 910,-1333.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"913.5,-1333.18 910,-1323.18 906.5,-1333.18 913.5,-1333.18\"/>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/tuple/group_deps -->\n",
       "<g id=\"node18\" class=\"node\"><title>gradients/MatMul_grad/tuple/group_deps</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"1569\" cy=\"-141\" rx=\"51.9908\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1569\" y=\"-137.3\" font-family=\"Times,serif\" font-size=\"14.00\">group_deps</text>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/tuple/control_dependency -->\n",
       "<g id=\"node19\" class=\"node\"><title>gradients/MatMul_grad/tuple/control_dependency</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"1417\" cy=\"-141\" rx=\"81.7856\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1417\" y=\"-137.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency</text>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/tuple/control_dependency_1 -->\n",
       "<g id=\"node20\" class=\"node\"><title>gradients/MatMul_grad/tuple/control_dependency_1</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"1226\" cy=\"-141\" rx=\"90.9839\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1226\" y=\"-137.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency_1</text>\n",
       "</g>\n",
       "<!-- GradientDescent/update_W/ApplyGradientDescent -->\n",
       "<g id=\"node59\" class=\"node\"><title>GradientDescent/update_W/ApplyGradientDescent</title>\n",
       "<ellipse fill=\"#e9ddb1\" stroke=\"black\" cx=\"324\" cy=\"-42\" rx=\"91.784\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"324\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">ApplyGradientDescent</text>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/tuple/control_dependency_1&#45;&gt;GradientDescent/update_W/ApplyGradientDescent -->\n",
       "<g id=\"edge91\" class=\"edge\"><title>gradients/MatMul_grad/tuple/control_dependency_1&#45;&gt;GradientDescent/update_W/ApplyGradientDescent</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1147.92,-131.604C980.624,-113.613 588.5,-71.4441 412.382,-52.5045\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"412.69,-49.0175 402.373,-51.4282 411.942,-55.9774 412.69,-49.0175\"/>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/MatMul -->\n",
       "<g id=\"node21\" class=\"node\"><title>gradients/MatMul_grad/MatMul</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"1417\" cy=\"-216\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1417\" y=\"-212.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul</text>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/MatMul&#45;&gt;gradients/MatMul_grad/tuple/control_dependency -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>gradients/MatMul_grad/MatMul&#45;&gt;gradients/MatMul_grad/tuple/control_dependency</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1417,-197.7C1417,-189.245 1417,-178.869 1417,-169.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1420.5,-169.176 1417,-159.176 1413.5,-169.176 1420.5,-169.176\"/>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/MatMul_1 -->\n",
       "<g id=\"node22\" class=\"node\"><title>gradients/MatMul_grad/MatMul_1</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"1268\" cy=\"-216\" rx=\"49.2915\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1268\" y=\"-212.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul_1</text>\n",
       "</g>\n",
       "<!-- gradients/MatMul_grad/MatMul_1&#45;&gt;gradients/MatMul_grad/tuple/control_dependency_1 -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>gradients/MatMul_grad/MatMul_1&#45;&gt;gradients/MatMul_grad/tuple/control_dependency_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1258.26,-198.069C1253.1,-189.104 1246.66,-177.903 1240.89,-167.875\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1243.85,-166.015 1235.83,-159.092 1237.79,-169.505 1243.85,-166.015\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/tuple/group_deps -->\n",
       "<g id=\"node23\" class=\"node\"><title>gradients/mul_grad/tuple/group_deps</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"1588\" cy=\"-1014\" rx=\"51.9908\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1588\" y=\"-1010.3\" font-family=\"Times,serif\" font-size=\"14.00\">group_deps</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/tuple/control_dependency -->\n",
       "<g id=\"node24\" class=\"node\"><title>gradients/mul_grad/tuple/control_dependency</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"1436\" cy=\"-1014\" rx=\"81.7856\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1436\" y=\"-1010.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/tuple/control_dependency_1 -->\n",
       "<g id=\"node25\" class=\"node\"><title>gradients/mul_grad/tuple/control_dependency_1</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"1245\" cy=\"-1014\" rx=\"90.9839\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1245\" y=\"-1010.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency_1</text>\n",
       "</g>\n",
       "<!-- gradients/Log_grad/mul -->\n",
       "<g id=\"node36\" class=\"node\"><title>gradients/Log_grad/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"1699\" cy=\"-942\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1699\" y=\"-938.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/tuple/control_dependency_1&#45;&gt;gradients/Log_grad/mul -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>gradients/mul_grad/tuple/control_dependency_1&#45;&gt;gradients/Log_grad/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1284.84,-997.671C1302.95,-991.273 1324.8,-984.336 1345,-980 1458.89,-955.553 1597,-947.047 1661.9,-944.262\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1662.2,-947.753 1672.05,-943.848 1661.91,-940.758 1662.2,-947.753\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Shape -->\n",
       "<g id=\"node26\" class=\"node\"><title>gradients/mul_grad/Shape</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"1354\" cy=\"-1305\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1354\" y=\"-1301.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/BroadcastGradientArgs -->\n",
       "<g id=\"node28\" class=\"node\"><title>gradients/mul_grad/BroadcastGradientArgs</title>\n",
       "<ellipse fill=\"#c7becf\" stroke=\"black\" cx=\"1332\" cy=\"-1233\" rx=\"93.6835\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1332\" y=\"-1229.3\" font-family=\"Times,serif\" font-size=\"14.00\">BroadcastGradientArgs</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Shape&#45;&gt;gradients/mul_grad/BroadcastGradientArgs -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>gradients/mul_grad/Shape&#45;&gt;gradients/mul_grad/BroadcastGradientArgs</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1348.67,-1287.05C1346.22,-1279.26 1343.26,-1269.82 1340.51,-1261.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1343.77,-1259.77 1337.43,-1251.28 1337.09,-1261.86 1343.77,-1259.77\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Reshape -->\n",
       "<g id=\"node31\" class=\"node\"><title>gradients/mul_grad/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"1436\" cy=\"-1089\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1436\" y=\"-1085.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Shape&#45;&gt;gradients/mul_grad/Reshape -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>gradients/mul_grad/Shape&#45;&gt;gradients/mul_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1386.66,-1304.17C1425.67,-1302.39 1489.49,-1292.77 1520,-1251 1553.54,-1205.08 1498.13,-1143.95 1462.45,-1111.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1464.57,-1109.03 1454.74,-1105.07 1459.95,-1114.29 1464.57,-1109.03\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Shape_1 -->\n",
       "<g id=\"node27\" class=\"node\"><title>gradients/mul_grad/Shape_1</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"1219\" cy=\"-1305\" rx=\"40.8928\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1219\" y=\"-1301.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape_1</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Shape_1&#45;&gt;gradients/mul_grad/BroadcastGradientArgs -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>gradients/mul_grad/Shape_1&#45;&gt;gradients/mul_grad/BroadcastGradientArgs</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1241.87,-1289.83C1257.72,-1280.02 1279.12,-1266.76 1297.09,-1255.62\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1298.94,-1258.6 1305.6,-1250.35 1295.26,-1252.64 1298.94,-1258.6\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Reshape_1 -->\n",
       "<g id=\"node34\" class=\"node\"><title>gradients/mul_grad/Reshape_1</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"1245\" cy=\"-1089\" rx=\"49.2915\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1245\" y=\"-1085.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape_1</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Shape_1&#45;&gt;gradients/mul_grad/Reshape_1 -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>gradients/mul_grad/Shape_1&#45;&gt;gradients/mul_grad/Reshape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1220.84,-1286.93C1222.79,-1269.01 1225.99,-1240.01 1229,-1215 1233.04,-1181.36 1238.05,-1142.67 1241.38,-1117.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1244.85,-1117.75 1242.69,-1107.38 1237.91,-1116.84 1244.85,-1117.75\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Sum -->\n",
       "<g id=\"node30\" class=\"node\"><title>gradients/mul_grad/Sum</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"1298\" cy=\"-1161\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1298\" y=\"-1157.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/BroadcastGradientArgs&#45;&gt;gradients/mul_grad/Sum -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>gradients/mul_grad/BroadcastGradientArgs&#45;&gt;gradients/mul_grad/Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1323.77,-1215.05C1319.74,-1206.77 1314.81,-1196.62 1310.35,-1187.42\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1313.44,-1185.77 1305.92,-1178.31 1307.14,-1188.83 1313.44,-1185.77\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Sum_1 -->\n",
       "<g id=\"node33\" class=\"node\"><title>gradients/mul_grad/Sum_1</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"1380\" cy=\"-1161\" rx=\"36.2938\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1380\" y=\"-1157.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum_1</text>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/BroadcastGradientArgs&#45;&gt;gradients/mul_grad/Sum_1 -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>gradients/mul_grad/BroadcastGradientArgs&#45;&gt;gradients/mul_grad/Sum_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1343.62,-1215.05C1349.48,-1206.5 1356.7,-1195.97 1363.16,-1186.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1366.05,-1188.53 1368.82,-1178.31 1360.28,-1184.57 1366.05,-1188.53\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/mul&#45;&gt;gradients/mul_grad/Sum -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>gradients/mul_grad/mul&#45;&gt;gradients/mul_grad/Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1193.08,-1220.75C1213.55,-1209.29 1245.52,-1191.39 1268.79,-1178.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1270.54,-1181.39 1277.55,-1173.45 1267.12,-1175.28 1270.54,-1181.39\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Sum&#45;&gt;gradients/mul_grad/Reshape -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>gradients/mul_grad/Sum&#45;&gt;gradients/mul_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1318.96,-1149.37C1340.72,-1138.33 1375.13,-1120.88 1401.02,-1107.75\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1402.79,-1110.77 1410.13,-1103.12 1399.63,-1104.53 1402.79,-1110.77\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Reshape&#45;&gt;gradients/mul_grad/tuple/control_dependency -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>gradients/mul_grad/Reshape&#45;&gt;gradients/mul_grad/tuple/control_dependency</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1436,-1070.7C1436,-1062.25 1436,-1051.87 1436,-1042.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1439.5,-1042.18 1436,-1032.18 1432.5,-1042.18 1439.5,-1042.18\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/mul_1&#45;&gt;gradients/mul_grad/Sum_1 -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>gradients/mul_grad/mul_1&#45;&gt;gradients/mul_grad/Sum_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1457.84,-1218.17C1443.67,-1207.95 1424.17,-1193.87 1408.22,-1182.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1409.95,-1179.3 1399.79,-1176.28 1405.85,-1184.97 1409.95,-1179.3\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Sum_1&#45;&gt;gradients/mul_grad/Reshape_1 -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>gradients/mul_grad/Sum_1&#45;&gt;gradients/mul_grad/Reshape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1355.86,-1147.49C1335.11,-1136.72 1304.87,-1121.04 1281.21,-1108.78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1282.66,-1105.59 1272.17,-1104.09 1279.44,-1111.8 1282.66,-1105.59\"/>\n",
       "</g>\n",
       "<!-- gradients/mul_grad/Reshape_1&#45;&gt;gradients/mul_grad/tuple/control_dependency_1 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>gradients/mul_grad/Reshape_1&#45;&gt;gradients/mul_grad/tuple/control_dependency_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1245,-1070.7C1245,-1062.25 1245,-1051.87 1245,-1042.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1248.5,-1042.18 1245,-1032.18 1241.5,-1042.18 1248.5,-1042.18\"/>\n",
       "</g>\n",
       "<!-- gradients/Log_grad/Inv -->\n",
       "<g id=\"node35\" class=\"node\"><title>gradients/Log_grad/Inv</title>\n",
       "<ellipse fill=\"#e9e2d0\" stroke=\"black\" cx=\"1699\" cy=\"-1014\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1699\" y=\"-1010.3\" font-family=\"Times,serif\" font-size=\"14.00\">Inv</text>\n",
       "</g>\n",
       "<!-- gradients/Log_grad/Inv&#45;&gt;gradients/Log_grad/mul -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>gradients/Log_grad/Inv&#45;&gt;gradients/Log_grad/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1699,-995.697C1699,-987.983 1699,-978.712 1699,-970.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1702.5,-970.104 1699,-960.104 1695.5,-970.104 1702.5,-970.104\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/mul -->\n",
       "<g id=\"node39\" class=\"node\"><title>gradients/y_softmax_grad/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"711\" cy=\"-828\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"711\" y=\"-824.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- gradients/Log_grad/mul&#45;&gt;gradients/y_softmax_grad/mul -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>gradients/Log_grad/mul&#45;&gt;gradients/y_softmax_grad/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1671.64,-940.89C1519.41,-940.174 782.043,-935.301 743,-908 726.172,-896.233 718.185,-874.011 714.399,-856.125\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"717.83,-855.433 712.62,-846.208 710.94,-856.668 717.83,-855.433\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/sub -->\n",
       "<g id=\"node42\" class=\"node\"><title>gradients/y_softmax_grad/sub</title>\n",
       "<ellipse fill=\"#b2edd8\" stroke=\"black\" cx=\"920\" cy=\"-598\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"920\" y=\"-594.3\" font-family=\"Times,serif\" font-size=\"14.00\">sub</text>\n",
       "</g>\n",
       "<!-- gradients/Log_grad/mul&#45;&gt;gradients/y_softmax_grad/sub -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>gradients/Log_grad/mul&#45;&gt;gradients/y_softmax_grad/sub</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1673.76,-935.048C1576,-911.177 1215.4,-815.456 966,-647 955.894,-640.174 946.295,-630.806 938.505,-622.158\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"941.084,-619.789 931.897,-614.511 935.787,-624.365 941.084,-619.789\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Reshape/shape -->\n",
       "<g id=\"node37\" class=\"node\"><title>gradients/y_softmax_grad/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"851\" cy=\"-745\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"851\" y=\"-741.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Reshape -->\n",
       "<g id=\"node41\" class=\"node\"><title>gradients/y_softmax_grad/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"851\" cy=\"-673\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"851\" y=\"-669.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Reshape/shape&#45;&gt;gradients/y_softmax_grad/Reshape -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>gradients/y_softmax_grad/Reshape/shape&#45;&gt;gradients/y_softmax_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M851,-726.697C851,-718.983 851,-709.712 851,-701.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"854.5,-701.104 851,-691.104 847.5,-701.104 854.5,-701.104\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Sum/reduction_indices -->\n",
       "<g id=\"node38\" class=\"node\"><title>gradients/y_softmax_grad/Sum/reduction_indices</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"829\" cy=\"-828\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"829\" y=\"-824.3\" font-family=\"Times,serif\" font-size=\"14.00\">reduction_indices</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Sum -->\n",
       "<g id=\"node40\" class=\"node\"><title>gradients/y_softmax_grad/Sum</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"711\" cy=\"-745\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"711\" y=\"-741.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Sum/reduction_indices&#45;&gt;gradients/y_softmax_grad/Sum -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>gradients/y_softmax_grad/Sum/reduction_indices&#45;&gt;gradients/y_softmax_grad/Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M780.055,-814.565C766.789,-809.631 753.054,-802.938 742,-794 734.49,-787.927 728.208,-779.531 723.307,-771.457\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"726.302,-769.641 718.368,-762.619 720.191,-773.056 726.302,-769.641\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/mul&#45;&gt;gradients/y_softmax_grad/Sum -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>gradients/y_softmax_grad/mul&#45;&gt;gradients/y_softmax_grad/Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M711,-809.822C711,-799.19 711,-785.306 711,-773.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"714.5,-773.153 711,-763.153 707.5,-773.153 714.5,-773.153\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Sum&#45;&gt;gradients/y_softmax_grad/Reshape -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>gradients/y_softmax_grad/Sum&#45;&gt;gradients/y_softmax_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M726.447,-730.1C731.227,-726.205 736.647,-722.173 742,-719 763.285,-706.383 788.767,-695.649 809.739,-687.833\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"811.129,-691.051 819.329,-684.343 808.735,-684.473 811.129,-691.051\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/Reshape&#45;&gt;gradients/y_softmax_grad/sub -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>gradients/y_softmax_grad/Reshape&#45;&gt;gradients/y_softmax_grad/sub</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M865.972,-656.16C875.633,-645.939 888.341,-632.495 899.004,-621.213\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"901.672,-623.485 905.998,-613.814 896.585,-618.677 901.672,-623.485\"/>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/mul_1 -->\n",
       "<g id=\"node43\" class=\"node\"><title>gradients/y_softmax_grad/mul_1</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"920\" cy=\"-526\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"920\" y=\"-522.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul_1</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/sub&#45;&gt;gradients/y_softmax_grad/mul_1 -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>gradients/y_softmax_grad/sub&#45;&gt;gradients/y_softmax_grad/mul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M920,-579.697C920,-571.983 920,-562.712 920,-554.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"923.5,-554.104 920,-544.104 916.5,-554.104 923.5,-554.104\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Sum -->\n",
       "<g id=\"node50\" class=\"node\"><title>gradients/add_grad/Sum</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"1157\" cy=\"-454\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1157\" y=\"-450.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/mul_1&#45;&gt;gradients/add_grad/Sum -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>gradients/y_softmax_grad/mul_1&#45;&gt;gradients/add_grad/Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M940.877,-511.83C948.533,-507.497 957.427,-503.021 966,-500 1031.61,-476.877 1053.65,-492.908 1120,-472 1122.14,-471.327 1124.31,-470.56 1126.48,-469.732\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1127.96,-472.908 1135.84,-465.824 1125.26,-466.448 1127.96,-472.908\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Sum_1 -->\n",
       "<g id=\"node52\" class=\"node\"><title>gradients/add_grad/Sum_1</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"1075\" cy=\"-454\" rx=\"36.2938\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1075\" y=\"-450.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum_1</text>\n",
       "</g>\n",
       "<!-- gradients/y_softmax_grad/mul_1&#45;&gt;gradients/add_grad/Sum_1 -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>gradients/y_softmax_grad/mul_1&#45;&gt;gradients/add_grad/Sum_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M942.214,-512.309C949.685,-508.208 958.123,-503.754 966,-500 989.4,-488.85 1016.32,-477.722 1037.55,-469.321\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1038.84,-472.575 1046.87,-465.663 1036.28,-466.058 1038.84,-472.575\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/tuple/group_deps -->\n",
       "<g id=\"node44\" class=\"node\"><title>gradients/add_grad/tuple/group_deps</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"1420\" cy=\"-307\" rx=\"51.9908\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1420\" y=\"-303.3\" font-family=\"Times,serif\" font-size=\"14.00\">group_deps</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/tuple/control_dependency -->\n",
       "<g id=\"node45\" class=\"node\"><title>gradients/add_grad/tuple/control_dependency</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"1268\" cy=\"-307\" rx=\"81.7856\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1268\" y=\"-303.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/tuple/control_dependency&#45;&gt;gradients/MatMul_grad/MatMul -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>gradients/add_grad/tuple/control_dependency&#45;&gt;gradients/MatMul_grad/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1294.98,-289.883C1320.3,-274.761 1358.18,-252.135 1385.01,-236.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1386.86,-239.082 1393.65,-230.949 1383.27,-233.072 1386.86,-239.082\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/tuple/control_dependency&#45;&gt;gradients/MatMul_grad/MatMul_1 -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>gradients/add_grad/tuple/control_dependency&#45;&gt;gradients/MatMul_grad/MatMul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1268,-288.84C1268,-276.281 1268,-258.979 1268,-244.502\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1271.5,-244.107 1268,-234.107 1264.5,-244.107 1271.5,-244.107\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/tuple/control_dependency_1 -->\n",
       "<g id=\"node46\" class=\"node\"><title>gradients/add_grad/tuple/control_dependency_1</title>\n",
       "<ellipse fill=\"#ceaaf0\" stroke=\"black\" cx=\"1077\" cy=\"-307\" rx=\"90.9839\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1077\" y=\"-303.3\" font-family=\"Times,serif\" font-size=\"14.00\">control_dependency_1</text>\n",
       "</g>\n",
       "<!-- GradientDescent/update_b/ApplyGradientDescent -->\n",
       "<g id=\"node60\" class=\"node\"><title>GradientDescent/update_b/ApplyGradientDescent</title>\n",
       "<ellipse fill=\"#e9ddb1\" stroke=\"black\" cx=\"116\" cy=\"-42\" rx=\"91.784\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"116\" y=\"-38.3\" font-family=\"Times,serif\" font-size=\"14.00\">ApplyGradientDescent</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/tuple/control_dependency_1&#45;&gt;GradientDescent/update_b/ApplyGradientDescent -->\n",
       "<g id=\"edge78\" class=\"edge\"><title>gradients/add_grad/tuple/control_dependency_1&#45;&gt;GradientDescent/update_b/ApplyGradientDescent</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1012.91,-294.184C868.748,-266.689 508.958,-192.924 220,-91 198.243,-83.3257 174.847,-72.749 155.765,-63.4768\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"157.288,-60.3254 146.771,-59.0481 154.196,-66.6055 157.288,-60.3254\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Shape -->\n",
       "<g id=\"node47\" class=\"node\"><title>gradients/add_grad/Shape</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"1262\" cy=\"-598\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1262\" y=\"-594.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/BroadcastGradientArgs -->\n",
       "<g id=\"node49\" class=\"node\"><title>gradients/add_grad/BroadcastGradientArgs</title>\n",
       "<ellipse fill=\"#c7becf\" stroke=\"black\" cx=\"1144\" cy=\"-526\" rx=\"93.6835\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1144\" y=\"-522.3\" font-family=\"Times,serif\" font-size=\"14.00\">BroadcastGradientArgs</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Shape&#45;&gt;gradients/add_grad/BroadcastGradientArgs -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>gradients/add_grad/Shape&#45;&gt;gradients/add_grad/BroadcastGradientArgs</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1240.63,-584.324C1223.81,-574.347 1200,-560.217 1180.21,-548.483\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1181.84,-545.375 1171.45,-543.284 1178.26,-551.396 1181.84,-545.375\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Reshape -->\n",
       "<g id=\"node51\" class=\"node\"><title>gradients/add_grad/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"1262\" cy=\"-382\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1262\" y=\"-378.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Shape&#45;&gt;gradients/add_grad/Reshape -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>gradients/add_grad/Shape&#45;&gt;gradients/add_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1262,-579.849C1262,-542.832 1262,-455.181 1262,-410.386\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1265.5,-410.232 1262,-400.232 1258.5,-410.232 1265.5,-410.232\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Shape_1 -->\n",
       "<g id=\"node48\" class=\"node\"><title>gradients/add_grad/Shape_1</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"1020\" cy=\"-598\" rx=\"40.8928\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1020\" y=\"-594.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape_1</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Shape_1&#45;&gt;gradients/add_grad/BroadcastGradientArgs -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>gradients/add_grad/Shape_1&#45;&gt;gradients/add_grad/BroadcastGradientArgs</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1044.2,-583.337C1061.81,-573.4 1085.98,-559.753 1106.13,-548.379\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1108.08,-551.298 1115.06,-543.335 1104.64,-545.203 1108.08,-551.298\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Reshape_1 -->\n",
       "<g id=\"node53\" class=\"node\"><title>gradients/add_grad/Reshape_1</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"1075\" cy=\"-382\" rx=\"49.2915\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1075\" y=\"-378.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape_1</text>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Shape_1&#45;&gt;gradients/add_grad/Reshape_1 -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>gradients/add_grad/Shape_1&#45;&gt;gradients/add_grad/Reshape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1017.63,-579.777C1014.2,-549.453 1010.22,-485.095 1030,-436 1034.42,-425.028 1042.05,-414.689 1049.79,-406.107\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1052.44,-408.395 1056.83,-398.752 1047.39,-403.553 1052.44,-408.395\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/BroadcastGradientArgs&#45;&gt;gradients/add_grad/Sum -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>gradients/add_grad/BroadcastGradientArgs&#45;&gt;gradients/add_grad/Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1147.21,-507.697C1148.65,-499.983 1150.37,-490.712 1151.96,-482.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1155.44,-482.575 1153.82,-472.104 1148.56,-481.297 1155.44,-482.575\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/BroadcastGradientArgs&#45;&gt;gradients/add_grad/Sum_1 -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>gradients/add_grad/BroadcastGradientArgs&#45;&gt;gradients/add_grad/Sum_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1127.3,-508.055C1118.4,-499.027 1107.32,-487.792 1097.66,-477.987\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1100.12,-475.494 1090.6,-470.829 1095.13,-480.408 1100.12,-475.494\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Sum&#45;&gt;gradients/add_grad/Reshape -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>gradients/add_grad/Sum&#45;&gt;gradients/add_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1175.77,-440.485C1191.39,-430.077 1213.9,-415.066 1232.01,-402.994\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1234.12,-405.796 1240.49,-397.337 1230.23,-399.972 1234.12,-405.796\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Reshape&#45;&gt;gradients/add_grad/tuple/control_dependency -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>gradients/add_grad/Reshape&#45;&gt;gradients/add_grad/tuple/control_dependency</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1263.42,-363.7C1264.12,-355.245 1264.97,-344.869 1265.75,-335.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1269.26,-335.429 1266.59,-325.176 1262.28,-334.855 1269.26,-335.429\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Sum_1&#45;&gt;gradients/add_grad/Reshape_1 -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>gradients/add_grad/Sum_1&#45;&gt;gradients/add_grad/Reshape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1075,-435.697C1075,-427.983 1075,-418.712 1075,-410.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1078.5,-410.104 1075,-400.104 1071.5,-410.104 1078.5,-410.104\"/>\n",
       "</g>\n",
       "<!-- gradients/add_grad/Reshape_1&#45;&gt;gradients/add_grad/tuple/control_dependency_1 -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>gradients/add_grad/Reshape_1&#45;&gt;gradients/add_grad/tuple/control_dependency_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1075.47,-363.7C1075.71,-355.245 1075.99,-344.869 1076.25,-335.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1079.75,-335.268 1076.53,-325.176 1072.76,-335.076 1079.75,-335.268\"/>\n",
       "</g>\n",
       "<!-- gradients/Neg_grad/Neg -->\n",
       "<g id=\"node54\" class=\"node\"><title>gradients/Neg_grad/Neg</title>\n",
       "<ellipse fill=\"#b0efb3\" stroke=\"black\" cx=\"598\" cy=\"-1524\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"598\" y=\"-1520.3\" font-family=\"Times,serif\" font-size=\"14.00\">Neg</text>\n",
       "</g>\n",
       "<!-- gradients/Neg_grad/Neg&#45;&gt;gradients/Sum_grad/Reshape -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>gradients/Neg_grad/Neg&#45;&gt;gradients/Sum_grad/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M615.089,-1510.05C621.643,-1505.63 629.382,-1501.04 637,-1498 697.846,-1473.7 772.969,-1462.24 821.414,-1457.04\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"822.04,-1460.49 831.631,-1455.99 821.326,-1453.53 822.04,-1460.49\"/>\n",
       "</g>\n",
       "<!-- gradients/Shape -->\n",
       "<g id=\"node55\" class=\"node\"><title>gradients/Shape</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"518\" cy=\"-1671\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"518\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape</text>\n",
       "</g>\n",
       "<!-- gradients/Fill -->\n",
       "<g id=\"node57\" class=\"node\"><title>gradients/Fill</title>\n",
       "<ellipse fill=\"#bbf9cf\" stroke=\"black\" cx=\"598\" cy=\"-1599\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"598\" y=\"-1595.3\" font-family=\"Times,serif\" font-size=\"14.00\">Fill</text>\n",
       "</g>\n",
       "<!-- gradients/Shape&#45;&gt;gradients/Fill -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>gradients/Shape&#45;&gt;gradients/Fill</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M534.576,-1655.5C546.174,-1645.35 561.827,-1631.65 574.681,-1620.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"577.041,-1622.99 582.262,-1613.77 572.431,-1617.72 577.041,-1622.99\"/>\n",
       "</g>\n",
       "<!-- gradients/Const -->\n",
       "<g id=\"node56\" class=\"node\"><title>gradients/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"600\" cy=\"-1671\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"600\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- gradients/Const&#45;&gt;gradients/Fill -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>gradients/Const&#45;&gt;gradients/Fill</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M599.506,-1652.7C599.285,-1644.98 599.02,-1635.71 598.775,-1627.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"602.273,-1627 598.489,-1617.1 595.276,-1627.2 602.273,-1627\"/>\n",
       "</g>\n",
       "<!-- gradients/Fill&#45;&gt;gradients/Neg_grad/Neg -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>gradients/Fill&#45;&gt;gradients/Neg_grad/Neg</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M598,-1580.7C598,-1572.25 598,-1561.87 598,-1552.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"601.5,-1552.18 598,-1542.18 594.5,-1552.18 601.5,-1552.18\"/>\n",
       "</g>\n",
       "<!-- W/Assign -->\n",
       "<g id=\"node58\" class=\"node\"><title>W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"1880\" cy=\"-2398\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1880\" y=\"-2394.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- W -->\n",
       "<g id=\"node65\" class=\"node\"><title>W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"1880\" cy=\"-2315\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1880\" y=\"-2311.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- W/Assign&#45;&gt;W -->\n",
       "<g id=\"edge86\" class=\"edge\"><title>W/Assign&#45;&gt;W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M1880,-2379.82C1880,-2369.19 1880,-2355.31 1880,-2343.2\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"1883.5,-2343.15 1880,-2333.15 1876.5,-2343.15 1883.5,-2343.15\"/>\n",
       "</g>\n",
       "<!-- GradientDescent/learning_rate -->\n",
       "<g id=\"node61\" class=\"node\"><title>GradientDescent/learning_rate</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"73\" cy=\"-141\" rx=\"57.3905\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"73\" y=\"-137.3\" font-family=\"Times,serif\" font-size=\"14.00\">learning_rate</text>\n",
       "</g>\n",
       "<!-- GradientDescent/learning_rate&#45;&gt;GradientDescent/update_W/ApplyGradientDescent -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>GradientDescent/learning_rate&#45;&gt;GradientDescent/update_W/ApplyGradientDescent</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M113.046,-127.969C142.849,-118.699 184.359,-105.115 220,-91 240.878,-82.7317 263.573,-72.4184 282.419,-63.4729\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"284.105,-66.5463 291.614,-59.0725 281.083,-60.232 284.105,-66.5463\"/>\n",
       "</g>\n",
       "<!-- GradientDescent/learning_rate&#45;&gt;GradientDescent/update_b/ApplyGradientDescent -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>GradientDescent/learning_rate&#45;&gt;GradientDescent/update_b/ApplyGradientDescent</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M80.49,-123.104C87.0473,-108.312 96.6842,-86.5729 104.265,-69.4727\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"107.599,-70.5884 108.452,-60.028 101.199,-67.7515 107.599,-70.5884\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node62\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"1791\" cy=\"-2315\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1791\" y=\"-2311.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;gradients/MatMul_grad/MatMul_1 -->\n",
       "<g id=\"edge98\" class=\"edge\"><title>x&#45;&gt;gradients/MatMul_grad/MatMul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1802.27,-2298.49C1809.18,-2288.29 1817.7,-2274.39 1823,-2261 1837.88,-2223.39 1842,-2212.45 1842,-2172 1842,-2172 1842,-2172 1842,-381 1842,-214.806 1647.48,-302.905 1484,-273 1433.17,-263.701 1417.97,-278.195 1368,-265 1344.48,-258.788 1319.82,-247.019 1300.87,-236.689\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1302.5,-233.588 1292.06,-231.764 1299.08,-239.698 1302.5,-233.588\"/>\n",
       "</g>\n",
       "<!-- MatMul -->\n",
       "<g id=\"node69\" class=\"node\"><title>MatMul</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"1774\" cy=\"-2243\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1774\" y=\"-2239.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;MatMul -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>x&#45;&gt;MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1786.88,-2297.05C1784.99,-2289.26 1782.7,-2279.82 1780.58,-2271.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1783.96,-2270.17 1778.2,-2261.28 1777.15,-2271.82 1783.96,-2270.17\"/>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node63\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"1354\" cy=\"-2027\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1354\" y=\"-2023.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "<!-- y_&#45;&gt;gradients/mul_grad/Shape -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>y_&#45;&gt;gradients/mul_grad/Shape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1354,-2008.59C1354,-1981.37 1354,-1927.07 1354,-1881 1354,-1881 1354,-1881 1354,-1451 1354,-1410.15 1354,-1362.83 1354,-1333.65\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1357.5,-1333.41 1354,-1323.41 1350.5,-1333.41 1357.5,-1333.41\"/>\n",
       "</g>\n",
       "<!-- y_&#45;&gt;gradients/mul_grad/mul_1 -->\n",
       "<g id=\"edge89\" class=\"edge\"><title>y_&#45;&gt;gradients/mul_grad/mul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1371.77,-2012.99C1400.99,-1989.94 1456,-1938.86 1456,-1881 1456,-1881 1456,-1881 1456,-1379 1456,-1337.54 1464.51,-1290.1 1470.7,-1261.1\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1474.12,-1261.81 1472.85,-1251.29 1467.29,-1260.31 1474.12,-1261.81\"/>\n",
       "</g>\n",
       "<!-- mul -->\n",
       "<g id=\"node73\" class=\"node\"><title>mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"803\" cy=\"-1955\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"803\" y=\"-1951.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- y_&#45;&gt;mul -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>y_&#45;&gt;mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1327.42,-2022.62C1237.23,-2011.16 942.662,-1973.74 839.269,-1960.61\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"839.671,-1957.13 829.309,-1959.34 838.788,-1964.07 839.671,-1957.13\"/>\n",
       "</g>\n",
       "<!-- zeros -->\n",
       "<g id=\"node64\" class=\"node\"><title>zeros</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1880\" cy=\"-2473\" rx=\"29.4969\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1880\" y=\"-2469.3\" font-family=\"Times,serif\" font-size=\"14.00\">zeros</text>\n",
       "</g>\n",
       "<!-- zeros&#45;&gt;W/Assign -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>zeros&#45;&gt;W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1880,-2454.7C1880,-2446.25 1880,-2435.87 1880,-2426.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1883.5,-2426.18 1880,-2416.18 1876.5,-2426.18 1883.5,-2426.18\"/>\n",
       "</g>\n",
       "<!-- W&#45;&gt;gradients/MatMul_grad/MatMul -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>W&#45;&gt;gradients/MatMul_grad/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1880,-2296.95C1880,-2270.29 1880,-2217.11 1880,-2172 1880,-2172 1880,-2172 1880,-381 1880,-331.829 1892.33,-306.137 1856,-273 1800.16,-222.064 1570.86,-216.344 1467.7,-216.425\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1467.52,-212.925 1457.53,-216.452 1467.54,-219.925 1467.52,-212.925\"/>\n",
       "</g>\n",
       "<!-- W&#45;&gt;GradientDescent/update_W/ApplyGradientDescent -->\n",
       "<g id=\"edge82\" class=\"edge\"><title>W&#45;&gt;GradientDescent/update_W/ApplyGradientDescent</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1887.51,-2297.34C1898.58,-2271.2 1918,-2218.7 1918,-2172 1918,-2172 1918,-2172 1918,-215 1918,-78.2432 1766.62,-128.13 1633,-99 1403.43,-48.9519 691.891,-43.2115 425.796,-42.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"425.758,-39.36 415.754,-42.849 425.751,-46.36 425.758,-39.36\"/>\n",
       "</g>\n",
       "<!-- W&#45;&gt;MatMul -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>W&#45;&gt;MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1861.54,-2301.81C1845.66,-2291.32 1822.47,-2276.01 1803.94,-2263.77\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1805.81,-2260.81 1795.54,-2258.22 1801.95,-2266.65 1805.81,-2260.81\"/>\n",
       "</g>\n",
       "<!-- zeros_1 -->\n",
       "<g id=\"node66\" class=\"node\"><title>zeros_1</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"196\" cy=\"-2398\" rx=\"38.1938\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"196\" y=\"-2394.3\" font-family=\"Times,serif\" font-size=\"14.00\">zeros_1</text>\n",
       "</g>\n",
       "<!-- zeros_1&#45;&gt;b/Assign -->\n",
       "<g id=\"edge79\" class=\"edge\"><title>zeros_1&#45;&gt;b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196,-2379.82C196,-2369.19 196,-2355.31 196,-2343.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"199.5,-2343.15 196,-2333.15 192.5,-2343.15 199.5,-2343.15\"/>\n",
       "</g>\n",
       "<!-- b&#45;&gt;gradients/add_grad/Shape_1 -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>b&#45;&gt;gradients/add_grad/Shape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196,-2224.95C196,-2198.29 196,-2145.11 196,-2100 196,-2100 196,-2100 196,-1013 196,-917.617 304.207,-778.247 672,-655 702.985,-644.617 935.615,-659.028 966,-647 979.068,-641.827 991.041,-631.987 1000.37,-622.557\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1003,-624.876 1007.26,-615.178 997.881,-620.098 1003,-624.876\"/>\n",
       "</g>\n",
       "<!-- b&#45;&gt;GradientDescent/update_b/ApplyGradientDescent -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>b&#45;&gt;GradientDescent/update_b/ApplyGradientDescent</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M188.492,-2225.34C177.418,-2199.2 158,-2146.7 158,-2100 158,-2100 158,-2100 158,-215 158,-163.07 158.943,-148.733 144,-99 140.936,-88.8014 136.116,-78.2073 131.333,-69.014\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"134.292,-67.1274 126.438,-60.016 128.143,-70.4728 134.292,-67.1274\"/>\n",
       "</g>\n",
       "<!-- add -->\n",
       "<g id=\"node70\" class=\"node\"><title>add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"310\" cy=\"-2171\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"310\" y=\"-2167.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- b&#45;&gt;add -->\n",
       "<g id=\"edge93\" class=\"edge\"><title>b&#45;&gt;add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M215.339,-2230.13C233.599,-2218.91 261.195,-2201.97 281.887,-2189.26\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"283.876,-2192.15 290.567,-2183.93 280.213,-2186.18 283.876,-2192.15\"/>\n",
       "</g>\n",
       "<!-- init -->\n",
       "<g id=\"node68\" class=\"node\"><title>init</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"1954\" cy=\"-2473\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1954\" y=\"-2469.3\" font-family=\"Times,serif\" font-size=\"14.00\">init</text>\n",
       "</g>\n",
       "<!-- MatMul&#45;&gt;gradients/add_grad/Shape -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>MatMul&#45;&gt;gradients/add_grad/Shape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1779.93,-2225.19C1788.67,-2198.86 1804,-2146.11 1804,-2100 1804,-2100 1804,-2100 1804,-744 1804,-640.73 1432.16,-609.056 1304.46,-601.223\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1304.47,-597.718 1294.28,-600.618 1304.06,-604.705 1304.47,-597.718\"/>\n",
       "</g>\n",
       "<!-- MatMul&#45;&gt;add -->\n",
       "<g id=\"edge96\" class=\"edge\"><title>MatMul&#45;&gt;add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1734.08,-2240.09C1524.17,-2230.05 550.047,-2183.48 347.105,-2173.77\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.117,-2170.27 336.961,-2173.29 346.783,-2177.26 347.117,-2170.27\"/>\n",
       "</g>\n",
       "<!-- y_softmax -->\n",
       "<g id=\"node71\" class=\"node\"><title>y_softmax</title>\n",
       "<ellipse fill=\"#b4d7bb\" stroke=\"black\" cx=\"310\" cy=\"-2099\" rx=\"48.1917\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"310\" y=\"-2095.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_softmax</text>\n",
       "</g>\n",
       "<!-- add&#45;&gt;y_softmax -->\n",
       "<g id=\"edge92\" class=\"edge\"><title>add&#45;&gt;y_softmax</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M310,-2152.7C310,-2144.98 310,-2135.71 310,-2127.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"313.5,-2127.1 310,-2117.1 306.5,-2127.1 313.5,-2127.1\"/>\n",
       "</g>\n",
       "<!-- y_softmax&#45;&gt;gradients/Log_grad/Inv -->\n",
       "<g id=\"edge88\" class=\"edge\"><title>y_softmax&#45;&gt;gradients/Log_grad/Inv</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M358.201,-2098.2C601.92,-2098.56 1687,-2092.54 1687,-1956 1687,-1956 1687,-1956 1687,-1160 1687,-1119.01 1691.82,-1071.73 1695.35,-1042.6\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1698.86,-1042.73 1696.63,-1032.37 1691.92,-1041.86 1698.86,-1042.73\"/>\n",
       "</g>\n",
       "<!-- y_softmax&#45;&gt;gradients/y_softmax_grad/mul -->\n",
       "<g id=\"edge95\" class=\"edge\"><title>y_softmax&#45;&gt;gradients/y_softmax_grad/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M302.31,-2080.91C291.206,-2054.62 272,-2002.45 272,-1956 272,-1956 272,-1956 272,-1013 272,-925.814 570.238,-857.349 675.08,-835.975\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"676.066,-839.347 685.177,-833.94 674.683,-832.484 676.066,-839.347\"/>\n",
       "</g>\n",
       "<!-- y_softmax&#45;&gt;gradients/y_softmax_grad/mul_1 -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>y_softmax&#45;&gt;gradients/y_softmax_grad/mul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M294.253,-2081.61C271.992,-2056.64 234,-2006.61 234,-1956 234,-1956 234,-1956 234,-672 234,-540.345 723.769,-527.653 875.642,-526.882\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"875.994,-530.38 885.981,-526.845 875.969,-523.38 875.994,-530.38\"/>\n",
       "</g>\n",
       "<!-- Log -->\n",
       "<g id=\"node72\" class=\"node\"><title>Log</title>\n",
       "<ellipse fill=\"#cddce7\" stroke=\"black\" cx=\"338\" cy=\"-2027\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"338\" y=\"-2023.3\" font-family=\"Times,serif\" font-size=\"14.00\">Log</text>\n",
       "</g>\n",
       "<!-- y_softmax&#45;&gt;Log -->\n",
       "<g id=\"edge77\" class=\"edge\"><title>y_softmax&#45;&gt;Log</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M316.778,-2081.05C320.011,-2072.97 323.951,-2063.12 327.555,-2054.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"330.82,-2055.37 331.284,-2044.79 324.321,-2052.77 330.82,-2055.37\"/>\n",
       "</g>\n",
       "<!-- Log&#45;&gt;gradients/mul_grad/Shape_1 -->\n",
       "<g id=\"edge94\" class=\"edge\"><title>Log&#45;&gt;gradients/mul_grad/Shape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M339.976,-2008.62C342.89,-1981.43 348,-1927.18 348,-1881 348,-1881 348,-1881 348,-1451 348,-1316.6 504.267,-1383.09 637,-1362 690.668,-1353.47 1073.3,-1367.26 1126,-1354 1148.51,-1348.34 1171.71,-1336.38 1189.31,-1325.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1191.3,-1328.69 1197.96,-1320.45 1187.62,-1322.74 1191.3,-1328.69\"/>\n",
       "</g>\n",
       "<!-- Log&#45;&gt;gradients/mul_grad/mul -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>Log&#45;&gt;gradients/mul_grad/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M332.601,-2009.24C324.465,-1982.44 310,-1928.18 310,-1881 310,-1881 310,-1881 310,-1379 310,-1293.89 971.207,-1246.68 1136.09,-1236.23\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1136.46,-1239.71 1146.23,-1235.6 1136.03,-1232.73 1136.46,-1239.71\"/>\n",
       "</g>\n",
       "<!-- Log&#45;&gt;mul -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>Log&#45;&gt;mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M363.997,-2022.09C442.371,-2010.29 676.496,-1975.04 766.861,-1961.44\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"767.532,-1964.88 776.9,-1959.93 766.49,-1957.96 767.532,-1964.88\"/>\n",
       "</g>\n",
       "<!-- mul&#45;&gt;gradients/Sum_grad/Shape -->\n",
       "<g id=\"edge83\" class=\"edge\"><title>mul&#45;&gt;gradients/Sum_grad/Shape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M829.903,-1952.76C858.155,-1950.63 903.342,-1944.82 939,-1929 1037.24,-1885.41 1083.41,-1878.36 1131,-1782 1157.96,-1727.4 1165.61,-1695.09 1131,-1645 1113.8,-1620.11 1080.93,-1608.97 1054.09,-1603.99\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1054.57,-1600.53 1044.14,-1602.37 1053.44,-1607.43 1054.57,-1600.53\"/>\n",
       "</g>\n",
       "<!-- mul&#45;&gt;gradients/Sum_grad/Rank -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>mul&#45;&gt;gradients/Sum_grad/Rank</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M825.73,-1945.15C875.832,-1924.48 996.36,-1868.15 1060,-1782 1078.08,-1757.52 1086.33,-1723.13 1090.05,-1699.35\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1093.54,-1699.63 1091.45,-1689.24 1086.61,-1698.67 1093.54,-1699.63\"/>\n",
       "</g>\n",
       "<!-- Rank -->\n",
       "<g id=\"node74\" class=\"node\"><title>Rank</title>\n",
       "<ellipse fill=\"#adc6aa\" stroke=\"black\" cx=\"775\" cy=\"-1880\" rx=\"29.4969\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"775\" y=\"-1876.3\" font-family=\"Times,serif\" font-size=\"14.00\">Rank</text>\n",
       "</g>\n",
       "<!-- mul&#45;&gt;Rank -->\n",
       "<g id=\"edge85\" class=\"edge\"><title>mul&#45;&gt;Rank</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M796.646,-1937.44C793.234,-1928.54 788.947,-1917.36 785.09,-1907.31\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"788.275,-1905.84 781.425,-1897.75 781.739,-1908.34 788.275,-1905.84\"/>\n",
       "</g>\n",
       "<!-- Sum -->\n",
       "<g id=\"node76\" class=\"node\"><title>Sum</title>\n",
       "<ellipse fill=\"#d3e1cd\" stroke=\"black\" cx=\"403\" cy=\"-1671\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"403\" y=\"-1667.3\" font-family=\"Times,serif\" font-size=\"14.00\">Sum</text>\n",
       "</g>\n",
       "<!-- mul&#45;&gt;Sum -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>mul&#45;&gt;Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M775.917,-1954.3C717.628,-1954.47 582.454,-1952.27 544,-1929 458.487,-1877.25 421.192,-1753.63 408.463,-1698.88\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"411.828,-1697.89 406.238,-1688.89 404.996,-1699.42 411.828,-1697.89\"/>\n",
       "</g>\n",
       "<!-- Rank&#45;&gt;range -->\n",
       "<g id=\"edge90\" class=\"edge\"><title>Rank&#45;&gt;range</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M751.982,-1868.67C725.521,-1856.8 681.883,-1837.22 651.856,-1823.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"653.181,-1820.5 642.624,-1819.6 650.315,-1826.89 653.181,-1820.5\"/>\n",
       "</g>\n",
       "<!-- range&#45;&gt;gradients/Sum_grad/Shape_1 -->\n",
       "<g id=\"edge99\" class=\"edge\"><title>range&#45;&gt;gradients/Sum_grad/Shape_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M635.791,-1792.87C639.125,-1789.54 642.396,-1785.83 645,-1782 662.594,-1756.1 675.142,-1722.25 682.444,-1698.98\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"685.888,-1699.68 685.429,-1689.1 679.187,-1697.66 685.888,-1699.68\"/>\n",
       "</g>\n",
       "<!-- range&#45;&gt;gradients/Sum_grad/DynamicStitch -->\n",
       "<g id=\"edge97\" class=\"edge\"><title>range&#45;&gt;gradients/Sum_grad/DynamicStitch</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M588.307,-1806.18C550.159,-1804.44 488.318,-1799 474,-1782 445.224,-1747.84 443.2,-1613.35 474,-1581 480.252,-1574.43 627.988,-1573.99 637,-1573 713.115,-1564.62 799.924,-1548.66 856.217,-1537.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"857.246,-1540.81 866.363,-1535.42 855.87,-1533.95 857.246,-1540.81\"/>\n",
       "</g>\n",
       "<!-- range&#45;&gt;Sum -->\n",
       "<g id=\"edge81\" class=\"edge\"><title>range&#45;&gt;Sum</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M589.016,-1804.39C541.991,-1800.04 455.085,-1790.93 444,-1782 418.999,-1761.87 409.242,-1724.87 405.435,-1699.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"408.879,-1698.72 404.14,-1689.24 401.935,-1699.61 408.879,-1698.72\"/>\n",
       "</g>\n",
       "<!-- Neg -->\n",
       "<g id=\"node77\" class=\"node\"><title>Neg</title>\n",
       "<ellipse fill=\"#b0efb3\" stroke=\"black\" cx=\"403\" cy=\"-1599\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"403\" y=\"-1595.3\" font-family=\"Times,serif\" font-size=\"14.00\">Neg</text>\n",
       "</g>\n",
       "<!-- Sum&#45;&gt;Neg -->\n",
       "<g id=\"edge80\" class=\"edge\"><title>Sum&#45;&gt;Neg</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M403,-1652.7C403,-1644.98 403,-1635.71 403,-1627.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"406.5,-1627.1 403,-1617.1 399.5,-1627.1 406.5,-1627.1\"/>\n",
       "</g>\n",
       "<!-- Neg&#45;&gt;gradients/Shape -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>Neg&#45;&gt;gradients/Shape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M422.427,-1611.83C440.419,-1622.78 467.443,-1639.23 488.169,-1651.84\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"486.545,-1654.95 496.907,-1657.16 490.185,-1648.97 486.545,-1654.95\"/>\n",
       "</g>\n",
       "<!-- GradientDescent -->\n",
       "<g id=\"node78\" class=\"node\"><title>GradientDescent</title>\n",
       "<ellipse fill=\"#ecfce8\" stroke=\"black\" cx=\"2069\" cy=\"-2473\" rx=\"69.5877\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"2069\" y=\"-2469.3\" font-family=\"Times,serif\" font-size=\"14.00\">GradientDescent</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f706c1fdd50>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(1000):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    train_step.run(feed_dict={x:batch[0], y_:batch[1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y.eval(feed_dict={x: mnist.test.images[:10]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prediction = tf.argmax(y, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print predictions\n",
    "prediction.eval(feed_dict={x: mnist.test.images[:10]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# print labels\n",
    "np.argmax(mnist.test.labels[:10],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "display_img_array(mnist.test.images[8].reshape(28,28), width=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "correct_prediction.eval(feed_dict={x: mnist.test.images[:10] , y_: mnist.test.labels[:10]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "accuracy.eval(feed_dict={x: mnist.test.images[:10] , y_: mnist.test.labels[:10]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "accuracy.eval(feed_dict={x: mnist.test.images , y_: mnist.test.labels})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for t in range(10):\n",
    "    for i in range(1000):\n",
    "        batch = mnist.train.next_batch(200)\n",
    "        train_step.run(feed_dict={x:batch[0], y_:batch[1]})\n",
    "    a = accuracy.eval(feed_dict={x: mnist.validation.images , y_: mnist.validation.labels})\n",
    "    print (t, a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "accuracy.eval(feed_dict={x: mnist.test.images , y_: mnist.test.labels})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " 91% accuracy on MNIST is bad. It's almost embarrassingly bad.\n",
    " \n",
    " http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multilayer Convolutional Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reset session and graph\n",
    "import tensorflow as tf\n",
    "from tfdot import tfdot\n",
    "tf.ops.reset_default_graph()\n",
    "if 'sess' in globals():\n",
    "    sess.close()\n",
    "sess = tf.InteractiveSession()\n",
    "x = tf.placeholder(\"float\", shape=[None, 784], name=\"x\")\n",
    "y_ = tf.placeholder(\"float\", shape=[None, 10], name=\"y_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial, name ='W')\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial, name = 'b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fisrt layer\n",
    "with tf.name_scope('conv1'):\n",
    "    ## variables\n",
    "    W_conv1 = weight_variable([5,5,1,32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    ## build the layer\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "    h_pool1 = max_pool_2x2(h_conv1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"359pt\" height=\"792pt\"\n",
       " viewBox=\"0.00 0.00 359.00 792.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 788)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-788 355,-788 355,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_conv1</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-8 8,-776 289,-776 289,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"148.5\" y=\"-760.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_conv1/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"16,-526 16,-745 264,-745 264,-526 16,-526\"/>\n",
       "<text text-anchor=\"middle\" x=\"140\" y=\"-729.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_conv1/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"182,-379 182,-454 281,-454 281,-379 182,-379\"/>\n",
       "<text text-anchor=\"middle\" x=\"231.5\" y=\"-438.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_conv1/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"16,-296 16,-371 102,-371 102,-296 16,-296\"/>\n",
       "<text text-anchor=\"middle\" x=\"59\" y=\"-355.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/b</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_conv1/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"88,-379 88,-454 174,-454 174,-379 88,-379\"/>\n",
       "<text text-anchor=\"middle\" x=\"131\" y=\"-438.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/W</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape -->\n",
       "<g id=\"node1\" class=\"node\"><title>conv1/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"146\" cy=\"-696\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-692.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node4\" class=\"node\"><title>conv1/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"183\" cy=\"-624\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.768,-678.411C159.083,-670.249 164.384,-660.22 169.223,-651.065\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"172.459,-652.433 174.037,-641.956 166.27,-649.162 172.459,-652.433\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean -->\n",
       "<g id=\"node2\" class=\"node\"><title>conv1/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"225\" cy=\"-696\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"225\" y=\"-692.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal -->\n",
       "<g id=\"node9\" class=\"node\"><title>conv1/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"157\" cy=\"-480\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"157\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M240.252,-680.342C249.364,-670.482 260.093,-656.647 265,-642 270.082,-626.829 270.581,-620.995 265,-606 249.317,-563.86 212.336,-526.452 186.026,-503.785\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"188.145,-500.994 178.239,-497.235 183.639,-506.351 188.145,-500.994\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev -->\n",
       "<g id=\"node3\" class=\"node\"><title>conv1/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"58\" cy=\"-624\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"58\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul -->\n",
       "<g id=\"node5\" class=\"node\"><title>conv1/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"157\" cy=\"-552\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"157\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M77.5592,-609.17C92.8188,-598.381 114.159,-583.291 130.879,-571.469\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"133.001,-574.256 139.145,-565.625 128.959,-568.54 133.001,-574.256\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176.706,-606.055C173.736,-598.059 170.123,-588.331 166.806,-579.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.999,-577.945 163.236,-569.789 163.437,-580.382 169.999,-577.945\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M157,-533.697C157,-525.983 157,-516.712 157,-508.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"160.5,-508.104 157,-498.104 153.5,-508.104 160.5,-508.104\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape -->\n",
       "<g id=\"node6\" class=\"node\"><title>conv1/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"233\" cy=\"-405\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"233\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape -->\n",
       "<g id=\"node13\" class=\"node\"><title>conv1/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"233\" cy=\"-322\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"233\" y=\"-318.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>conv1/Reshape/shape&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M233,-386.822C233,-376.19 233,-362.306 233,-350.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"236.5,-350.153 233,-340.153 229.5,-350.153 236.5,-350.153\"/>\n",
       "</g>\n",
       "<!-- conv1/b/Assign -->\n",
       "<g id=\"node7\" class=\"node\"><title>conv1/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"59\" cy=\"-322\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"59\" y=\"-318.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/b -->\n",
       "<g id=\"node12\" class=\"node\"><title>conv1/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"59\" cy=\"-250\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"59\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv1/b/Assign&#45;&gt;conv1/b -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>conv1/b/Assign&#45;&gt;conv1/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M59,-303.697C59,-295.983 59,-286.712 59,-278.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"62.5001,-278.104 59,-268.104 55.5001,-278.104 62.5001,-278.104\"/>\n",
       "</g>\n",
       "<!-- conv1/W/Assign -->\n",
       "<g id=\"node8\" class=\"node\"><title>conv1/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"131\" cy=\"-405\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"131\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/W -->\n",
       "<g id=\"node10\" class=\"node\"><title>conv1/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"141\" cy=\"-322\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"141\" y=\"-318.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv1/W/Assign&#45;&gt;conv1/W -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>conv1/W/Assign&#45;&gt;conv1/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M133.121,-386.822C134.433,-376.19 136.147,-362.306 137.642,-350.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"141.131,-350.507 138.882,-340.153 134.183,-349.649 141.131,-350.507\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal&#45;&gt;conv1/W/Assign -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>conv1/truncated_normal&#45;&gt;conv1/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M150.838,-461.7C147.74,-453.001 143.917,-442.267 140.457,-432.552\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"143.68,-431.17 137.028,-422.924 137.086,-433.519 143.68,-431.17\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D -->\n",
       "<g id=\"node14\" class=\"node\"><title>conv1/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"144\" cy=\"-250\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv1/W&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>conv1/W&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M141.742,-303.697C142.072,-295.983 142.469,-286.712 142.838,-278.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"146.335,-278.245 143.267,-268.104 139.342,-277.945 146.335,-278.245\"/>\n",
       "</g>\n",
       "<!-- conv1/Const -->\n",
       "<g id=\"node11\" class=\"node\"><title>conv1/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"47\" cy=\"-405\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv1/Const&#45;&gt;conv1/b/Assign -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>conv1/Const&#45;&gt;conv1/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M49.5449,-386.822C51.12,-376.19 53.1769,-362.306 54.9698,-350.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"58.4554,-350.558 56.4588,-340.153 51.531,-349.532 58.4554,-350.558\"/>\n",
       "</g>\n",
       "<!-- conv1/add -->\n",
       "<g id=\"node15\" class=\"node\"><title>conv1/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"102\" cy=\"-178\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"102\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv1/b&#45;&gt;conv1/add -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>conv1/b&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M68.7574,-233.116C74.1452,-224.345 80.9523,-213.264 87.0015,-203.416\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"90.0293,-205.174 92.2812,-194.821 84.0647,-201.51 90.0293,-205.174\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>conv1/Reshape&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M213.688,-305.811C201.241,-296.021 184.874,-283.148 171.129,-272.337\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"172.992,-269.35 162.968,-265.919 168.665,-274.852 172.992,-269.35\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D&#45;&gt;conv1/add -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>conv1/Conv2D&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M134.047,-232.411C128.899,-223.832 122.514,-213.191 116.802,-203.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"119.739,-201.762 111.593,-194.988 113.737,-205.364 119.739,-201.762\"/>\n",
       "</g>\n",
       "<!-- conv1/Relu -->\n",
       "<g id=\"node16\" class=\"node\"><title>conv1/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"102\" cy=\"-106\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"102\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv1/add&#45;&gt;conv1/Relu -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>conv1/add&#45;&gt;conv1/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M102,-159.697C102,-151.983 102,-142.712 102,-134.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"105.5,-134.104 102,-124.104 98.5001,-134.104 105.5,-134.104\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool -->\n",
       "<g id=\"node17\" class=\"node\"><title>conv1/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"102\" cy=\"-34\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"102\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv1/Relu&#45;&gt;conv1/MaxPool -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>conv1/Relu&#45;&gt;conv1/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M102,-87.6966C102,-79.9827 102,-70.7125 102,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"105.5,-62.1043 102,-52.1043 98.5001,-62.1044 105.5,-62.1043\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node18\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"324\" cy=\"-405\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"324\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>x&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M308.149,-389.891C294.233,-377.504 273.763,-359.284 257.7,-344.986\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"260.013,-342.359 250.217,-338.325 255.359,-347.588 260.013,-342.359\"/>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node19\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"324\" cy=\"-696\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"324\" y=\"-692.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f706c1a8ed0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# second layer\n",
    "with tf.name_scope('conv2'):\n",
    "    ## variables\n",
    "    W_conv2 = weight_variable([5,5,32,64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    ## build the layer\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "    h_pool2 = max_pool_2x2(h_conv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"643pt\" height=\"1125pt\"\n",
       " viewBox=\"0.00 0.00 643.00 1125.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1121)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-1121 639,-1121 639,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_conv2</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-8 8,-787 284,-787 284,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-771.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_conv2/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"16,-526 16,-756 264,-756 264,-526 16,-526\"/>\n",
       "<text text-anchor=\"middle\" x=\"140\" y=\"-740.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_conv2/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"91,-296 91,-371 177,-371 177,-296 91,-296\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-355.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/b</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_conv2/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"175,-379 175,-454 261,-454 261,-379 175,-379\"/>\n",
       "<text text-anchor=\"middle\" x=\"218\" y=\"-438.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/W</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_conv1</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"292,-296 292,-1109 573,-1109 573,-296 292,-296\"/>\n",
       "<text text-anchor=\"middle\" x=\"432.5\" y=\"-1093.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_conv1/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"300,-859 300,-1078 548,-1078 548,-859 300,-859\"/>\n",
       "<text text-anchor=\"middle\" x=\"424\" y=\"-1062.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_conv1/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"466,-681 466,-756 565,-756 565,-681 466,-681\"/>\n",
       "<text text-anchor=\"middle\" x=\"515.5\" y=\"-740.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_conv1/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"300,-598 300,-673 386,-673 386,-598 300,-598\"/>\n",
       "<text text-anchor=\"middle\" x=\"343\" y=\"-657.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/b</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_conv1/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"372,-681 372,-756 458,-756 458,-681 372,-681\"/>\n",
       "<text text-anchor=\"middle\" x=\"415\" y=\"-740.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/W</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/shape -->\n",
       "<g id=\"node1\" class=\"node\"><title>conv2/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"146\" cy=\"-707\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-703.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node4\" class=\"node\"><title>conv2/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"183\" cy=\"-624\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/shape&#45;&gt;conv2/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>conv2/truncated_normal/shape&#45;&gt;conv2/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M153.666,-689.217C158.644,-678.321 165.236,-663.888 170.911,-651.466\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"174.245,-652.591 175.216,-642.04 167.878,-649.682 174.245,-652.591\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mean -->\n",
       "<g id=\"node2\" class=\"node\"><title>conv2/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"225\" cy=\"-707\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"225\" y=\"-703.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal -->\n",
       "<g id=\"node8\" class=\"node\"><title>conv2/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"203\" cy=\"-480\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"203\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mean&#45;&gt;conv2/truncated_normal -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>conv2/truncated_normal/mean&#45;&gt;conv2/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M246.337,-693.871C253.658,-688.503 261.008,-681.469 265,-673 279.213,-642.849 273.041,-630.349 265,-598 256.53,-563.928 236.099,-528.867 220.994,-506.125\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.817,-504.052 215.3,-497.75 218.028,-507.988 223.817,-504.052\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/stddev -->\n",
       "<g id=\"node3\" class=\"node\"><title>conv2/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"58\" cy=\"-624\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"58\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mul -->\n",
       "<g id=\"node5\" class=\"node\"><title>conv2/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"183\" cy=\"-552\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/stddev&#45;&gt;conv2/truncated_normal/mul -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>conv2/truncated_normal/stddev&#45;&gt;conv2/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M79.8151,-610.177C86.5843,-606.236 94.0831,-601.908 101,-598 118.328,-588.21 137.86,-577.479 153.513,-568.953\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155.589,-571.808 162.704,-563.957 152.246,-565.658 155.589,-571.808\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/TruncatedNormal&#45;&gt;conv2/truncated_normal/mul -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>conv2/truncated_normal/TruncatedNormal&#45;&gt;conv2/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M183,-605.697C183,-597.983 183,-588.712 183,-580.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"186.5,-580.104 183,-570.104 179.5,-580.104 186.5,-580.104\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mul&#45;&gt;conv2/truncated_normal -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>conv2/truncated_normal/mul&#45;&gt;conv2/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.841,-534.055C190.068,-526.261 192.765,-516.822 195.263,-508.079\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"198.682,-508.852 198.064,-498.275 191.952,-506.929 198.682,-508.852\"/>\n",
       "</g>\n",
       "<!-- conv2/b/Assign -->\n",
       "<g id=\"node6\" class=\"node\"><title>conv2/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"134\" cy=\"-322\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-318.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv2/b -->\n",
       "<g id=\"node11\" class=\"node\"><title>conv2/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"134\" cy=\"-250\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv2/b/Assign&#45;&gt;conv2/b -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>conv2/b/Assign&#45;&gt;conv2/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M134,-303.697C134,-295.983 134,-286.712 134,-278.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"137.5,-278.104 134,-268.104 130.5,-278.104 137.5,-278.104\"/>\n",
       "</g>\n",
       "<!-- conv2/W/Assign -->\n",
       "<g id=\"node7\" class=\"node\"><title>conv2/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"218\" cy=\"-405\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"218\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv2/W -->\n",
       "<g id=\"node9\" class=\"node\"><title>conv2/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"219\" cy=\"-322\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"219\" y=\"-318.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv2/W/Assign&#45;&gt;conv2/W -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>conv2/W/Assign&#45;&gt;conv2/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M218.212,-386.822C218.343,-376.19 218.515,-362.306 218.664,-350.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"222.164,-350.196 218.788,-340.153 215.165,-350.109 222.164,-350.196\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal&#45;&gt;conv2/W/Assign -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>conv2/truncated_normal&#45;&gt;conv2/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M206.555,-461.7C208.323,-453.092 210.501,-442.493 212.481,-432.859\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"215.938,-433.424 214.522,-422.924 209.081,-432.015 215.938,-433.424\"/>\n",
       "</g>\n",
       "<!-- conv2/Conv2D -->\n",
       "<g id=\"node12\" class=\"node\"><title>conv2/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"219\" cy=\"-250\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"219\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv2/W&#45;&gt;conv2/Conv2D -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>conv2/W&#45;&gt;conv2/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M219,-303.697C219,-295.983 219,-286.712 219,-278.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"222.5,-278.104 219,-268.104 215.5,-278.104 222.5,-278.104\"/>\n",
       "</g>\n",
       "<!-- conv2/Const -->\n",
       "<g id=\"node10\" class=\"node\"><title>conv2/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"134\" cy=\"-405\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv2/Const&#45;&gt;conv2/b/Assign -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>conv2/Const&#45;&gt;conv2/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M134,-386.822C134,-376.19 134,-362.306 134,-350.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.5,-350.153 134,-340.153 130.5,-350.153 137.5,-350.153\"/>\n",
       "</g>\n",
       "<!-- conv2/add -->\n",
       "<g id=\"node13\" class=\"node\"><title>conv2/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"176\" cy=\"-178\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv2/b&#45;&gt;conv2/add -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>conv2/b&#45;&gt;conv2/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.53,-233.116C148.666,-224.557 155.122,-213.797 160.921,-204.132\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"164.072,-205.683 166.216,-195.307 158.07,-202.081 164.072,-205.683\"/>\n",
       "</g>\n",
       "<!-- conv2/Conv2D&#45;&gt;conv2/add -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>conv2/Conv2D&#45;&gt;conv2/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M208.81,-232.411C203.54,-223.832 197.003,-213.191 191.154,-203.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194.038,-201.677 185.821,-194.988 188.073,-205.341 194.038,-201.677\"/>\n",
       "</g>\n",
       "<!-- conv2/Relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>conv2/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"176\" cy=\"-106\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv2/add&#45;&gt;conv2/Relu -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>conv2/add&#45;&gt;conv2/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-159.697C176,-151.983 176,-142.712 176,-134.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"179.5,-134.104 176,-124.104 172.5,-134.104 179.5,-134.104\"/>\n",
       "</g>\n",
       "<!-- conv2/MaxPool -->\n",
       "<g id=\"node15\" class=\"node\"><title>conv2/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"176\" cy=\"-34\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"176\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv2/Relu&#45;&gt;conv2/MaxPool -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>conv2/Relu&#45;&gt;conv2/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176,-87.6966C176,-79.9827 176,-70.7125 176,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"179.5,-62.1043 176,-52.1043 172.5,-62.1044 179.5,-62.1043\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape -->\n",
       "<g id=\"node16\" class=\"node\"><title>conv1/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"430\" cy=\"-1029\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"430\" y=\"-1025.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node19\" class=\"node\"><title>conv1/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"467\" cy=\"-957\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M438.768,-1011.41C443.083,-1003.25 448.384,-993.22 453.223,-984.065\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"456.459,-985.433 458.037,-974.956 450.27,-982.162 456.459,-985.433\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean -->\n",
       "<g id=\"node17\" class=\"node\"><title>conv1/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"509\" cy=\"-1029\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"509\" y=\"-1025.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal -->\n",
       "<g id=\"node24\" class=\"node\"><title>conv1/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"441\" cy=\"-813\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"441\" y=\"-809.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M524.252,-1013.34C533.364,-1003.48 544.093,-989.647 549,-975 554.082,-959.829 554.581,-953.995 549,-939 533.317,-896.86 496.336,-859.452 470.026,-836.785\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"472.145,-833.994 462.239,-830.235 467.639,-839.351 472.145,-833.994\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev -->\n",
       "<g id=\"node18\" class=\"node\"><title>conv1/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"342\" cy=\"-957\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"342\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul -->\n",
       "<g id=\"node20\" class=\"node\"><title>conv1/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"441\" cy=\"-885\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"441\" y=\"-881.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M361.559,-942.17C376.819,-931.381 398.159,-916.291 414.879,-904.469\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"417.001,-907.256 423.145,-898.625 412.959,-901.54 417.001,-907.256\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M460.706,-939.055C457.736,-931.059 454.123,-921.331 450.806,-912.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"453.999,-910.945 447.236,-902.789 447.437,-913.382 453.999,-910.945\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M441,-866.697C441,-858.983 441,-849.712 441,-841.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"444.5,-841.104 441,-831.104 437.5,-841.104 444.5,-841.104\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape -->\n",
       "<g id=\"node21\" class=\"node\"><title>conv1/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"517\" cy=\"-707\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"517\" y=\"-703.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape -->\n",
       "<g id=\"node28\" class=\"node\"><title>conv1/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"517\" cy=\"-624\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"517\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>conv1/Reshape/shape&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M517,-688.822C517,-678.19 517,-664.306 517,-652.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"520.5,-652.153 517,-642.153 513.5,-652.153 520.5,-652.153\"/>\n",
       "</g>\n",
       "<!-- conv1/b/Assign -->\n",
       "<g id=\"node22\" class=\"node\"><title>conv1/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"343\" cy=\"-624\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"343\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/b -->\n",
       "<g id=\"node27\" class=\"node\"><title>conv1/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"343\" cy=\"-552\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"343\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv1/b/Assign&#45;&gt;conv1/b -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>conv1/b/Assign&#45;&gt;conv1/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M343,-605.697C343,-597.983 343,-588.712 343,-580.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"346.5,-580.104 343,-570.104 339.5,-580.104 346.5,-580.104\"/>\n",
       "</g>\n",
       "<!-- conv1/W/Assign -->\n",
       "<g id=\"node23\" class=\"node\"><title>conv1/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"415\" cy=\"-707\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"415\" y=\"-703.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/W -->\n",
       "<g id=\"node25\" class=\"node\"><title>conv1/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"425\" cy=\"-624\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"425\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv1/W/Assign&#45;&gt;conv1/W -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>conv1/W/Assign&#45;&gt;conv1/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M417.121,-688.822C418.433,-678.19 420.147,-664.306 421.642,-652.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"425.131,-652.507 422.882,-642.153 418.183,-651.649 425.131,-652.507\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal&#45;&gt;conv1/W/Assign -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>conv1/truncated_normal&#45;&gt;conv1/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M436.707,-794.827C432.642,-778.568 426.476,-753.903 421.753,-735.011\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"425.121,-734.052 419.3,-725.199 418.33,-735.749 425.121,-734.052\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D -->\n",
       "<g id=\"node29\" class=\"node\"><title>conv1/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"428\" cy=\"-552\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"428\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv1/W&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>conv1/W&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M425.742,-605.697C426.072,-597.983 426.469,-588.712 426.838,-580.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"430.335,-580.245 427.267,-570.104 423.342,-579.945 430.335,-580.245\"/>\n",
       "</g>\n",
       "<!-- conv1/Const -->\n",
       "<g id=\"node26\" class=\"node\"><title>conv1/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"331\" cy=\"-707\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"331\" y=\"-703.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv1/Const&#45;&gt;conv1/b/Assign -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>conv1/Const&#45;&gt;conv1/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M333.545,-688.822C335.12,-678.19 337.177,-664.306 338.97,-652.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"342.455,-652.558 340.459,-642.153 335.531,-651.532 342.455,-652.558\"/>\n",
       "</g>\n",
       "<!-- conv1/add -->\n",
       "<g id=\"node30\" class=\"node\"><title>conv1/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"344\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv1/b&#45;&gt;conv1/add -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>conv1/b&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M343.247,-533.697C343.357,-525.983 343.49,-516.712 343.613,-508.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.112,-508.153 343.756,-498.104 340.113,-508.053 347.112,-508.153\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>conv1/Reshape&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M497.688,-607.811C485.241,-598.021 468.874,-585.148 455.129,-574.337\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"456.992,-571.35 446.968,-567.919 452.665,-576.852 456.992,-571.35\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D&#45;&gt;conv1/add -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>conv1/Conv2D&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M409.773,-535.811C397.546,-525.622 381.31,-512.092 368.029,-501.024\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370.127,-498.216 360.204,-494.503 365.645,-503.594 370.127,-498.216\"/>\n",
       "</g>\n",
       "<!-- conv1/Relu -->\n",
       "<g id=\"node31\" class=\"node\"><title>conv1/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"344\" cy=\"-405\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-401.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv1/add&#45;&gt;conv1/Relu -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>conv1/add&#45;&gt;conv1/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M344,-461.7C344,-453.245 344,-442.869 344,-433.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.5,-433.176 344,-423.176 340.5,-433.176 347.5,-433.176\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool -->\n",
       "<g id=\"node32\" class=\"node\"><title>conv1/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"344\" cy=\"-322\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-318.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv1/Relu&#45;&gt;conv1/MaxPool -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>conv1/Relu&#45;&gt;conv1/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M344,-386.822C344,-376.19 344,-362.306 344,-350.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.5,-350.153 344,-340.153 340.5,-350.153 347.5,-350.153\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool&#45;&gt;conv2/Conv2D -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>conv1/MaxPool&#45;&gt;conv2/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M319.304,-307.17C300.091,-296.411 273.243,-281.376 252.158,-269.568\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"253.655,-266.395 243.219,-264.563 250.234,-272.503 253.655,-266.395\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node33\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"608\" cy=\"-707\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"608\" y=\"-703.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>x&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M592.149,-691.891C578.233,-679.504 557.763,-661.284 541.7,-646.986\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"544.013,-644.359 534.217,-640.325 539.359,-649.588 544.013,-644.359\"/>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node34\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"608\" cy=\"-1029\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"608\" y=\"-1025.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f70bfb75290>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fully-connected layer\n",
    "with tf.name_scope('full'):\n",
    "    W_fc1 = weight_variable([7*7*64, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1)+b_fc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"873pt\" height=\"1458pt\"\n",
       " viewBox=\"0.00 0.00 873.00 1458.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1454)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-1454 869,-1454 869,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_conv2</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-307 8,-1120 284,-1120 284,-307 8,-307\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-1104.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_conv2/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"16,-859 16,-1089 264,-1089 264,-859 16,-859\"/>\n",
       "<text text-anchor=\"middle\" x=\"140\" y=\"-1073.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_conv2/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"105,-598 105,-673 191,-673 191,-598 105,-598\"/>\n",
       "<text text-anchor=\"middle\" x=\"148\" y=\"-657.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/b</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_conv2/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"189,-712 189,-787 275,-787 275,-712 189,-712\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-771.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/W</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_conv1</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"292,-598 292,-1442 573,-1442 573,-598 292,-598\"/>\n",
       "<text text-anchor=\"middle\" x=\"432.5\" y=\"-1426.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_conv1/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"300,-1192 300,-1411 548,-1411 548,-1192 300,-1192\"/>\n",
       "<text text-anchor=\"middle\" x=\"424\" y=\"-1395.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_conv1/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"466,-1014 466,-1089 565,-1089 565,-1014 466,-1014\"/>\n",
       "<text text-anchor=\"middle\" x=\"515.5\" y=\"-1073.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_conv1/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"300,-931 300,-1006 386,-1006 386,-931 300,-931\"/>\n",
       "<text text-anchor=\"middle\" x=\"343\" y=\"-990.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/b</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_conv1/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"372,-1014 372,-1089 458,-1089 458,-1014 372,-1014\"/>\n",
       "<text text-anchor=\"middle\" x=\"415\" y=\"-1073.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/W</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_full</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"581,-8 581,-704 857,-704 857,-8 581,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"719\" y=\"-688.8\" font-family=\"Times,serif\" font-size=\"14.00\">full</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_full/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"590,-307 590,-382 675,-382 675,-307 590,-307\"/>\n",
       "<text text-anchor=\"middle\" x=\"632.5\" y=\"-366.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_full/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"679,-224 679,-299 765,-299 765,-224 679,-224\"/>\n",
       "<text text-anchor=\"middle\" x=\"722\" y=\"-283.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/b</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_full/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"589,-454 589,-673 837,-673 837,-454 589,-454\"/>\n",
       "<text text-anchor=\"middle\" x=\"713\" y=\"-657.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_full/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"761,-307 761,-382 847,-382 847,-307 761,-307\"/>\n",
       "<text text-anchor=\"middle\" x=\"804\" y=\"-366.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/W</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/shape -->\n",
       "<g id=\"node1\" class=\"node\"><title>conv2/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"146\" cy=\"-1040\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-1036.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node4\" class=\"node\"><title>conv2/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"183\" cy=\"-957\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/shape&#45;&gt;conv2/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>conv2/truncated_normal/shape&#45;&gt;conv2/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M153.666,-1022.22C158.644,-1011.32 165.236,-996.888 170.911,-984.466\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"174.245,-985.591 175.216,-975.04 167.878,-982.682 174.245,-985.591\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mean -->\n",
       "<g id=\"node2\" class=\"node\"><title>conv2/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"225\" cy=\"-1040\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"225\" y=\"-1036.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal -->\n",
       "<g id=\"node8\" class=\"node\"><title>conv2/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"203\" cy=\"-813\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"203\" y=\"-809.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mean&#45;&gt;conv2/truncated_normal -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>conv2/truncated_normal/mean&#45;&gt;conv2/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M246.337,-1026.87C253.658,-1021.5 261.008,-1014.47 265,-1006 279.213,-975.849 273.041,-963.349 265,-931 256.53,-896.928 236.099,-861.867 220.994,-839.125\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.817,-837.052 215.3,-830.75 218.028,-840.988 223.817,-837.052\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/stddev -->\n",
       "<g id=\"node3\" class=\"node\"><title>conv2/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"58\" cy=\"-957\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"58\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mul -->\n",
       "<g id=\"node5\" class=\"node\"><title>conv2/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"183\" cy=\"-885\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-881.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/stddev&#45;&gt;conv2/truncated_normal/mul -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>conv2/truncated_normal/stddev&#45;&gt;conv2/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M79.8151,-943.177C86.5843,-939.236 94.0831,-934.908 101,-931 118.328,-921.21 137.86,-910.479 153.513,-901.953\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"155.589,-904.808 162.704,-896.957 152.246,-898.658 155.589,-904.808\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/TruncatedNormal&#45;&gt;conv2/truncated_normal/mul -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>conv2/truncated_normal/TruncatedNormal&#45;&gt;conv2/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M183,-938.697C183,-930.983 183,-921.712 183,-913.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"186.5,-913.104 183,-903.104 179.5,-913.104 186.5,-913.104\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mul&#45;&gt;conv2/truncated_normal -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>conv2/truncated_normal/mul&#45;&gt;conv2/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M187.841,-867.055C190.068,-859.261 192.765,-849.822 195.263,-841.079\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"198.682,-841.852 198.064,-831.275 191.952,-839.929 198.682,-841.852\"/>\n",
       "</g>\n",
       "<!-- conv2/b/Assign -->\n",
       "<g id=\"node6\" class=\"node\"><title>conv2/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"148\" cy=\"-624\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"148\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv2/b -->\n",
       "<g id=\"node11\" class=\"node\"><title>conv2/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"148\" cy=\"-552\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"148\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv2/b/Assign&#45;&gt;conv2/b -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>conv2/b/Assign&#45;&gt;conv2/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M148,-605.697C148,-597.983 148,-588.712 148,-580.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"151.5,-580.104 148,-570.104 144.5,-580.104 151.5,-580.104\"/>\n",
       "</g>\n",
       "<!-- conv2/W/Assign -->\n",
       "<g id=\"node7\" class=\"node\"><title>conv2/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"232\" cy=\"-738\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-734.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv2/W -->\n",
       "<g id=\"node9\" class=\"node\"><title>conv2/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"232\" cy=\"-624\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv2/W/Assign&#45;&gt;conv2/W -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>conv2/W/Assign&#45;&gt;conv2/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M232,-719.99C232,-702.063 232,-673.555 232,-652.442\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"235.5,-652.17 232,-642.17 228.5,-652.171 235.5,-652.17\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal&#45;&gt;conv2/W/Assign -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>conv2/truncated_normal&#45;&gt;conv2/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M209.726,-795.069C213.24,-786.224 217.618,-775.202 221.561,-765.277\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"224.871,-766.425 225.31,-755.839 218.366,-763.841 224.871,-766.425\"/>\n",
       "</g>\n",
       "<!-- conv2/Conv2D -->\n",
       "<g id=\"node12\" class=\"node\"><title>conv2/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"233\" cy=\"-552\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"233\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv2/W&#45;&gt;conv2/Conv2D -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>conv2/W&#45;&gt;conv2/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M232.247,-605.697C232.357,-597.983 232.49,-588.712 232.613,-580.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"236.112,-580.153 232.756,-570.104 229.113,-580.053 236.112,-580.153\"/>\n",
       "</g>\n",
       "<!-- conv2/Const -->\n",
       "<g id=\"node10\" class=\"node\"><title>conv2/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"148\" cy=\"-738\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"148\" y=\"-734.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv2/Const&#45;&gt;conv2/b/Assign -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>conv2/Const&#45;&gt;conv2/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M148,-719.99C148,-702.063 148,-673.555 148,-652.442\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"151.5,-652.17 148,-642.17 144.5,-652.171 151.5,-652.17\"/>\n",
       "</g>\n",
       "<!-- conv2/add -->\n",
       "<g id=\"node13\" class=\"node\"><title>conv2/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"232\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv2/b&#45;&gt;conv2/add -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>conv2/b&#45;&gt;conv2/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M164.196,-537.503C176.63,-527.142 193.991,-512.674 208.032,-500.973\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"210.412,-503.546 215.854,-494.455 205.931,-498.168 210.412,-503.546\"/>\n",
       "</g>\n",
       "<!-- conv2/Conv2D&#45;&gt;conv2/add -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>conv2/Conv2D&#45;&gt;conv2/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M232.753,-533.697C232.643,-525.983 232.51,-516.712 232.387,-508.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"235.887,-508.053 232.244,-498.104 228.888,-508.153 235.887,-508.053\"/>\n",
       "</g>\n",
       "<!-- conv2/Relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>conv2/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"232\" cy=\"-408\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv2/add&#45;&gt;conv2/Relu -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>conv2/add&#45;&gt;conv2/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M232,-461.697C232,-453.983 232,-444.712 232,-436.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"235.5,-436.104 232,-426.104 228.5,-436.104 235.5,-436.104\"/>\n",
       "</g>\n",
       "<!-- conv2/MaxPool -->\n",
       "<g id=\"node15\" class=\"node\"><title>conv2/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"232\" cy=\"-333\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv2/Relu&#45;&gt;conv2/MaxPool -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>conv2/Relu&#45;&gt;conv2/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M232,-389.7C232,-381.245 232,-370.869 232,-361.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"235.5,-361.176 232,-351.176 228.5,-361.176 235.5,-361.176\"/>\n",
       "</g>\n",
       "<!-- full/Reshape -->\n",
       "<g id=\"node45\" class=\"node\"><title>full/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"629\" cy=\"-250\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"629\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- conv2/MaxPool&#45;&gt;full/Reshape -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>conv2/MaxPool&#45;&gt;full/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M270.066,-324.233C343.338,-309.284 503.722,-276.56 582.903,-260.405\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"583.963,-263.761 593.062,-258.332 582.564,-256.902 583.963,-263.761\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape -->\n",
       "<g id=\"node16\" class=\"node\"><title>conv1/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"430\" cy=\"-1362\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"430\" y=\"-1358.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node19\" class=\"node\"><title>conv1/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"467\" cy=\"-1290\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"467\" y=\"-1286.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M438.768,-1344.41C443.083,-1336.25 448.384,-1326.22 453.223,-1317.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"456.459,-1318.43 458.037,-1307.96 450.27,-1315.16 456.459,-1318.43\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean -->\n",
       "<g id=\"node17\" class=\"node\"><title>conv1/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"509\" cy=\"-1362\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"509\" y=\"-1358.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal -->\n",
       "<g id=\"node24\" class=\"node\"><title>conv1/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"441\" cy=\"-1146\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"441\" y=\"-1142.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M524.252,-1346.34C533.364,-1336.48 544.093,-1322.65 549,-1308 554.082,-1292.83 554.581,-1287 549,-1272 533.317,-1229.86 496.336,-1192.45 470.026,-1169.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"472.145,-1166.99 462.239,-1163.24 467.639,-1172.35 472.145,-1166.99\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev -->\n",
       "<g id=\"node18\" class=\"node\"><title>conv1/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"342\" cy=\"-1290\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"342\" y=\"-1286.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul -->\n",
       "<g id=\"node20\" class=\"node\"><title>conv1/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"441\" cy=\"-1218\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"441\" y=\"-1214.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M361.559,-1275.17C376.819,-1264.38 398.159,-1249.29 414.879,-1237.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"417.001,-1240.26 423.145,-1231.62 412.959,-1234.54 417.001,-1240.26\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M460.706,-1272.05C457.736,-1264.06 454.123,-1254.33 450.806,-1245.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"453.999,-1243.94 447.236,-1235.79 447.437,-1246.38 453.999,-1243.94\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M441,-1199.7C441,-1191.98 441,-1182.71 441,-1174.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"444.5,-1174.1 441,-1164.1 437.5,-1174.1 444.5,-1174.1\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape -->\n",
       "<g id=\"node21\" class=\"node\"><title>conv1/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"517\" cy=\"-1040\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"517\" y=\"-1036.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape -->\n",
       "<g id=\"node28\" class=\"node\"><title>conv1/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"517\" cy=\"-957\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"517\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>conv1/Reshape/shape&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M517,-1021.82C517,-1011.19 517,-997.306 517,-985.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"520.5,-985.153 517,-975.153 513.5,-985.153 520.5,-985.153\"/>\n",
       "</g>\n",
       "<!-- conv1/b/Assign -->\n",
       "<g id=\"node22\" class=\"node\"><title>conv1/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"343\" cy=\"-957\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"343\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/b -->\n",
       "<g id=\"node27\" class=\"node\"><title>conv1/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"343\" cy=\"-885\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"343\" y=\"-881.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv1/b/Assign&#45;&gt;conv1/b -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>conv1/b/Assign&#45;&gt;conv1/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M343,-938.697C343,-930.983 343,-921.712 343,-913.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"346.5,-913.104 343,-903.104 339.5,-913.104 346.5,-913.104\"/>\n",
       "</g>\n",
       "<!-- conv1/W/Assign -->\n",
       "<g id=\"node23\" class=\"node\"><title>conv1/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"415\" cy=\"-1040\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"415\" y=\"-1036.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/W -->\n",
       "<g id=\"node25\" class=\"node\"><title>conv1/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"425\" cy=\"-957\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"425\" y=\"-953.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv1/W/Assign&#45;&gt;conv1/W -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>conv1/W/Assign&#45;&gt;conv1/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M417.121,-1021.82C418.433,-1011.19 420.147,-997.306 421.642,-985.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"425.131,-985.507 422.882,-975.153 418.183,-984.649 425.131,-985.507\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal&#45;&gt;conv1/W/Assign -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>conv1/truncated_normal&#45;&gt;conv1/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M436.707,-1127.83C432.642,-1111.57 426.476,-1086.9 421.753,-1068.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"425.121,-1067.05 419.3,-1058.2 418.33,-1068.75 425.121,-1067.05\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D -->\n",
       "<g id=\"node29\" class=\"node\"><title>conv1/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"428\" cy=\"-885\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"428\" y=\"-881.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv1/W&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>conv1/W&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M425.742,-938.697C426.072,-930.983 426.469,-921.712 426.838,-913.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"430.335,-913.245 427.267,-903.104 423.342,-912.945 430.335,-913.245\"/>\n",
       "</g>\n",
       "<!-- conv1/Const -->\n",
       "<g id=\"node26\" class=\"node\"><title>conv1/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"331\" cy=\"-1040\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"331\" y=\"-1036.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv1/Const&#45;&gt;conv1/b/Assign -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>conv1/Const&#45;&gt;conv1/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M333.545,-1021.82C335.12,-1011.19 337.177,-997.306 338.97,-985.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"342.455,-985.558 340.459,-975.153 335.531,-984.532 342.455,-985.558\"/>\n",
       "</g>\n",
       "<!-- conv1/add -->\n",
       "<g id=\"node30\" class=\"node\"><title>conv1/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"344\" cy=\"-813\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-809.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv1/b&#45;&gt;conv1/add -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>conv1/b&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M343.247,-866.697C343.357,-858.983 343.49,-849.712 343.613,-841.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.112,-841.153 343.756,-831.104 340.113,-841.053 347.112,-841.153\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>conv1/Reshape&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M497.688,-940.811C485.241,-931.021 468.874,-918.148 455.129,-907.337\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"456.992,-904.35 446.968,-900.919 452.665,-909.852 456.992,-904.35\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D&#45;&gt;conv1/add -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>conv1/Conv2D&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M409.773,-868.811C397.546,-858.622 381.31,-845.092 368.029,-834.024\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"370.127,-831.216 360.204,-827.503 365.645,-836.594 370.127,-831.216\"/>\n",
       "</g>\n",
       "<!-- conv1/Relu -->\n",
       "<g id=\"node31\" class=\"node\"><title>conv1/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"344\" cy=\"-738\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-734.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv1/add&#45;&gt;conv1/Relu -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>conv1/add&#45;&gt;conv1/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M344,-794.7C344,-786.245 344,-775.869 344,-766.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.5,-766.176 344,-756.176 340.5,-766.176 347.5,-766.176\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool -->\n",
       "<g id=\"node32\" class=\"node\"><title>conv1/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"344\" cy=\"-624\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"344\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv1/Relu&#45;&gt;conv1/MaxPool -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>conv1/Relu&#45;&gt;conv1/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M344,-719.99C344,-702.063 344,-673.555 344,-652.442\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"347.5,-652.17 344,-642.17 340.5,-652.171 347.5,-652.17\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool&#45;&gt;conv2/Conv2D -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>conv1/MaxPool&#45;&gt;conv2/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M321.001,-608.496C304.52,-598.103 282.136,-583.987 264.068,-572.593\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"265.916,-569.62 255.59,-567.246 262.182,-575.541 265.916,-569.62\"/>\n",
       "</g>\n",
       "<!-- full/Reshape/shape -->\n",
       "<g id=\"node33\" class=\"node\"><title>full/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"629\" cy=\"-333\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"629\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- full/Reshape/shape&#45;&gt;full/Reshape -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>full/Reshape/shape&#45;&gt;full/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M629,-314.822C629,-304.19 629,-290.306 629,-278.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"632.5,-278.153 629,-268.153 625.5,-278.153 632.5,-278.153\"/>\n",
       "</g>\n",
       "<!-- full/b/Assign -->\n",
       "<g id=\"node34\" class=\"node\"><title>full/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"722\" cy=\"-250\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"722\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- full/b -->\n",
       "<g id=\"node44\" class=\"node\"><title>full/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"719\" cy=\"-178\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"719\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- full/b/Assign&#45;&gt;full/b -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>full/b/Assign&#45;&gt;full/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M721.258,-231.697C720.928,-223.983 720.531,-214.712 720.162,-206.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"723.658,-205.945 719.733,-196.104 716.665,-206.245 723.658,-205.945\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/shape -->\n",
       "<g id=\"node35\" class=\"node\"><title>full/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"719\" cy=\"-624\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"719\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node38\" class=\"node\"><title>full/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"756\" cy=\"-552\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"756\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/shape&#45;&gt;full/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>full/truncated_normal/shape&#45;&gt;full/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M727.768,-606.411C732.083,-598.249 737.384,-588.22 742.223,-579.065\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"745.459,-580.433 747.037,-569.956 739.27,-577.162 745.459,-580.433\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mean -->\n",
       "<g id=\"node36\" class=\"node\"><title>full/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"798\" cy=\"-624\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"798\" y=\"-620.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal -->\n",
       "<g id=\"node41\" class=\"node\"><title>full/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"776\" cy=\"-408\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"776\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mean&#45;&gt;full/truncated_normal -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>full/truncated_normal/mean&#45;&gt;full/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M813.252,-608.342C822.364,-598.482 833.093,-584.647 838,-570 843.082,-554.829 841.651,-549.578 838,-534 829.373,-497.193 808.223,-458.808 793.025,-434.448\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"795.857,-432.38 787.53,-425.831 789.955,-436.144 795.857,-432.38\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/stddev -->\n",
       "<g id=\"node37\" class=\"node\"><title>full/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"631\" cy=\"-552\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"631\" y=\"-548.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mul -->\n",
       "<g id=\"node39\" class=\"node\"><title>full/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"756\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"756\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/stddev&#45;&gt;full/truncated_normal/mul -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>full/truncated_normal/stddev&#45;&gt;full/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M653.347,-538.485C673.962,-526.941 704.683,-509.738 727.144,-497.159\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"728.895,-500.19 735.91,-492.25 725.475,-494.083 728.895,-500.19\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/TruncatedNormal&#45;&gt;full/truncated_normal/mul -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>full/truncated_normal/TruncatedNormal&#45;&gt;full/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M756,-533.697C756,-525.983 756,-516.712 756,-508.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"759.5,-508.104 756,-498.104 752.5,-508.104 759.5,-508.104\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mul&#45;&gt;full/truncated_normal -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>full/truncated_normal/mul&#45;&gt;full/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M760.841,-462.055C763.068,-454.261 765.765,-444.822 768.263,-436.079\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"771.682,-436.852 771.064,-426.275 764.952,-434.929 771.682,-436.852\"/>\n",
       "</g>\n",
       "<!-- full/W/Assign -->\n",
       "<g id=\"node40\" class=\"node\"><title>full/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"804\" cy=\"-333\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"804\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- full/W -->\n",
       "<g id=\"node42\" class=\"node\"><title>full/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"804\" cy=\"-250\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"804\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- full/W/Assign&#45;&gt;full/W -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>full/W/Assign&#45;&gt;full/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M804,-314.822C804,-304.19 804,-290.306 804,-278.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"807.5,-278.153 804,-268.153 800.5,-278.153 807.5,-278.153\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal&#45;&gt;full/W/Assign -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>full/truncated_normal&#45;&gt;full/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M782.494,-390.069C785.887,-381.224 790.114,-370.202 793.921,-360.277\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"797.228,-361.43 797.541,-350.839 790.692,-358.923 797.228,-361.43\"/>\n",
       "</g>\n",
       "<!-- full/MatMul -->\n",
       "<g id=\"node46\" class=\"node\"><title>full/MatMul</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"804\" cy=\"-178\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"804\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul</text>\n",
       "</g>\n",
       "<!-- full/W&#45;&gt;full/MatMul -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>full/W&#45;&gt;full/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M804,-231.697C804,-223.983 804,-214.712 804,-206.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"807.5,-206.104 804,-196.104 800.5,-206.104 807.5,-206.104\"/>\n",
       "</g>\n",
       "<!-- full/Const -->\n",
       "<g id=\"node43\" class=\"node\"><title>full/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"720\" cy=\"-333\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"720\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- full/Const&#45;&gt;full/b/Assign -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>full/Const&#45;&gt;full/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M720.424,-314.822C720.687,-304.19 721.029,-290.306 721.328,-278.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"724.828,-278.237 721.576,-268.153 717.831,-278.064 724.828,-278.237\"/>\n",
       "</g>\n",
       "<!-- full/add -->\n",
       "<g id=\"node47\" class=\"node\"><title>full/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"761\" cy=\"-106\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"761\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- full/b&#45;&gt;full/add -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>full/b&#45;&gt;full/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M728.53,-161.116C733.666,-152.557 740.122,-141.797 745.921,-132.132\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"749.072,-133.683 751.216,-123.307 743.07,-130.081 749.072,-133.683\"/>\n",
       "</g>\n",
       "<!-- full/Reshape&#45;&gt;full/MatMul -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>full/Reshape&#45;&gt;full/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M652.506,-235.274C659.614,-231.384 667.52,-227.311 675,-224 677.159,-223.044 726.562,-205.86 763.589,-193.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"764.876,-196.268 773.176,-189.684 762.581,-189.655 764.876,-196.268\"/>\n",
       "</g>\n",
       "<!-- full/MatMul&#45;&gt;full/add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>full/MatMul&#45;&gt;full/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M793.81,-160.411C788.54,-151.832 782.003,-141.191 776.154,-131.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"779.038,-129.677 770.821,-122.988 773.073,-133.341 779.038,-129.677\"/>\n",
       "</g>\n",
       "<!-- full/Relu -->\n",
       "<g id=\"node48\" class=\"node\"><title>full/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"761\" cy=\"-34\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"761\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- full/add&#45;&gt;full/Relu -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>full/add&#45;&gt;full/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M761,-87.6966C761,-79.9827 761,-70.7125 761,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"764.5,-62.1043 761,-52.1043 757.5,-62.1044 764.5,-62.1043\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node49\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"608\" cy=\"-1040\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"608\" y=\"-1036.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>x&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M592.149,-1024.89C578.233,-1012.5 557.763,-994.284 541.7,-979.986\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"544.013,-977.359 534.217,-973.325 539.359,-982.588 544.013,-977.359\"/>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node50\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"608\" cy=\"-1362\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"608\" y=\"-1358.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f706c1c77d0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Dropout:  A Simple Way to Prevent Neural Networks from Over fitting\n",
    "# https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf\n",
    "with tf.name_scope('dropout'):\n",
    "    keep_prob = tf.placeholder(\"float\")\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Readout\n",
    "with tf.name_scope('readout'):\n",
    "    W_fc2 = weight_variable([1024,10])\n",
    "    b_fc2 = bias_variable([10])\n",
    "    y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2)+b_fc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: root Pages: 1 -->\n",
       "<svg width=\"1436pt\" height=\"2279pt\"\n",
       " viewBox=\"0.00 0.00 1436.00 2279.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 2275)\">\n",
       "<title>root</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-2275 1432,-2275 1432,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_conv2</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"359,-1128 359,-1941 635,-1941 635,-1128 359,-1128\"/>\n",
       "<text text-anchor=\"middle\" x=\"497\" y=\"-1925.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2</text>\n",
       "</g>\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_conv2/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"367,-1680 367,-1910 615,-1910 615,-1680 367,-1680\"/>\n",
       "<text text-anchor=\"middle\" x=\"491\" y=\"-1894.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_conv2/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"540,-1419 540,-1494 626,-1494 626,-1419 540,-1419\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1478.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/b</text>\n",
       "</g>\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_conv2/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"456,-1533 456,-1608 542,-1608 542,-1533 456,-1533\"/>\n",
       "<text text-anchor=\"middle\" x=\"499\" y=\"-1592.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv2/W</text>\n",
       "</g>\n",
       "<g id=\"clust5\" class=\"cluster\"><title>cluster_conv1</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-1419 8,-2263 289,-2263 289,-1419 8,-1419\"/>\n",
       "<text text-anchor=\"middle\" x=\"148.5\" y=\"-2247.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1</text>\n",
       "</g>\n",
       "<g id=\"clust6\" class=\"cluster\"><title>cluster_conv1/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"16,-2013 16,-2232 264,-2232 264,-2013 16,-2013\"/>\n",
       "<text text-anchor=\"middle\" x=\"140\" y=\"-2216.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust7\" class=\"cluster\"><title>cluster_conv1/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"182,-1835 182,-1910 281,-1910 281,-1835 182,-1835\"/>\n",
       "<text text-anchor=\"middle\" x=\"231.5\" y=\"-1894.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust8\" class=\"cluster\"><title>cluster_conv1/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"20,-1752 20,-1827 106,-1827 106,-1752 20,-1752\"/>\n",
       "<text text-anchor=\"middle\" x=\"63\" y=\"-1811.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/b</text>\n",
       "</g>\n",
       "<g id=\"clust9\" class=\"cluster\"><title>cluster_conv1/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"88,-1835 88,-1910 174,-1910 174,-1835 88,-1835\"/>\n",
       "<text text-anchor=\"middle\" x=\"131\" y=\"-1894.8\" font-family=\"Times,serif\" font-size=\"14.00\">conv1/W</text>\n",
       "</g>\n",
       "<g id=\"clust10\" class=\"cluster\"><title>cluster_full</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"643,-829 643,-1525 919,-1525 919,-829 643,-829\"/>\n",
       "<text text-anchor=\"middle\" x=\"781\" y=\"-1509.8\" font-family=\"Times,serif\" font-size=\"14.00\">full</text>\n",
       "</g>\n",
       "<g id=\"clust11\" class=\"cluster\"><title>cluster_full/Reshape</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"652,-1128 652,-1203 737,-1203 737,-1128 652,-1128\"/>\n",
       "<text text-anchor=\"middle\" x=\"694.5\" y=\"-1187.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/Reshape</text>\n",
       "</g>\n",
       "<g id=\"clust12\" class=\"cluster\"><title>cluster_full/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"741,-1045 741,-1120 827,-1120 827,-1045 741,-1045\"/>\n",
       "<text text-anchor=\"middle\" x=\"784\" y=\"-1104.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/b</text>\n",
       "</g>\n",
       "<g id=\"clust13\" class=\"cluster\"><title>cluster_full/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"651,-1275 651,-1494 899,-1494 899,-1275 651,-1275\"/>\n",
       "<text text-anchor=\"middle\" x=\"775\" y=\"-1478.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust14\" class=\"cluster\"><title>cluster_full/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"823,-1128 823,-1203 909,-1203 909,-1128 823,-1128\"/>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-1187.8\" font-family=\"Times,serif\" font-size=\"14.00\">full/W</text>\n",
       "</g>\n",
       "<g id=\"clust15\" class=\"cluster\"><title>cluster_dropout</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"424,-216 424,-821 1116,-821 1116,-216 424,-216\"/>\n",
       "<text text-anchor=\"middle\" x=\"770\" y=\"-805.8\" font-family=\"Times,serif\" font-size=\"14.00\">dropout</text>\n",
       "</g>\n",
       "<g id=\"clust16\" class=\"cluster\"><title>cluster_dropout/LessEqual</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"989,-454 989,-529 1108,-529 1108,-454 989,-454\"/>\n",
       "<text text-anchor=\"middle\" x=\"1048.5\" y=\"-513.8\" font-family=\"Times,serif\" font-size=\"14.00\">dropout/LessEqual</text>\n",
       "</g>\n",
       "<g id=\"clust17\" class=\"cluster\"><title>cluster_dropout/dropout</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"432,-224 432,-790 758,-790 758,-224 432,-224\"/>\n",
       "<text text-anchor=\"middle\" x=\"595\" y=\"-774.8\" font-family=\"Times,serif\" font-size=\"14.00\">dropout/dropout</text>\n",
       "</g>\n",
       "<g id=\"clust18\" class=\"cluster\"><title>cluster_dropout/dropout/div</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"625,-454 625,-529 750,-529 750,-454 625,-454\"/>\n",
       "<text text-anchor=\"middle\" x=\"687.5\" y=\"-513.8\" font-family=\"Times,serif\" font-size=\"14.00\">dropout/dropout/div</text>\n",
       "</g>\n",
       "<g id=\"clust19\" class=\"cluster\"><title>cluster_dropout/dropout/random_uniform</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"440,-537 440,-684 750,-684 750,-537 440,-537\"/>\n",
       "<text text-anchor=\"middle\" x=\"595\" y=\"-668.8\" font-family=\"Times,serif\" font-size=\"14.00\">dropout/dropout/random_uniform</text>\n",
       "</g>\n",
       "<g id=\"clust20\" class=\"cluster\"><title>cluster_dropout/Greater</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"878,-454 878,-529 981,-529 981,-454 878,-454\"/>\n",
       "<text text-anchor=\"middle\" x=\"929.5\" y=\"-513.8\" font-family=\"Times,serif\" font-size=\"14.00\">dropout/Greater</text>\n",
       "</g>\n",
       "<g id=\"clust21\" class=\"cluster\"><title>cluster_readout</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1144,-8 1144,-715 1420,-715 1420,-8 1144,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282\" y=\"-699.8\" font-family=\"Times,serif\" font-size=\"14.00\">readout</text>\n",
       "</g>\n",
       "<g id=\"clust22\" class=\"cluster\"><title>cluster_readout/truncated_normal</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1152,-454 1152,-684 1400,-684 1400,-454 1152,-454\"/>\n",
       "<text text-anchor=\"middle\" x=\"1276\" y=\"-668.8\" font-family=\"Times,serif\" font-size=\"14.00\">readout/truncated_normal</text>\n",
       "</g>\n",
       "<g id=\"clust23\" class=\"cluster\"><title>cluster_readout/b</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1275,-224 1275,-299 1361,-299 1361,-224 1275,-224\"/>\n",
       "<text text-anchor=\"middle\" x=\"1318\" y=\"-283.8\" font-family=\"Times,serif\" font-size=\"14.00\">readout/b</text>\n",
       "</g>\n",
       "<g id=\"clust24\" class=\"cluster\"><title>cluster_readout/W</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1191,-307 1191,-382 1277,-382 1277,-307 1191,-307\"/>\n",
       "<text text-anchor=\"middle\" x=\"1234\" y=\"-366.8\" font-family=\"Times,serif\" font-size=\"14.00\">readout/W</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/shape -->\n",
       "<g id=\"node1\" class=\"node\"><title>conv2/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"497\" cy=\"-1861\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"497\" y=\"-1857.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node4\" class=\"node\"><title>conv2/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"534\" cy=\"-1778\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"534\" y=\"-1774.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/shape&#45;&gt;conv2/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>conv2/truncated_normal/shape&#45;&gt;conv2/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M504.666,-1843.22C509.644,-1832.32 516.236,-1817.89 521.911,-1805.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"525.245,-1806.59 526.216,-1796.04 518.878,-1803.68 525.245,-1806.59\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mean -->\n",
       "<g id=\"node2\" class=\"node\"><title>conv2/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"576\" cy=\"-1861\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"576\" y=\"-1857.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal -->\n",
       "<g id=\"node8\" class=\"node\"><title>conv2/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"516\" cy=\"-1634\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"516\" y=\"-1630.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mean&#45;&gt;conv2/truncated_normal -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>conv2/truncated_normal/mean&#45;&gt;conv2/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M597.337,-1847.87C604.658,-1842.5 612.008,-1835.47 616,-1827 630.213,-1796.85 627.543,-1783.27 616,-1752 601.867,-1713.71 569.003,-1679.46 544.789,-1658.06\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"546.81,-1655.19 536.948,-1651.31 542.242,-1660.49 546.81,-1655.19\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/stddev -->\n",
       "<g id=\"node3\" class=\"node\"><title>conv2/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"409\" cy=\"-1778\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"409\" y=\"-1774.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mul -->\n",
       "<g id=\"node5\" class=\"node\"><title>conv2/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"516\" cy=\"-1706\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"516\" y=\"-1702.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/stddev&#45;&gt;conv2/truncated_normal/mul -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>conv2/truncated_normal/stddev&#45;&gt;conv2/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M429.631,-1763.5C446.435,-1752.51 470.304,-1736.89 488.692,-1724.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"490.772,-1727.69 497.224,-1719.28 486.94,-1721.83 490.772,-1727.69\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/TruncatedNormal&#45;&gt;conv2/truncated_normal/mul -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>conv2/truncated_normal/TruncatedNormal&#45;&gt;conv2/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M529.551,-1759.7C527.545,-1751.9 525.131,-1742.51 522.899,-1733.83\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"526.278,-1732.92 520.398,-1724.1 519.499,-1734.66 526.278,-1732.92\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal/mul&#45;&gt;conv2/truncated_normal -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>conv2/truncated_normal/mul&#45;&gt;conv2/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M516,-1687.7C516,-1679.98 516,-1670.71 516,-1662.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"519.5,-1662.1 516,-1652.1 512.5,-1662.1 519.5,-1662.1\"/>\n",
       "</g>\n",
       "<!-- conv2/b/Assign -->\n",
       "<g id=\"node6\" class=\"node\"><title>conv2/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"583\" cy=\"-1445\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1441.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv2/b -->\n",
       "<g id=\"node11\" class=\"node\"><title>conv2/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"583\" cy=\"-1373\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1369.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv2/b/Assign&#45;&gt;conv2/b -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>conv2/b/Assign&#45;&gt;conv2/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M583,-1426.7C583,-1418.98 583,-1409.71 583,-1401.11\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"586.5,-1401.1 583,-1391.1 579.5,-1401.1 586.5,-1401.1\"/>\n",
       "</g>\n",
       "<!-- conv2/W/Assign -->\n",
       "<g id=\"node7\" class=\"node\"><title>conv2/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"499\" cy=\"-1559\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"499\" y=\"-1555.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv2/W -->\n",
       "<g id=\"node9\" class=\"node\"><title>conv2/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"498\" cy=\"-1445\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"498\" y=\"-1441.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv2/W/Assign&#45;&gt;conv2/W -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>conv2/W/Assign&#45;&gt;conv2/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M498.848,-1540.99C498.688,-1523.06 498.434,-1494.56 498.245,-1473.44\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"501.743,-1473.14 498.153,-1463.17 494.743,-1473.2 501.743,-1473.14\"/>\n",
       "</g>\n",
       "<!-- conv2/truncated_normal&#45;&gt;conv2/W/Assign -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>conv2/truncated_normal&#45;&gt;conv2/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M511.971,-1615.7C509.967,-1607.09 507.498,-1596.49 505.255,-1586.86\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"508.618,-1585.87 502.941,-1576.92 501.801,-1587.46 508.618,-1585.87\"/>\n",
       "</g>\n",
       "<!-- conv2/Conv2D -->\n",
       "<g id=\"node12\" class=\"node\"><title>conv2/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"498\" cy=\"-1373\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"498\" y=\"-1369.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv2/W&#45;&gt;conv2/Conv2D -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>conv2/W&#45;&gt;conv2/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M498,-1426.7C498,-1418.98 498,-1409.71 498,-1401.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"501.5,-1401.1 498,-1391.1 494.5,-1401.1 501.5,-1401.1\"/>\n",
       "</g>\n",
       "<!-- conv2/Const -->\n",
       "<g id=\"node10\" class=\"node\"><title>conv2/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"583\" cy=\"-1559\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1555.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv2/Const&#45;&gt;conv2/b/Assign -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>conv2/Const&#45;&gt;conv2/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M583,-1540.99C583,-1523.06 583,-1494.56 583,-1473.44\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"586.5,-1473.17 583,-1463.17 579.5,-1473.17 586.5,-1473.17\"/>\n",
       "</g>\n",
       "<!-- conv2/add -->\n",
       "<g id=\"node13\" class=\"node\"><title>conv2/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"583\" cy=\"-1301\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1297.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv2/b&#45;&gt;conv2/add -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>conv2/b&#45;&gt;conv2/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M583,-1354.7C583,-1346.98 583,-1337.71 583,-1329.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"586.5,-1329.1 583,-1319.1 579.5,-1329.1 586.5,-1329.1\"/>\n",
       "</g>\n",
       "<!-- conv2/Conv2D&#45;&gt;conv2/add -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>conv2/Conv2D&#45;&gt;conv2/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M516.444,-1356.81C528.817,-1346.62 545.246,-1333.09 558.685,-1322.02\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"561.109,-1324.56 566.603,-1315.5 556.659,-1319.16 561.109,-1324.56\"/>\n",
       "</g>\n",
       "<!-- conv2/Relu -->\n",
       "<g id=\"node14\" class=\"node\"><title>conv2/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"583\" cy=\"-1229\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1225.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv2/add&#45;&gt;conv2/Relu -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>conv2/add&#45;&gt;conv2/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M583,-1282.7C583,-1274.98 583,-1265.71 583,-1257.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"586.5,-1257.1 583,-1247.1 579.5,-1257.1 586.5,-1257.1\"/>\n",
       "</g>\n",
       "<!-- conv2/MaxPool -->\n",
       "<g id=\"node15\" class=\"node\"><title>conv2/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"583\" cy=\"-1154\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"583\" y=\"-1150.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv2/Relu&#45;&gt;conv2/MaxPool -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>conv2/Relu&#45;&gt;conv2/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M583,-1210.7C583,-1202.25 583,-1191.87 583,-1182.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"586.5,-1182.18 583,-1172.18 579.5,-1182.18 586.5,-1182.18\"/>\n",
       "</g>\n",
       "<!-- full/Reshape -->\n",
       "<g id=\"node45\" class=\"node\"><title>full/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"691\" cy=\"-1071\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"691\" y=\"-1067.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- conv2/MaxPool&#45;&gt;full/Reshape -->\n",
       "<g id=\"edge77\" class=\"edge\"><title>conv2/MaxPool&#45;&gt;full/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M603.061,-1137.95C619.883,-1125.34 644.095,-1107.18 662.856,-1093.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"665.147,-1095.77 671.047,-1086.97 660.947,-1090.17 665.147,-1095.77\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape -->\n",
       "<g id=\"node16\" class=\"node\"><title>conv1/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"146\" cy=\"-2183\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-2179.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node19\" class=\"node\"><title>conv1/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"183\" cy=\"-2111\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"183\" y=\"-2107.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>conv1/truncated_normal/shape&#45;&gt;conv1/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M154.768,-2165.41C159.083,-2157.25 164.384,-2147.22 169.223,-2138.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"172.459,-2139.43 174.037,-2128.96 166.27,-2136.16 172.459,-2139.43\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean -->\n",
       "<g id=\"node17\" class=\"node\"><title>conv1/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"225\" cy=\"-2183\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"225\" y=\"-2179.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal -->\n",
       "<g id=\"node24\" class=\"node\"><title>conv1/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"157\" cy=\"-1967\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"157\" y=\"-1963.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>conv1/truncated_normal/mean&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M240.252,-2167.34C249.364,-2157.48 260.093,-2143.65 265,-2129 270.082,-2113.83 270.581,-2108 265,-2093 249.317,-2050.86 212.336,-2013.45 186.026,-1990.79\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"188.145,-1987.99 178.239,-1984.24 183.639,-1993.35 188.145,-1987.99\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev -->\n",
       "<g id=\"node18\" class=\"node\"><title>conv1/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"58\" cy=\"-2111\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"58\" y=\"-2107.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul -->\n",
       "<g id=\"node20\" class=\"node\"><title>conv1/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"157\" cy=\"-2039\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"157\" y=\"-2035.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>conv1/truncated_normal/stddev&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M77.5592,-2096.17C92.8188,-2085.38 114.159,-2070.29 130.879,-2058.47\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"133.001,-2061.26 139.145,-2052.62 128.959,-2055.54 133.001,-2061.26\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>conv1/truncated_normal/TruncatedNormal&#45;&gt;conv1/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M176.706,-2093.05C173.736,-2085.06 170.123,-2075.33 166.806,-2066.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"169.999,-2064.94 163.236,-2056.79 163.437,-2067.38 169.999,-2064.94\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>conv1/truncated_normal/mul&#45;&gt;conv1/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M157,-2020.7C157,-2012.98 157,-2003.71 157,-1995.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"160.5,-1995.1 157,-1985.1 153.5,-1995.1 160.5,-1995.1\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape -->\n",
       "<g id=\"node21\" class=\"node\"><title>conv1/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"237\" cy=\"-1861\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"237\" y=\"-1857.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape -->\n",
       "<g id=\"node28\" class=\"node\"><title>conv1/Reshape</title>\n",
       "<ellipse fill=\"#c8cbd7\" stroke=\"black\" cx=\"237\" cy=\"-1778\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"237\" y=\"-1774.3\" font-family=\"Times,serif\" font-size=\"14.00\">Reshape</text>\n",
       "</g>\n",
       "<!-- conv1/Reshape/shape&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge29\" class=\"edge\"><title>conv1/Reshape/shape&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M237,-1842.82C237,-1832.19 237,-1818.31 237,-1806.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"240.5,-1806.15 237,-1796.15 233.5,-1806.15 240.5,-1806.15\"/>\n",
       "</g>\n",
       "<!-- conv1/b/Assign -->\n",
       "<g id=\"node22\" class=\"node\"><title>conv1/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"63\" cy=\"-1778\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"63\" y=\"-1774.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/b -->\n",
       "<g id=\"node27\" class=\"node\"><title>conv1/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"83\" cy=\"-1706\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"83\" y=\"-1702.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- conv1/b/Assign&#45;&gt;conv1/b -->\n",
       "<g id=\"edge26\" class=\"edge\"><title>conv1/b/Assign&#45;&gt;conv1/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M67.8414,-1760.05C70.1016,-1752.14 72.8458,-1742.54 75.3749,-1733.69\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"78.8212,-1734.37 78.2031,-1723.79 72.0905,-1732.44 78.8212,-1734.37\"/>\n",
       "</g>\n",
       "<!-- conv1/W/Assign -->\n",
       "<g id=\"node23\" class=\"node\"><title>conv1/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"131\" cy=\"-1861\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"131\" y=\"-1857.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- conv1/W -->\n",
       "<g id=\"node25\" class=\"node\"><title>conv1/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"149\" cy=\"-1778\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"149\" y=\"-1774.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- conv1/W/Assign&#45;&gt;conv1/W -->\n",
       "<g id=\"edge30\" class=\"edge\"><title>conv1/W/Assign&#45;&gt;conv1/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M134.817,-1842.82C137.196,-1832.12 140.307,-1818.12 143.009,-1805.96\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"146.497,-1806.4 145.25,-1795.88 139.664,-1804.88 146.497,-1806.4\"/>\n",
       "</g>\n",
       "<!-- conv1/truncated_normal&#45;&gt;conv1/W/Assign -->\n",
       "<g id=\"edge21\" class=\"edge\"><title>conv1/truncated_normal&#45;&gt;conv1/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M152.707,-1948.83C148.642,-1932.57 142.476,-1907.9 137.753,-1889.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"141.121,-1888.05 135.3,-1879.2 134.33,-1889.75 141.121,-1888.05\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D -->\n",
       "<g id=\"node29\" class=\"node\"><title>conv1/Conv2D</title>\n",
       "<ellipse fill=\"#b2ad9b\" stroke=\"black\" cx=\"208\" cy=\"-1706\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"208\" y=\"-1702.3\" font-family=\"Times,serif\" font-size=\"14.00\">Conv2D</text>\n",
       "</g>\n",
       "<!-- conv1/W&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge22\" class=\"edge\"><title>conv1/W&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M161.802,-1761.81C169.397,-1752.8 179.194,-1741.18 187.81,-1730.95\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"190.633,-1733.04 194.402,-1723.13 185.281,-1728.52 190.633,-1733.04\"/>\n",
       "</g>\n",
       "<!-- conv1/Const -->\n",
       "<g id=\"node26\" class=\"node\"><title>conv1/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"47\" cy=\"-1861\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"47\" y=\"-1857.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- conv1/Const&#45;&gt;conv1/b/Assign -->\n",
       "<g id=\"edge23\" class=\"edge\"><title>conv1/Const&#45;&gt;conv1/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M50.3932,-1842.82C52.4933,-1832.19 55.2358,-1818.31 57.6265,-1806.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"61.1074,-1806.64 59.6117,-1796.15 54.2401,-1805.29 61.1074,-1806.64\"/>\n",
       "</g>\n",
       "<!-- conv1/add -->\n",
       "<g id=\"node30\" class=\"node\"><title>conv1/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"208\" cy=\"-1634\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"208\" y=\"-1630.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- conv1/b&#45;&gt;conv1/add -->\n",
       "<g id=\"edge24\" class=\"edge\"><title>conv1/b&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M103.083,-1693.75C123.676,-1682.22 155.901,-1664.18 179.206,-1651.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"180.951,-1654.16 187.966,-1646.22 177.531,-1648.05 180.951,-1654.16\"/>\n",
       "</g>\n",
       "<!-- conv1/Reshape&#45;&gt;conv1/Conv2D -->\n",
       "<g id=\"edge28\" class=\"edge\"><title>conv1/Reshape&#45;&gt;conv1/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M229.98,-1760.05C226.632,-1751.97 222.55,-1742.12 218.818,-1733.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"222.016,-1731.69 214.955,-1723.79 215.549,-1734.37 222.016,-1731.69\"/>\n",
       "</g>\n",
       "<!-- conv1/Conv2D&#45;&gt;conv1/add -->\n",
       "<g id=\"edge27\" class=\"edge\"><title>conv1/Conv2D&#45;&gt;conv1/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M208,-1687.7C208,-1679.98 208,-1670.71 208,-1662.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"211.5,-1662.1 208,-1652.1 204.5,-1662.1 211.5,-1662.1\"/>\n",
       "</g>\n",
       "<!-- conv1/Relu -->\n",
       "<g id=\"node31\" class=\"node\"><title>conv1/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"216\" cy=\"-1559\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"216\" y=\"-1555.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- conv1/add&#45;&gt;conv1/Relu -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>conv1/add&#45;&gt;conv1/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M209.896,-1615.7C210.822,-1607.25 211.96,-1596.87 213,-1587.37\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"216.507,-1587.5 214.118,-1577.18 209.549,-1586.73 216.507,-1587.5\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool -->\n",
       "<g id=\"node32\" class=\"node\"><title>conv1/MaxPool</title>\n",
       "<ellipse fill=\"#bdb3a1\" stroke=\"black\" cx=\"232\" cy=\"-1445\" rx=\"43.5923\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"232\" y=\"-1441.3\" font-family=\"Times,serif\" font-size=\"14.00\">MaxPool</text>\n",
       "</g>\n",
       "<!-- conv1/Relu&#45;&gt;conv1/MaxPool -->\n",
       "<g id=\"edge25\" class=\"edge\"><title>conv1/Relu&#45;&gt;conv1/MaxPool</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M218.43,-1540.99C221.003,-1522.98 225.101,-1494.29 228.121,-1473.15\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"231.598,-1473.56 229.547,-1463.17 224.668,-1472.57 231.598,-1473.56\"/>\n",
       "</g>\n",
       "<!-- conv1/MaxPool&#45;&gt;conv2/Conv2D -->\n",
       "<g id=\"edge79\" class=\"edge\"><title>conv1/MaxPool&#45;&gt;conv2/Conv2D</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M267.598,-1434.63C315.908,-1421.92 402.171,-1399.22 454.335,-1385.49\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"455.512,-1388.8 464.292,-1382.87 453.73,-1382.03 455.512,-1388.8\"/>\n",
       "</g>\n",
       "<!-- full/Reshape/shape -->\n",
       "<g id=\"node33\" class=\"node\"><title>full/Reshape/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"691\" cy=\"-1154\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"691\" y=\"-1150.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- full/Reshape/shape&#45;&gt;full/Reshape -->\n",
       "<g id=\"edge45\" class=\"edge\"><title>full/Reshape/shape&#45;&gt;full/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M691,-1135.82C691,-1125.19 691,-1111.31 691,-1099.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"694.5,-1099.15 691,-1089.15 687.5,-1099.15 694.5,-1099.15\"/>\n",
       "</g>\n",
       "<!-- full/b/Assign -->\n",
       "<g id=\"node34\" class=\"node\"><title>full/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"784\" cy=\"-1071\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"784\" y=\"-1067.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- full/b -->\n",
       "<g id=\"node44\" class=\"node\"><title>full/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"781\" cy=\"-999\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"781\" y=\"-995.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- full/b/Assign&#45;&gt;full/b -->\n",
       "<g id=\"edge35\" class=\"edge\"><title>full/b/Assign&#45;&gt;full/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M783.258,-1052.7C782.928,-1044.98 782.531,-1035.71 782.162,-1027.11\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"785.658,-1026.95 781.733,-1017.1 778.665,-1027.25 785.658,-1026.95\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/shape -->\n",
       "<g id=\"node35\" class=\"node\"><title>full/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"781\" cy=\"-1445\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"781\" y=\"-1441.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node38\" class=\"node\"><title>full/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"818\" cy=\"-1373\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"818\" y=\"-1369.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/shape&#45;&gt;full/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge32\" class=\"edge\"><title>full/truncated_normal/shape&#45;&gt;full/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M789.768,-1427.41C794.083,-1419.25 799.384,-1409.22 804.223,-1400.07\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"807.459,-1401.43 809.037,-1390.96 801.27,-1398.16 807.459,-1401.43\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mean -->\n",
       "<g id=\"node36\" class=\"node\"><title>full/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"860\" cy=\"-1445\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"860\" y=\"-1441.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal -->\n",
       "<g id=\"node41\" class=\"node\"><title>full/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"838\" cy=\"-1229\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"838\" y=\"-1225.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mean&#45;&gt;full/truncated_normal -->\n",
       "<g id=\"edge37\" class=\"edge\"><title>full/truncated_normal/mean&#45;&gt;full/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M875.252,-1429.34C884.364,-1419.48 895.093,-1405.65 900,-1391 905.082,-1375.83 903.651,-1370.58 900,-1355 891.373,-1318.19 870.223,-1279.81 855.025,-1255.45\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"857.857,-1253.38 849.53,-1246.83 851.955,-1257.14 857.857,-1253.38\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/stddev -->\n",
       "<g id=\"node37\" class=\"node\"><title>full/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"693\" cy=\"-1373\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"693\" y=\"-1369.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mul -->\n",
       "<g id=\"node39\" class=\"node\"><title>full/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"818\" cy=\"-1301\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"818\" y=\"-1297.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- full/truncated_normal/stddev&#45;&gt;full/truncated_normal/mul -->\n",
       "<g id=\"edge31\" class=\"edge\"><title>full/truncated_normal/stddev&#45;&gt;full/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M715.347,-1359.49C735.962,-1347.94 766.683,-1330.74 789.144,-1318.16\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"790.895,-1321.19 797.91,-1313.25 787.475,-1315.08 790.895,-1321.19\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/TruncatedNormal&#45;&gt;full/truncated_normal/mul -->\n",
       "<g id=\"edge33\" class=\"edge\"><title>full/truncated_normal/TruncatedNormal&#45;&gt;full/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M818,-1354.7C818,-1346.98 818,-1337.71 818,-1329.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"821.5,-1329.1 818,-1319.1 814.5,-1329.1 821.5,-1329.1\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal/mul&#45;&gt;full/truncated_normal -->\n",
       "<g id=\"edge36\" class=\"edge\"><title>full/truncated_normal/mul&#45;&gt;full/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M822.841,-1283.05C825.068,-1275.26 827.765,-1265.82 830.263,-1257.08\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"833.682,-1257.85 833.064,-1247.28 826.952,-1255.93 833.682,-1257.85\"/>\n",
       "</g>\n",
       "<!-- full/W/Assign -->\n",
       "<g id=\"node40\" class=\"node\"><title>full/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"866\" cy=\"-1154\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-1150.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- full/W -->\n",
       "<g id=\"node42\" class=\"node\"><title>full/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"866\" cy=\"-1071\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-1067.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- full/W/Assign&#45;&gt;full/W -->\n",
       "<g id=\"edge38\" class=\"edge\"><title>full/W/Assign&#45;&gt;full/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M866,-1135.82C866,-1125.19 866,-1111.31 866,-1099.2\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"869.5,-1099.15 866,-1089.15 862.5,-1099.15 869.5,-1099.15\"/>\n",
       "</g>\n",
       "<!-- full/truncated_normal&#45;&gt;full/W/Assign -->\n",
       "<g id=\"edge39\" class=\"edge\"><title>full/truncated_normal&#45;&gt;full/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M844.494,-1211.07C847.887,-1202.22 852.114,-1191.2 855.921,-1181.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"859.228,-1182.43 859.541,-1171.84 852.692,-1179.92 859.228,-1182.43\"/>\n",
       "</g>\n",
       "<!-- full/MatMul -->\n",
       "<g id=\"node46\" class=\"node\"><title>full/MatMul</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"866\" cy=\"-999\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-995.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul</text>\n",
       "</g>\n",
       "<!-- full/W&#45;&gt;full/MatMul -->\n",
       "<g id=\"edge41\" class=\"edge\"><title>full/W&#45;&gt;full/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M866,-1052.7C866,-1044.98 866,-1035.71 866,-1027.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"869.5,-1027.1 866,-1017.1 862.5,-1027.1 869.5,-1027.1\"/>\n",
       "</g>\n",
       "<!-- full/Const -->\n",
       "<g id=\"node43\" class=\"node\"><title>full/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"782\" cy=\"-1154\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"782\" y=\"-1150.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- full/Const&#45;&gt;full/b/Assign -->\n",
       "<g id=\"edge40\" class=\"edge\"><title>full/Const&#45;&gt;full/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M782.424,-1135.82C782.687,-1125.19 783.029,-1111.31 783.328,-1099.2\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"786.828,-1099.24 783.576,-1089.15 779.831,-1099.06 786.828,-1099.24\"/>\n",
       "</g>\n",
       "<!-- full/add -->\n",
       "<g id=\"node47\" class=\"node\"><title>full/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"823\" cy=\"-927\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"823\" y=\"-923.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- full/b&#45;&gt;full/add -->\n",
       "<g id=\"edge44\" class=\"edge\"><title>full/b&#45;&gt;full/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M790.53,-982.116C795.666,-973.557 802.122,-962.797 807.921,-953.132\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"811.072,-954.683 813.216,-944.307 805.07,-951.081 811.072,-954.683\"/>\n",
       "</g>\n",
       "<!-- full/Reshape&#45;&gt;full/MatMul -->\n",
       "<g id=\"edge42\" class=\"edge\"><title>full/Reshape&#45;&gt;full/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M714.506,-1056.27C721.614,-1052.38 729.52,-1048.31 737,-1045 739.159,-1044.04 788.562,-1026.86 825.589,-1014.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"826.876,-1017.27 835.176,-1010.68 824.581,-1010.66 826.876,-1017.27\"/>\n",
       "</g>\n",
       "<!-- full/MatMul&#45;&gt;full/add -->\n",
       "<g id=\"edge34\" class=\"edge\"><title>full/MatMul&#45;&gt;full/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M855.81,-981.411C850.54,-972.832 844.003,-962.191 838.154,-952.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"841.038,-950.677 832.821,-943.988 835.073,-954.341 841.038,-950.677\"/>\n",
       "</g>\n",
       "<!-- full/Relu -->\n",
       "<g id=\"node48\" class=\"node\"><title>full/Relu</title>\n",
       "<ellipse fill=\"#c4aee5\" stroke=\"black\" cx=\"823\" cy=\"-855\" rx=\"27.0966\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"823\" y=\"-851.3\" font-family=\"Times,serif\" font-size=\"14.00\">Relu</text>\n",
       "</g>\n",
       "<!-- full/add&#45;&gt;full/Relu -->\n",
       "<g id=\"edge43\" class=\"edge\"><title>full/add&#45;&gt;full/Relu</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M823,-908.697C823,-900.983 823,-891.712 823,-883.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"826.5,-883.104 823,-873.104 819.5,-883.104 826.5,-883.104\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/Shape -->\n",
       "<g id=\"node55\" class=\"node\"><title>dropout/dropout/Shape</title>\n",
       "<ellipse fill=\"#d7fee8\" stroke=\"black\" cx=\"618\" cy=\"-741\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"618\" y=\"-737.3\" font-family=\"Times,serif\" font-size=\"14.00\">Shape</text>\n",
       "</g>\n",
       "<!-- full/Relu&#45;&gt;dropout/dropout/Shape -->\n",
       "<g id=\"edge76\" class=\"edge\"><title>full/Relu&#45;&gt;dropout/dropout/Shape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M802.594,-842.852C766.647,-823.212 691.821,-782.332 649.301,-759.101\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"650.945,-756.011 640.491,-754.288 647.589,-762.154 650.945,-756.011\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/mul -->\n",
       "<g id=\"node60\" class=\"node\"><title>dropout/dropout/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"723\" cy=\"-333\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"723\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- full/Relu&#45;&gt;dropout/dropout/mul -->\n",
       "<g id=\"edge78\" class=\"edge\"><title>full/Relu&#45;&gt;dropout/dropout/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M848.572,-848.656C921.622,-831.744 1126,-772.251 1126,-636 1126,-636 1126,-636 1126,-479 1126,-438.958 1140.5,-418.129 1112,-390 1062.77,-341.408 847.503,-334.659 760.091,-333.954\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"760.105,-330.454 750.085,-333.895 760.064,-337.454 760.105,-330.454\"/>\n",
       "</g>\n",
       "<!-- dropout/LessEqual/y -->\n",
       "<g id=\"node49\" class=\"node\"><title>dropout/LessEqual/y</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1024\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1024\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">y</text>\n",
       "</g>\n",
       "<!-- dropout/LessEqual -->\n",
       "<g id=\"node65\" class=\"node\"><title>dropout/LessEqual</title>\n",
       "<ellipse fill=\"#bbdbe1\" stroke=\"black\" cx=\"972\" cy=\"-408\" rx=\"47.3916\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"972\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">LessEqual</text>\n",
       "</g>\n",
       "<!-- dropout/LessEqual/y&#45;&gt;dropout/LessEqual -->\n",
       "<g id=\"edge57\" class=\"edge\"><title>dropout/LessEqual/y&#45;&gt;dropout/LessEqual</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1012.46,-463.465C1005.98,-454.742 997.743,-443.654 990.395,-433.762\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"993.006,-431.407 984.233,-425.467 987.386,-435.582 993.006,-431.407\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/div/x -->\n",
       "<g id=\"node50\" class=\"node\"><title>dropout/dropout/div/x</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"715\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"715\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/div -->\n",
       "<g id=\"node59\" class=\"node\"><title>dropout/dropout/div</title>\n",
       "<ellipse fill=\"#abafb2\" stroke=\"black\" cx=\"723\" cy=\"-408\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"723\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">div</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/div/x&#45;&gt;dropout/dropout/div -->\n",
       "<g id=\"edge54\" class=\"edge\"><title>dropout/dropout/div/x&#45;&gt;dropout/dropout/div</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M716.978,-461.697C717.859,-453.983 718.919,-444.712 719.901,-436.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"723.387,-436.437 721.045,-426.104 716.432,-435.642 723.387,-436.437\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/min -->\n",
       "<g id=\"node51\" class=\"node\"><title>dropout/dropout/random_uniform/min</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"715\" cy=\"-635\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"715\" y=\"-631.3\" font-family=\"Times,serif\" font-size=\"14.00\">min</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform -->\n",
       "<g id=\"node56\" class=\"node\"><title>dropout/dropout/random_uniform</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"546\" cy=\"-480\" rx=\"71.4873\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"546\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">random_uniform</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/min&#45;&gt;dropout/dropout/random_uniform -->\n",
       "<g id=\"edge48\" class=\"edge\"><title>dropout/dropout/random_uniform/min&#45;&gt;dropout/dropout/random_uniform</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M699.704,-620.152C670.697,-593.892 607.414,-536.599 571.756,-504.317\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.928,-501.563 564.166,-497.446 569.23,-506.752 573.928,-501.563\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/range -->\n",
       "<g id=\"node52\" class=\"node\"><title>dropout/dropout/random_uniform/range</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"639\" cy=\"-635\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"639\" y=\"-631.3\" font-family=\"Times,serif\" font-size=\"14.00\">range</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/mul -->\n",
       "<g id=\"node54\" class=\"node\"><title>dropout/dropout/random_uniform/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"546\" cy=\"-563\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"546\" y=\"-559.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/range&#45;&gt;dropout/dropout/random_uniform/mul -->\n",
       "<g id=\"edge47\" class=\"edge\"><title>dropout/dropout/random_uniform/range&#45;&gt;dropout/dropout/random_uniform/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M620.626,-620.17C606.602,-609.614 587.109,-594.942 571.563,-583.241\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"573.487,-580.308 563.392,-577.091 569.277,-585.901 573.487,-580.308\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/RandomUniform -->\n",
       "<g id=\"node53\" class=\"node\"><title>dropout/dropout/random_uniform/RandomUniform</title>\n",
       "<ellipse fill=\"#e3a9fb\" stroke=\"black\" cx=\"519\" cy=\"-635\" rx=\"71.4873\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"519\" y=\"-631.3\" font-family=\"Times,serif\" font-size=\"14.00\">RandomUniform</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/RandomUniform&#45;&gt;dropout/dropout/random_uniform/mul -->\n",
       "<g id=\"edge46\" class=\"edge\"><title>dropout/dropout/random_uniform/RandomUniform&#45;&gt;dropout/dropout/random_uniform/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M525.536,-617.055C528.62,-609.059 532.372,-599.331 535.817,-590.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"539.191,-591.379 539.524,-580.789 532.66,-588.859 539.191,-591.379\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform/mul&#45;&gt;dropout/dropout/random_uniform -->\n",
       "<g id=\"edge50\" class=\"edge\"><title>dropout/dropout/random_uniform/mul&#45;&gt;dropout/dropout/random_uniform</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M546,-544.822C546,-534.19 546,-520.306 546,-508.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"549.5,-508.153 546,-498.153 542.5,-508.153 549.5,-508.153\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/Shape&#45;&gt;dropout/dropout/random_uniform/RandomUniform -->\n",
       "<g id=\"edge49\" class=\"edge\"><title>dropout/dropout/Shape&#45;&gt;dropout/dropout/random_uniform/RandomUniform</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M603.392,-724.654C587.248,-707.695 561.029,-680.151 542.096,-660.262\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"544.412,-657.619 534.982,-652.789 539.341,-662.445 544.412,-657.619\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/add -->\n",
       "<g id=\"node57\" class=\"node\"><title>dropout/dropout/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"648\" cy=\"-408\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"648\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/random_uniform&#45;&gt;dropout/dropout/add -->\n",
       "<g id=\"edge55\" class=\"edge\"><title>dropout/dropout/random_uniform&#45;&gt;dropout/dropout/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M569.401,-462.941C584.884,-452.315 605.224,-438.356 621.334,-427.301\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"623.525,-430.041 629.79,-421.497 619.565,-424.27 623.525,-430.041\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/Floor -->\n",
       "<g id=\"node58\" class=\"node\"><title>dropout/dropout/Floor</title>\n",
       "<ellipse fill=\"#e3d2d7\" stroke=\"black\" cx=\"648\" cy=\"-333\" rx=\"29.795\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"648\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">Floor</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/add&#45;&gt;dropout/dropout/Floor -->\n",
       "<g id=\"edge53\" class=\"edge\"><title>dropout/dropout/add&#45;&gt;dropout/dropout/Floor</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M648,-389.7C648,-381.245 648,-370.869 648,-361.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"651.5,-361.176 648,-351.176 644.5,-361.176 651.5,-361.176\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/mul_1 -->\n",
       "<g id=\"node61\" class=\"node\"><title>dropout/dropout/mul_1</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"716\" cy=\"-250\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"716\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul_1</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/Floor&#45;&gt;dropout/dropout/mul_1 -->\n",
       "<g id=\"edge51\" class=\"edge\"><title>dropout/dropout/Floor&#45;&gt;dropout/dropout/mul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M661.111,-316.382C671.019,-304.581 684.79,-288.176 696.06,-274.752\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"698.948,-276.755 702.697,-266.846 693.587,-272.255 698.948,-276.755\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/div&#45;&gt;dropout/dropout/mul -->\n",
       "<g id=\"edge56\" class=\"edge\"><title>dropout/dropout/div&#45;&gt;dropout/dropout/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M723,-389.7C723,-381.245 723,-370.869 723,-361.373\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"726.5,-361.176 723,-351.176 719.5,-361.176 726.5,-361.176\"/>\n",
       "</g>\n",
       "<!-- dropout/dropout/mul&#45;&gt;dropout/dropout/mul_1 -->\n",
       "<g id=\"edge52\" class=\"edge\"><title>dropout/dropout/mul&#45;&gt;dropout/dropout/mul_1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M721.515,-314.822C720.597,-304.19 719.397,-290.306 718.351,-278.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"721.83,-277.815 717.482,-268.153 714.856,-278.417 721.83,-277.815\"/>\n",
       "</g>\n",
       "<!-- readout/MatMul -->\n",
       "<g id=\"node77\" class=\"node\"><title>readout/MatMul</title>\n",
       "<ellipse fill=\"#bbffbb\" stroke=\"black\" cx=\"1233\" cy=\"-178\" rx=\"40.0939\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1233\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">MatMul</text>\n",
       "</g>\n",
       "<!-- dropout/dropout/mul_1&#45;&gt;readout/MatMul -->\n",
       "<g id=\"edge80\" class=\"edge\"><title>dropout/dropout/mul_1&#45;&gt;readout/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M748.341,-244.621C836.376,-232.701 1080.68,-199.624 1184.89,-185.514\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1185.38,-188.979 1194.82,-184.169 1184.45,-182.042 1185.38,-188.979\"/>\n",
       "</g>\n",
       "<!-- dropout/Greater/y -->\n",
       "<g id=\"node62\" class=\"node\"><title>dropout/Greater/y</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"915\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"915\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">y</text>\n",
       "</g>\n",
       "<!-- dropout/Greater -->\n",
       "<g id=\"node64\" class=\"node\"><title>dropout/Greater</title>\n",
       "<ellipse fill=\"#f8cac9\" stroke=\"black\" cx=\"844\" cy=\"-408\" rx=\"37.0935\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"844\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">Greater</text>\n",
       "</g>\n",
       "<!-- dropout/Greater/y&#45;&gt;dropout/Greater -->\n",
       "<g id=\"edge61\" class=\"edge\"><title>dropout/Greater/y&#45;&gt;dropout/Greater</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M900.289,-464.496C890.652,-454.995 877.86,-442.383 866.906,-431.583\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"869.256,-428.985 859.677,-424.457 864.341,-433.97 869.256,-428.985\"/>\n",
       "</g>\n",
       "<!-- dropout/Placeholder -->\n",
       "<g id=\"node63\" class=\"node\"><title>dropout/Placeholder</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"818\" cy=\"-480\" rx=\"51.9908\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"818\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">Placeholder</text>\n",
       "</g>\n",
       "<!-- dropout/Placeholder&#45;&gt;dropout/dropout/add -->\n",
       "<g id=\"edge60\" class=\"edge\"><title>dropout/Placeholder&#45;&gt;dropout/dropout/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M785.06,-465.955C758.508,-455.372 720.249,-439.97 687,-426 684.47,-424.937 681.853,-423.824 679.227,-422.699\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"680.388,-419.388 669.82,-418.633 677.611,-425.814 680.388,-419.388\"/>\n",
       "</g>\n",
       "<!-- dropout/Placeholder&#45;&gt;dropout/dropout/div -->\n",
       "<g id=\"edge58\" class=\"edge\"><title>dropout/Placeholder&#45;&gt;dropout/dropout/div</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M796.916,-463.465C782.751,-453.027 763.989,-439.203 748.913,-428.094\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"750.652,-425.027 740.525,-421.913 746.499,-430.663 750.652,-425.027\"/>\n",
       "</g>\n",
       "<!-- dropout/Placeholder&#45;&gt;dropout/Greater -->\n",
       "<g id=\"edge59\" class=\"edge\"><title>dropout/Placeholder&#45;&gt;dropout/Greater</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M824.294,-462.055C827.264,-454.059 830.877,-444.331 834.194,-435.4\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"837.563,-436.382 837.764,-425.789 831.001,-433.945 837.563,-436.382\"/>\n",
       "</g>\n",
       "<!-- dropout/Placeholder&#45;&gt;dropout/LessEqual -->\n",
       "<g id=\"edge62\" class=\"edge\"><title>dropout/Placeholder&#45;&gt;dropout/LessEqual</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M848.425,-465.17C872.783,-454.099 907.101,-438.5 933.393,-426.549\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"935.027,-429.65 942.683,-422.326 932.131,-423.278 935.027,-429.65\"/>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/shape -->\n",
       "<g id=\"node66\" class=\"node\"><title>readout/truncated_normal/shape</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1282\" cy=\"-635\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1282\" y=\"-631.3\" font-family=\"Times,serif\" font-size=\"14.00\">shape</text>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"node69\" class=\"node\"><title>readout/truncated_normal/TruncatedNormal</title>\n",
       "<ellipse fill=\"#d3bff4\" stroke=\"black\" cx=\"1319\" cy=\"-563\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1319\" y=\"-559.3\" font-family=\"Times,serif\" font-size=\"14.00\">TruncatedNormal</text>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/shape&#45;&gt;readout/truncated_normal/TruncatedNormal -->\n",
       "<g id=\"edge65\" class=\"edge\"><title>readout/truncated_normal/shape&#45;&gt;readout/truncated_normal/TruncatedNormal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1290.77,-617.411C1295.08,-609.249 1300.38,-599.22 1305.22,-590.065\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1308.46,-591.433 1310.04,-580.956 1302.27,-588.162 1308.46,-591.433\"/>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/mean -->\n",
       "<g id=\"node67\" class=\"node\"><title>readout/truncated_normal/mean</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1361\" cy=\"-635\" rx=\"30.5947\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1361\" y=\"-631.3\" font-family=\"Times,serif\" font-size=\"14.00\">mean</text>\n",
       "</g>\n",
       "<!-- readout/truncated_normal -->\n",
       "<g id=\"node73\" class=\"node\"><title>readout/truncated_normal</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"1273\" cy=\"-408\" rx=\"73.387\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1273\" y=\"-404.3\" font-family=\"Times,serif\" font-size=\"14.00\">truncated_normal</text>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/mean&#45;&gt;readout/truncated_normal -->\n",
       "<g id=\"edge68\" class=\"edge\"><title>readout/truncated_normal/mean&#45;&gt;readout/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1376.25,-619.342C1385.36,-609.482 1396.09,-595.647 1401,-581 1407.21,-562.457 1408.48,-555.067 1401,-537 1382.04,-491.219 1337.88,-453.275 1306.75,-430.882\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1308.54,-427.865 1298.35,-424.984 1304.52,-433.594 1308.54,-427.865\"/>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/stddev -->\n",
       "<g id=\"node68\" class=\"node\"><title>readout/truncated_normal/stddev</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1194\" cy=\"-563\" rx=\"33.5952\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1194\" y=\"-559.3\" font-family=\"Times,serif\" font-size=\"14.00\">stddev</text>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/mul -->\n",
       "<g id=\"node70\" class=\"node\"><title>readout/truncated_normal/mul</title>\n",
       "<ellipse fill=\"#cae3c6\" stroke=\"black\" cx=\"1273\" cy=\"-480\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1273\" y=\"-476.3\" font-family=\"Times,serif\" font-size=\"14.00\">mul</text>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/stddev&#45;&gt;readout/truncated_normal/mul -->\n",
       "<g id=\"edge63\" class=\"edge\"><title>readout/truncated_normal/stddev&#45;&gt;readout/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1208.86,-546.765C1220.93,-534.391 1238.1,-516.779 1251.65,-502.895\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1254.26,-505.224 1258.74,-495.622 1249.25,-500.337 1254.26,-505.224\"/>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/TruncatedNormal&#45;&gt;readout/truncated_normal/mul -->\n",
       "<g id=\"edge64\" class=\"edge\"><title>readout/truncated_normal/TruncatedNormal&#45;&gt;readout/truncated_normal/mul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1309.24,-544.822C1302.86,-533.572 1294.4,-518.68 1287.26,-506.111\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1290.1,-504.019 1282.12,-497.051 1284.01,-507.475 1290.1,-504.019\"/>\n",
       "</g>\n",
       "<!-- readout/truncated_normal/mul&#45;&gt;readout/truncated_normal -->\n",
       "<g id=\"edge73\" class=\"edge\"><title>readout/truncated_normal/mul&#45;&gt;readout/truncated_normal</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1273,-461.697C1273,-453.983 1273,-444.712 1273,-436.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1276.5,-436.104 1273,-426.104 1269.5,-436.104 1276.5,-436.104\"/>\n",
       "</g>\n",
       "<!-- readout/b/Assign -->\n",
       "<g id=\"node71\" class=\"node\"><title>readout/b/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"1318\" cy=\"-250\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1318\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- readout/b -->\n",
       "<g id=\"node76\" class=\"node\"><title>readout/b</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"1318\" cy=\"-178\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1318\" y=\"-174.3\" font-family=\"Times,serif\" font-size=\"14.00\">b</text>\n",
       "</g>\n",
       "<!-- readout/b/Assign&#45;&gt;readout/b -->\n",
       "<g id=\"edge70\" class=\"edge\"><title>readout/b/Assign&#45;&gt;readout/b</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M1318,-231.697C1318,-223.983 1318,-214.712 1318,-206.112\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"1321.5,-206.104 1318,-196.104 1314.5,-206.104 1321.5,-206.104\"/>\n",
       "</g>\n",
       "<!-- readout/W/Assign -->\n",
       "<g id=\"node72\" class=\"node\"><title>readout/W/Assign</title>\n",
       "<ellipse fill=\"#bbbbff\" stroke=\"black\" cx=\"1234\" cy=\"-333\" rx=\"35.194\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1234\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">Assign</text>\n",
       "</g>\n",
       "<!-- readout/W -->\n",
       "<g id=\"node74\" class=\"node\"><title>readout/W</title>\n",
       "<ellipse fill=\"#ffbbbb\" stroke=\"black\" cx=\"1233\" cy=\"-250\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1233\" y=\"-246.3\" font-family=\"Times,serif\" font-size=\"14.00\">W</text>\n",
       "</g>\n",
       "<!-- readout/W/Assign&#45;&gt;readout/W -->\n",
       "<g id=\"edge67\" class=\"edge\"><title>readout/W/Assign&#45;&gt;readout/W</title>\n",
       "<path fill=\"none\" stroke=\"red\" d=\"M1233.79,-314.822C1233.66,-304.19 1233.49,-290.306 1233.34,-278.204\"/>\n",
       "<polygon fill=\"red\" stroke=\"red\" points=\"1236.84,-278.109 1233.21,-268.153 1229.84,-278.196 1236.84,-278.109\"/>\n",
       "</g>\n",
       "<!-- readout/truncated_normal&#45;&gt;readout/W/Assign -->\n",
       "<g id=\"edge71\" class=\"edge\"><title>readout/truncated_normal&#45;&gt;readout/W/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1263.95,-390.069C1259.1,-380.975 1253.01,-369.58 1247.59,-359.443\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1250.66,-357.759 1242.86,-350.588 1244.49,-361.057 1250.66,-357.759\"/>\n",
       "</g>\n",
       "<!-- readout/W&#45;&gt;readout/MatMul -->\n",
       "<g id=\"edge69\" class=\"edge\"><title>readout/W&#45;&gt;readout/MatMul</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1233,-231.697C1233,-223.983 1233,-214.712 1233,-206.112\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1236.5,-206.104 1233,-196.104 1229.5,-206.104 1236.5,-206.104\"/>\n",
       "</g>\n",
       "<!-- readout/Const -->\n",
       "<g id=\"node75\" class=\"node\"><title>readout/Const</title>\n",
       "<ellipse fill=\"yellow\" stroke=\"black\" cx=\"1318\" cy=\"-333\" rx=\"31.3957\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1318\" y=\"-329.3\" font-family=\"Times,serif\" font-size=\"14.00\">Const</text>\n",
       "</g>\n",
       "<!-- readout/Const&#45;&gt;readout/b/Assign -->\n",
       "<g id=\"edge72\" class=\"edge\"><title>readout/Const&#45;&gt;readout/b/Assign</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1318,-314.822C1318,-304.19 1318,-290.306 1318,-278.204\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1321.5,-278.153 1318,-268.153 1314.5,-278.153 1321.5,-278.153\"/>\n",
       "</g>\n",
       "<!-- readout/add -->\n",
       "<g id=\"node78\" class=\"node\"><title>readout/add</title>\n",
       "<ellipse fill=\"#d3d3f8\" stroke=\"black\" cx=\"1275\" cy=\"-106\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1275\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">add</text>\n",
       "</g>\n",
       "<!-- readout/b&#45;&gt;readout/add -->\n",
       "<g id=\"edge74\" class=\"edge\"><title>readout/b&#45;&gt;readout/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1308.24,-161.116C1302.85,-152.345 1296.05,-141.264 1290,-131.416\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1292.94,-129.51 1284.72,-122.821 1286.97,-133.174 1292.94,-129.51\"/>\n",
       "</g>\n",
       "<!-- readout/MatMul&#45;&gt;readout/add -->\n",
       "<g id=\"edge66\" class=\"edge\"><title>readout/MatMul&#45;&gt;readout/add</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1242.95,-160.411C1248.1,-151.832 1254.49,-141.191 1260.2,-131.67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1263.26,-133.364 1265.41,-122.988 1257.26,-129.762 1263.26,-133.364\"/>\n",
       "</g>\n",
       "<!-- readout/Softmax -->\n",
       "<g id=\"node79\" class=\"node\"><title>readout/Softmax</title>\n",
       "<ellipse fill=\"#b4d7bb\" stroke=\"black\" cx=\"1275\" cy=\"-34\" rx=\"40.8928\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1275\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">Softmax</text>\n",
       "</g>\n",
       "<!-- readout/add&#45;&gt;readout/Softmax -->\n",
       "<g id=\"edge75\" class=\"edge\"><title>readout/add&#45;&gt;readout/Softmax</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1275,-87.6966C1275,-79.9827 1275,-70.7125 1275,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1278.5,-62.1043 1275,-52.1043 1271.5,-62.1044 1278.5,-62.1043\"/>\n",
       "</g>\n",
       "<!-- x -->\n",
       "<g id=\"node80\" class=\"node\"><title>x</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"324\" cy=\"-1861\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"324\" y=\"-1857.3\" font-family=\"Times,serif\" font-size=\"14.00\">x</text>\n",
       "</g>\n",
       "<!-- x&#45;&gt;conv1/Reshape -->\n",
       "<g id=\"edge81\" class=\"edge\"><title>x&#45;&gt;conv1/Reshape</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M308.846,-1845.89C295.745,-1833.69 276.57,-1815.84 261.323,-1801.65\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"263.466,-1798.86 253.762,-1794.61 258.696,-1803.98 263.466,-1798.86\"/>\n",
       "</g>\n",
       "<!-- y_ -->\n",
       "<g id=\"node81\" class=\"node\"><title>y_</title>\n",
       "<ellipse fill=\"#dcf0b7\" stroke=\"black\" cx=\"324\" cy=\"-2183\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"324\" y=\"-2179.3\" font-family=\"Times,serif\" font-size=\"14.00\">y_</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f706c1b3e10>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfdot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cross_entropy = - tf.reduce_sum(y_*tf.log(y_conv))\n",
    "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "prediction = tf.argmax(y_conv, 1)\n",
    "correct_prediction = tf.equal(prediction, tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%timeit -r 1 -n 1\n",
    "tf.initialize_all_variables().run()\n",
    "for i in range(20000):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict = {\n",
    "                x: batch[0], y_: batch[1], keep_prob: 1.0 })\n",
    "        print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "    train_step.run(feed_dict= {x: batch[0], y_: batch[1], keep_prob: 0.5 })\n",
    "for i in range(0, mnist.test.num_examples, 1000):\n",
    "    print(\"test accuracy %g\"%accuracy.eval(feed_dict={x: mnist.test.images[i:i+1000], \n",
    "                                                  y_: mnist.test.labels[i:i+1000],\n",
    "                                                  keep_prob: 1.0}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.mean([accuracy.eval(feed_dict={x: mnist.test.images[i:i+1000], \n",
    "                                   y_: mnist.test.labels[i:i+1000],\n",
    "                                   keep_prob: 1.0}) \n",
    "        for i in range(0, mnist.test.num_examples, 1000)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.scalar_summary(accuracy.op.name, accuracy)\n",
    "summary_op = tf.merge_all_summaries()\n",
    "summary_writer = tf.train.SummaryWriter(\"log1\", graph_def=sess.graph_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%timeit -r 1 -n 1\n",
    "tf.initialize_all_variables().run()\n",
    "for i in range(20000):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict = {\n",
    "                x: batch[0], y_: batch[1], keep_prob: 1.0 })\n",
    "        print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "        summary_str = sess.run(summary_op, feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0 })\n",
    "        summary_writer.add_summary(summary_str, i)\n",
    "    train_step.run(feed_dict= {x: batch[0], y_: batch[1], keep_prob: 0.5 })\n",
    "for i in range(0, mnist.test.num_examples, 1000):\n",
    "    print(\"test accuracy %g\"%accuracy.eval(feed_dict={x: mnist.test.images[i:i+1000], \n",
    "                                                  y_: mnist.test.labels[i:i+1000],\n",
    "                                                  keep_prob: 1.0}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.mean([accuracy.eval(feed_dict={x: mnist.test.images[i:i+1000], \n",
    "                                   y_: mnist.test.labels[i:i+1000],\n",
    "                                   keep_prob: 1.0}) \n",
    "        for i in range(0, mnist.test.num_examples, 1000)]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}