explain-03.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a2b5c0-f230-4fe0-9f2f-4b0d923cd79c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44de3be2-be25-4ff5-80ee-c06ee004c6a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "  return 3*x**2 - 4*x + 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5274645f-92d4-44e0-9ffd-e83f406b4d47",
   "metadata": {},
   "outputs": [],
   "source": [
    "f(3.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98dfdab3-b3a9-45c1-b7cd-dda2d6311e5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = np.arange(-5, 5, 0.25)\n",
    "ys = f(xs)\n",
    "plt.plot(xs,ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f86b8a34-38b5-4b9d-9ccd-60d6c07a93aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "h = 0.00001\n",
    "x = 2/3\n",
    "f(x+h)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49a0a223-9aaa-4ea9-91f4-93d15ae483e1",
   "metadata": {},
   "source": [
    "# So how do we update \"Value\" to track the differentiation steps?\n",
    "## We assign a new item called $gradient$ or `grad`\n",
    "## and we add a `backward()` method to calculate the gradient at each step"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d31ddc6-1ad8-46b4-9741-36b33cafdac1",
   "metadata": {},
   "source": [
    "## So where do we start?\n",
    "### recall $d(x)/d(x) = 1$ always\n",
    "#### So our final value of our expression starts with its .grad set to 1.0\n",
    "### What's the next step?\n",
    "#### Assume the expression was L = a + b (a = 2, b=3), so L = 5\n",
    " - $d(L)/d(L) = 1$\n",
    " - $d(L)/d(a) = d(L)/d(L) * d(L)/d(a) = 1.0 $\n",
    " - $d(L)/d(b) = d(L)/d(L) * d(L)/d(b) = 1.0 $\n",
    "#### Assume expression was L = a*b (a=2, b=3), so L = 6\n",
    " - $d(L)/d(L) = 1$\n",
    " - $d(L)/d(a) = d(L)/d(L) * d(L)/d(a) = 1.0*b = 3 $\n",
    " - $d(L)/d(b) = d(L)/d(L) * d(L)/d(b) = 1.0*a = 2 $\n",
    "#### Assume expression was L = pow(a,b) (a=2,b=3) so L = 8\n",
    " - $d(L)/d(L) = 1$\n",
    " - $d(L)/d(a) = b*a^{b-1}$\n",
    " - $d(L)/d(a) = d(L)/d(L) * d(L)/d(a) = (1*)b*a^{b-1}) = (1*)3*((2)^2) = 12$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5816ebb1-ff1d-4bc7-b4a3-423a1dd50a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "  def __init__(s, data, _c = (), _op = '', label = ''):\n",
    "    s.data = data\n",
    "    s.grad = 0.0\n",
    "    s._backward = lambda : None\n",
    "    s._prev = set(_c)\n",
    "    s._op = _op\n",
    "    s.label = label\n",
    "  def __repr__(s):\n",
    "    return f'V(data={s.data} {s.label} {s._op})'\n",
    "  # when variables are used multiple times, the grad field is overwritten\n",
    "  # so the solution is to accumulate the gradients\n",
    "  def __add__(s, o):\n",
    "    o = o if isinstance(o, Value) else Value(o)\n",
    "    out = Value(s.data + o.data, (s, o), '+')\n",
    "    def _backward():\n",
    "      s.grad += 1.0 * out.grad\n",
    "      o.grad += 1.0 * out.grad\n",
    "    out._backward = _backward\n",
    "    return out\n",
    "  def __mul__(s, o):\n",
    "    o = o if isinstance(o, Value) else Value(o)\n",
    "    out = Value(s.data * o.data, (s, o), '*')\n",
    "    def _backward():\n",
    "      s.grad += o.data * out.grad\n",
    "      o.grad += s.data * out.grad\n",
    "    out._backward = _backward\n",
    "    return out\n",
    "  def __pow__(s, o):\n",
    "    assert isinstance(o, (int, float)), \"only supporting ints and floats for now\"\n",
    "    out = Value(s.data ** o, (s,), f'**{o}')\n",
    "    def _backward():\n",
    "      s.grad += o * ((s.data ** (o - 1))) * out.grad \n",
    "    out._backward = _backward\n",
    "    return out\n",
    "  def exp(s):\n",
    "    x = s.data\n",
    "    out = Value(math.exp(x), (s,), 'exp')\n",
    "    def _backward():\n",
    "      s.grad += out.data * out.grad\n",
    "    out._backward = _backward\n",
    "    return out\n",
    "\n",
    "  def tanh(s):\n",
    "    x = s.data\n",
    "    t = (math.exp(2*x) - 1)/(math.exp(2*x)+1)\n",
    "    out = Value(t, (s,), 'tanh')\n",
    "    def _backward():\n",
    "      s.grad += (1 - t**2) * out.grad\n",
    "      # print(f\" grad:{s.grad} {t} {out.grad}\")\n",
    "    out._backward = _backward \n",
    "    return out\n",
    "  def __rmul__(s, o):\n",
    "    return s * o\n",
    "  def __truediv__(s, o):\n",
    "    return s * (o ** -1)\n",
    "  def __radd__(s, o):\n",
    "    return s + o\n",
    "  def __neg__(s):\n",
    "    return s * -1\n",
    "  def __sub__(s, o):\n",
    "    return s + (-o)\n",
    "\n",
    "  def backward(s):\n",
    "    topo = []\n",
    "    visited = set()\n",
    "    def dfs(v):\n",
    "      if v not in visited:\n",
    "        visited.add(v)\n",
    "        for child in v._prev:\n",
    "          dfs(child)\n",
    "        topo.append(v)\n",
    "    dfs(s)\n",
    "    s.grad = 1.0\n",
    "    for node in reversed(topo):\n",
    "      #print('calling', node, '_backward()')\n",
    "      node._backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "325b9fd9-1a14-4dee-af59-80e406b3ca66",
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "def trace(r):\n",
    "  # builds a set of all nodes and edges in a graph\n",
    "  nodes, edges = set(), set()\n",
    "  def build(v):\n",
    "    if v not in nodes:\n",
    "      nodes.add(v)\n",
    "      for child in v._prev:\n",
    "        edges.add((child, v))\n",
    "        build(child)\n",
    "  build(r)\n",
    "  return nodes, edges\n",
    "def draw_dot(r):\n",
    "  dot = Digraph(format='svg', graph_attr={'rankdir' : 'LR'}) # LR = left to right\n",
    "  nodes, edges = trace(r)\n",
    "  for n in nodes:\n",
    "    uid = str(id(n))\n",
    "    dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
    "    if n._op:\n",
    "      dot.node(name = uid + n._op, label = n._op)\n",
    "      dot.edge(uid + n._op, uid)\n",
    "  for n1, n2 in edges:\n",
    "    dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "  return dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db042258-60b7-46ad-9021-87734230bbaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# L = Value(2.0, label='a') + Value(3.0, label='b')\n",
    "# L = Value(2.0, label='a') * Value(3.0, label='b')\n",
    "# L = Value(2.0, label='a') ** 3.0\n",
    "# L.backward()\n",
    "# draw_dot(L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da10a455-fd90-4e93-9986-9a9d17a9d9de",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = Value(2.0, label='a')\n",
    "b = Value(-3.00, label='b')\n",
    "c = Value(10.0, label='c')\n",
    "# d = a*b + c\n",
    "e = a*b; e.label = 'e'\n",
    "d = e + c; d.label = 'd'\n",
    "# d = a*b + c\n",
    "print(f'{d=} is the value we calculate earlier. Now here is some more expressions')\n",
    "f = Value(-2.0, label = 'f')\n",
    "L = d*f\n",
    "L.label = 'L'\n",
    "print(f'final value {L=}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a01f52d-c6e0-40a3-a840-4f3973debc14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# a = Value(2.)\n",
    "# b = Value(4.0)\n",
    "# a/ b\n",
    "# a * (1/b)\n",
    "# a * (b**-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0ab0e19-e9bd-4712-9c75-29197aaf8ec0",
   "metadata": {},
   "outputs": [],
   "source": [
    "L.backward()\n",
    "draw_dot(L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93abbd40-64fb-4c0f-bb57-b1e5b643612f",
   "metadata": {},
   "outputs": [],
   "source": [
    "L = d * f; L.label='L'\n",
    "# dl/dd =? f\n",
    "# (f(x+h) - f(x))/h\n",
    "# ((d+h)*f - d*f)/h\n",
    "# d*f + h* f - d*f)/h\n",
    "# (h*f)/g\n",
    "# f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f56af73-01dc-49b2-a4ac-51b39375d580",
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple1():\n",
    "  h = 0.0001\n",
    "  a = Value(2.0, label='a')\n",
    "  b = Value(-3.00, label='b')\n",
    "  c = Value(10.0, label='c')\n",
    "  e = a*b; e.label = 'e'\n",
    "  d = e + c; d.label = 'd'\n",
    "  f = Value(-2.0, label = 'f')\n",
    "  L = d*f\n",
    "  L.label = 'L'\n",
    "  L1 = L.data\n",
    "\n",
    "  a = Value(2.0, label='a')\n",
    "  b = Value(-3.0, label='b')\n",
    "  c = Value(10.0, label='c')\n",
    "  e = a*b; e.label = 'e'\n",
    "  d = e + c; d.label = 'd'\n",
    "  f = Value(-2.0, label = 'f')\n",
    "  L = d*f\n",
    "  # L.data += h\n",
    "  L2 = L.data\n",
    "  if L2 != L1:\n",
    "    print(f'{L1=} {L2=}  {(L2 - L1)/h=}')\n",
    "  else:\n",
    "    print(f'{L=}')\n",
    "simple1()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82e7c8cf-b35d-4f72-938f-a646034f6d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "#L.grad = 1.0\n",
    "#f.grad = 4.0\n",
    "#d.grad = -2\n",
    "#e.grad = -2\n",
    "#c.grad = -2\n",
    "#b.grad = -4\n",
    "#a.grad = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f75319c-d98a-4890-90a6-348a73bf8719",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "a = Value(-2.0, label='a')\n",
    "b = Value(3.0, label = 'b')\n",
    "d = a * b; d.label = 'd'\n",
    "e = a + b; e.label = 'e'\n",
    "f = d * e; f.label = 'f'\n",
    "\n",
    "draw_dot(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37799b8f-e698-40f7-8f92-83867b55927a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "f.backward()\n",
    "draw_dot(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4cded34-20d0-485b-b367-3dc087ee079f",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.arange(-5,5, 0.2), np.tanh(np.arange(-5,5,0.2))); plt.grid();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22dec391-e903-410f-9337-05a3199ef242",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs x1, x2\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "# weights\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "# bias of the neuron\n",
    "bb = Value(6.8813735870195432, label='b')\n",
    "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + bb; n.label = 'n'\n",
    "o = n.tanh(); o.label = 'o'\n",
    "o.backward()\n",
    "draw_dot(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20d3c47d-c929-489f-90d2-89c3c7a80c91",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs x1, x2data:image/svg+xml,%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22UTF-8%22%20standalone%3D%22no%22%3F%3E%0A%3C!DOCTYPE%20svg%20PUBLIC%20%22-%2F%2FW3C%2F%2FDTD%20SVG%201.1%2F%2FEN%22%0A%20%22http%3A%2F%2Fwww.w3.org%2FGraphics%2FSVG%2F1.1%2FDTD%2Fsvg11.dtd%22%3E%0A%3C!--%20Generated%20by%20graphviz%20version%202.48.0%20(0)%0A%20--%3E%0A%3C!--%20Pages%3A%201%20--%3E%0A%3Csvg%20width%3D%221580pt%22%20height%3D%22210pt%22%0A%20viewBox%3D%220.00%200.00%201580.00%20210.00%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20xmlns%3Axlink%3D%22http%3A%2F%2Fwww.w3.org%2F1999%2Fxlink%22%3E%0A%3Cg%20id%3D%22graph0%22%20class%3D%22graph%22%20transform%3D%22scale(1%201)%20rotate(0)%20translate(4%20206)%22%3E%0A%3Cpolygon%20fill%3D%22white%22%20stroke%3D%22transparent%22%20points%3D%22-4%2C4%20-4%2C-206%201576%2C-206%201576%2C4%20-4%2C4%22%2F%3E%0A%3C!--%20139917773591568%20--%3E%0A%3Cg%20id%3D%22node1%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773591568%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22327%2C-55.5%20327%2C-91.5%20548%2C-91.5%20548%2C-55.5%20327%2C-55.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22353.5%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ex1*w1%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22380%2C-55.5%20380%2C-91.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22422.5%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%20%26%2345%3B6.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22465%2C-55.5%20465%2C-91.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22506.5%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139918157674320%2B%20--%3E%0A%3Cg%20id%3D%22node12%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139918157674320%2B%3C%2Ftitle%3E%0A%3Cellipse%20fill%3D%22none%22%20stroke%3D%22black%22%20cx%3D%22611%22%20cy%3D%22-100.5%22%20rx%3D%2227%22%20ry%3D%2218%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22611%22%20y%3D%22-96.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3E%2B%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773591568%26%2345%3B%26gt%3B139918157674320%2B%20--%3E%0A%3Cg%20id%3D%22edge10%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773591568%26%2345%3B%26gt%3B139918157674320%2B%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M548.08%2C-90.75C557.49%2C-92.23%20566.48%2C-93.65%20574.54%2C-94.92%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22574.16%2C-98.4%20584.58%2C-96.5%20575.25%2C-91.49%20574.16%2C-98.4%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773591568*%20--%3E%0A%3Cg%20id%3D%22node2%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773591568*%3C%2Ftitle%3E%0A%3Cellipse%20fill%3D%22none%22%20stroke%3D%22black%22%20cx%3D%22264%22%20cy%3D%22-73.5%22%20rx%3D%2227%22%20ry%3D%2218%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22264%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3E*%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773591568*%26%2345%3B%26gt%3B139917773591568%20--%3E%0A%3Cg%20id%3D%22edge1%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773591568*%26%2345%3B%26gt%3B139917773591568%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M291.05%2C-73.5C298.48%2C-73.5%20307.1%2C-73.5%20316.36%2C-73.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22316.64%2C-77%20326.64%2C-73.5%20316.64%2C-70%20316.64%2C-77%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773593616%20--%3E%0A%3Cg%20id%3D%22node3%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773593616%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%224%2C-165.5%204%2C-201.5%20197%2C-201.5%20197%2C-165.5%204%2C-165.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2219%22%20y%3D%22-179.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ex2%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%2234%2C-165.5%2034%2C-201.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2274%22%20y%3D%22-179.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%200.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22114%2C-165.5%20114%2C-201.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22155.5%22%20y%3D%22-179.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773589968*%20--%3E%0A%3Cg%20id%3D%22node15%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773589968*%3C%2Ftitle%3E%0A%3Cellipse%20fill%3D%22none%22%20stroke%3D%22black%22%20cx%3D%22264%22%20cy%3D%22-128.5%22%20rx%3D%2227%22%20ry%3D%2218%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22264%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3E*%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773593616%26%2345%3B%26gt%3B139917773589968*%20--%3E%0A%3Cg%20id%3D%22edge11%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773593616%26%2345%3B%26gt%3B139917773589968*%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M173.4%2C-165.46C182.75%2C-162.68%20192.16%2C-159.67%20201%2C-156.5%20211.53%2C-152.72%20222.76%2C-147.9%20232.72%2C-143.34%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22234.25%2C-146.48%20241.83%2C-139.07%20231.29%2C-140.14%20234.25%2C-146.48%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917810380880%20--%3E%0A%3Cg%20id%3D%22node4%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917810380880%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22718%2C-137.5%20718%2C-173.5%20904%2C-173.5%20904%2C-137.5%20718%2C-137.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22729.5%22%20y%3D%22-151.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Eb%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22741%2C-137.5%20741%2C-173.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22781%22%20y%3D%22-151.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%206.8814%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22821%2C-137.5%20821%2C-173.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22862.5%22%20y%3D%22-151.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917807855888%2B%20--%3E%0A%3Cg%20id%3D%22node9%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917807855888%2B%3C%2Ftitle%3E%0A%3Cellipse%20fill%3D%22none%22%20stroke%3D%22black%22%20cx%3D%221011%22%20cy%3D%22-127.5%22%20rx%3D%2227%22%20ry%3D%2218%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221011%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3E%2B%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917810380880%26%2345%3B%26gt%3B139917807855888%2B%20--%3E%0A%3Cg%20id%3D%22edge7%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917810380880%26%2345%3B%26gt%3B139917807855888%2B%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M904.04%2C-142.48C929.04%2C-138.95%20954.63%2C-135.33%20974.4%2C-132.53%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22975.05%2C-135.98%20984.46%2C-131.11%20974.07%2C-129.05%20975.05%2C-135.98%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917809751120%20--%3E%0A%3Cg%20id%3D%22node5%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917809751120%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%221386%2C-109.5%201386%2C-145.5%201572%2C-145.5%201572%2C-109.5%201386%2C-109.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221397.5%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Eo%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%221409%2C-109.5%201409%2C-145.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221449%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%200.7071%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%221489%2C-109.5%201489%2C-145.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221530.5%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%201.0000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917809751120tanh%20--%3E%0A%3Cg%20id%3D%22node6%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917809751120tanh%3C%2Ftitle%3E%0A%3Cellipse%20fill%3D%22none%22%20stroke%3D%22black%22%20cx%3D%221323%22%20cy%3D%22-127.5%22%20rx%3D%2227%22%20ry%3D%2218%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221323%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Etanh%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917809751120tanh%26%2345%3B%26gt%3B139917809751120%20--%3E%0A%3Cg%20id%3D%22edge2%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917809751120tanh%26%2345%3B%26gt%3B139917809751120%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M1350.12%2C-127.5C1357.62%2C-127.5%201366.29%2C-127.5%201375.5%2C-127.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%221375.69%2C-131%201385.69%2C-127.5%201375.69%2C-124%201375.69%2C-131%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773164176%20--%3E%0A%3Cg%20id%3D%22node7%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773164176%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%220%2C-55.5%200%2C-91.5%20201%2C-91.5%20201%2C-55.5%200%2C-55.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2216.5%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ew1%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%2233%2C-55.5%2033%2C-91.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2275.5%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%20%26%2345%3B3.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22118%2C-55.5%20118%2C-91.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22159.5%22%20y%3D%22-69.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%201.0000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773164176%26%2345%3B%26gt%3B139917773591568*%20--%3E%0A%3Cg%20id%3D%22edge13%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773164176%26%2345%3B%26gt%3B139917773591568*%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M201.02%2C-73.5C210.04%2C-73.5%20218.72%2C-73.5%20226.59%2C-73.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22226.77%2C-77%20236.77%2C-73.5%20226.77%2C-70%20226.77%2C-77%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917807855888%20--%3E%0A%3Cg%20id%3D%22node8%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917807855888%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%221074%2C-109.5%201074%2C-145.5%201260%2C-145.5%201260%2C-109.5%201074%2C-109.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221085.5%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3En%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%221097%2C-109.5%201097%2C-145.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221137%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%200.8814%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%221177%2C-109.5%201177%2C-145.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%221218.5%22%20y%3D%22-123.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917807855888%26%2345%3B%26gt%3B139917809751120tanh%20--%3E%0A%3Cg%20id%3D%22edge8%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917807855888%26%2345%3B%26gt%3B139917809751120tanh%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M1260.25%2C-127.5C1269.15%2C-127.5%201277.76%2C-127.5%201285.59%2C-127.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%221285.73%2C-131%201295.73%2C-127.5%201285.73%2C-124%201285.73%2C-131%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917807855888%2B%26%2345%3B%26gt%3B139917807855888%20--%3E%0A%3Cg%20id%3D%22edge3%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917807855888%2B%26%2345%3B%26gt%3B139917807855888%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M1038.12%2C-127.5C1045.62%2C-127.5%201054.29%2C-127.5%201063.5%2C-127.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%221063.69%2C-131%201073.69%2C-127.5%201063.69%2C-124%201063.69%2C-131%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917810381584%20--%3E%0A%3Cg%20id%3D%22node10%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917810381584%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%222.5%2C-110.5%202.5%2C-146.5%20198.5%2C-146.5%20198.5%2C-110.5%202.5%2C-110.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2219%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ew2%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%2235.5%2C-110.5%2035.5%2C-146.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2275.5%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%201.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22115.5%2C-110.5%20115.5%2C-146.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22157%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.0000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917810381584%26%2345%3B%26gt%3B139917773589968*%20--%3E%0A%3Cg%20id%3D%22edge12%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917810381584%26%2345%3B%26gt%3B139917773589968*%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M198.71%2C-128.5C208.57%2C-128.5%20218.09%2C-128.5%20226.65%2C-128.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22226.92%2C-132%20236.92%2C-128.5%20226.92%2C-125%20226.92%2C-132%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139918157674320%20--%3E%0A%3Cg%20id%3D%22node11%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139918157674320%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22674%2C-82.5%20674%2C-118.5%20948%2C-118.5%20948%2C-82.5%20674%2C-82.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22727%22%20y%3D%22-96.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ex1*w1%20%2B%20x2*w2%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22780%2C-82.5%20780%2C-118.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22822.5%22%20y%3D%22-96.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%20%26%2345%3B6.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22865%2C-82.5%20865%2C-118.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22906.5%22%20y%3D%22-96.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139918157674320%26%2345%3B%26gt%3B139917807855888%2B%20--%3E%0A%3Cg%20id%3D%22edge14%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139918157674320%26%2345%3B%26gt%3B139917807855888%2B%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M944.07%2C-118.51C954.98%2C-120%20965.29%2C-121.4%20974.36%2C-122.64%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22974.04%2C-126.13%20984.42%2C-124.01%20974.99%2C-119.19%20974.04%2C-126.13%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139918157674320%2B%26%2345%3B%26gt%3B139918157674320%20--%3E%0A%3Cg%20id%3D%22edge4%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139918157674320%2B%26%2345%3B%26gt%3B139918157674320%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M638.29%2C-100.5C645.71%2C-100.5%20654.34%2C-100.5%20663.71%2C-100.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22663.72%2C-104%20673.72%2C-100.5%20663.72%2C-97%20663.72%2C-104%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773590864%20--%3E%0A%3Cg%20id%3D%22node13%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773590864%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%222%2C-0.5%202%2C-36.5%20199%2C-36.5%20199%2C-0.5%202%2C-0.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2217%22%20y%3D%22-14.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ex1%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%2232%2C-0.5%2032%2C-36.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%2272%22%20y%3D%22-14.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%202.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22112%2C-0.5%20112%2C-36.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22155.5%22%20y%3D%22-14.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%20%26%2345%3B1.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773590864%26%2345%3B%26gt%3B139917773591568*%20--%3E%0A%3Cg%20id%3D%22edge9%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773590864%26%2345%3B%26gt%3B139917773591568*%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M170.4%2C-36.57C180.73%2C-39.66%20191.2%2C-43%20201%2C-46.5%20211.28%2C-50.17%20222.28%2C-54.73%20232.11%2C-59.06%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22230.93%2C-62.37%20241.48%2C-63.27%20233.79%2C-55.98%20230.93%2C-62.37%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773589968%20--%3E%0A%3Cg%20id%3D%22node14%22%20class%3D%22node%22%3E%0A%3Ctitle%3E139917773589968%3C%2Ftitle%3E%0A%3Cpolygon%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22329.5%2C-110.5%20329.5%2C-146.5%20545.5%2C-146.5%20545.5%2C-110.5%20329.5%2C-110.5%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22356%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Ex2*w2%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22382.5%2C-110.5%20382.5%2C-146.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22422.5%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Edata%200.0000%3C%2Ftext%3E%0A%3Cpolyline%20fill%3D%22none%22%20stroke%3D%22black%22%20points%3D%22462.5%2C-110.5%20462.5%2C-146.5%20%22%2F%3E%0A%3Ctext%20text-anchor%3D%22middle%22%20x%3D%22504%22%20y%3D%22-124.8%22%20font-family%3D%22Times%2Cserif%22%20font-size%3D%2214.00%22%3Egrad%200.5000%3C%2Ftext%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773589968%26%2345%3B%26gt%3B139918157674320%2B%20--%3E%0A%3Cg%20id%3D%22edge6%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773589968%26%2345%3B%26gt%3B139918157674320%2B%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M545.65%2C-111.01C555.95%2C-109.32%20565.81%2C-107.71%20574.59%2C-106.28%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22575.4%2C-109.7%20584.7%2C-104.63%20574.27%2C-102.79%20575.4%2C-109.7%22%2F%3E%0A%3C%2Fg%3E%0A%3C!--%20139917773589968*%26%2345%3B%26gt%3B139917773589968%20--%3E%0A%3Cg%20id%3D%22edge5%22%20class%3D%22edge%22%3E%0A%3Ctitle%3E139917773589968*%26%2345%3B%26gt%3B139917773589968%3C%2Ftitle%3E%0A%3Cpath%20fill%3D%22none%22%20stroke%3D%22black%22%20d%3D%22M291.05%2C-128.5C299.21%2C-128.5%20308.81%2C-128.5%20319.12%2C-128.5%22%2F%3E%0A%3Cpolygon%20fill%3D%22black%22%20stroke%3D%22black%22%20points%3D%22319.2%2C-132%20329.2%2C-128.5%20319.2%2C-125%20319.2%2C-132%22%2F%3E%0A%3C%2Fg%3E%0A%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "# weights\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "# bias of the neuron\n",
    "bb = Value(6.8813735870195432, label='b')\n",
    "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n",
    "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n",
    "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label='x1*w1 + x2*w2'\n",
    "n = x1w1x2w2 + bb; n.label = 'n'\n",
    "# o = n.tanh(); o.label = 'o'\n",
    "e = (2*n).exp()\n",
    "o = (e-1)/(e+1)\n",
    "o.backward()\n",
    "draw_dot(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e0cf9cd-691d-4677-8b58-bdc2a49018e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "x1 = torch.Tensor([2.0]).double()   ; x1.requires_grad = True\n",
    "x2 = torch.Tensor([0.0]).double()   ; x2.requires_grad = True \n",
    "w1 = torch.Tensor([-3.0]).double()  ; w1.requires_grad = True\n",
    "w2 = torch.Tensor([1.0]).double()   ; w2.requires_grad = True\n",
    "b = torch.Tensor([6.8813735870195432]).double() ; b.requires_grad = True\n",
    "n = x1 * w1 + x2 * w2 + b\n",
    "o = torch.tanh(n)\n",
    "\n",
    "print(o.data.item())\n",
    "o.backward()\n",
    "\n",
    "print('-----')\n",
    "print('x1.grad', x1.grad.item())\n",
    "print('x2.grad', x2.grad.item())\n",
    "print('w1.grad', w1.grad.item())\n",
    "print('w2.grad', w2.grad.item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc6ef5ee-b49b-4774-851b-01d7140850d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "class Neuron:\n",
    "  def __init__(s, nin):\n",
    "    s.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "    s.b = Value(random.uniform(-1,1))\n",
    "  def __call__(s, x):\n",
    "    # w * x + b\n",
    "    act = sum((wi * xi for wi, xi in zip(s.w, x)), s.b)\n",
    "    out = act.tanh()\n",
    "    return out\n",
    "  def parameters(s):\n",
    "    return s.w + [s.b]\n",
    "class Layer:\n",
    "  def __init__(s, nin, nout):\n",
    "    s.neurons = [ Neuron(nin) for _ in range(nout) ]\n",
    "  def __call__(s, x):\n",
    "    outs = [ n(x) for n in s.neurons ]\n",
    "    return outs\n",
    "  def parameters(s):\n",
    "    return[ p for neuron in s.neurons for p in neuron.parameters() ]\n",
    "\n",
    "class MLP:\n",
    "  def __init__(s, nin, nouts):\n",
    "    \"\"\" nouts in a list of sizes for the individual layers \"\"\"\n",
    "    sz = [nin] + nouts\n",
    "    s.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
    "  def __call__(s,x):\n",
    "    for layer in s.layers:\n",
    "      x = layer(x)\n",
    "    return x[0] if len(x) == 1 else x\n",
    "  def parameters(s):\n",
    "    return [ p for layer in s.layers for p in layer.parameters() ]\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2340efef-91d5-4c54-a8a4-ad3119dc46f8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "xs = [\n",
    "  [2.0, 3.0, -1.0],\n",
    "  [3.0, -1.0, 0.5],\n",
    "  [0.5, 1.0, 1.0],\n",
    "  [1.0, 1.0, -1.0],\n",
    "]\n",
    "ys = [ 1.0, -1.0, -1.0, 1.0] # desired targets\n",
    "np.random.seed(1337)\n",
    "random.seed(1337)\n",
    "\n",
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "z = n(x)\n",
    "print('number of params', len(n.parameters()))\n",
    "draw_dot(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d3c4886-d06a-4e97-859a-71d3fdef7e4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('num params', len(n.parameters()))\n",
    "ypred = [ n(x) for x in xs ]\n",
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57af7f5a-f032-416c-b5d5-60a8cfdf4c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred = [ n(x) for x in xs ]\n",
    "print(ypred)\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss.backward()\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b78da75-07a1-4a88-9b1c-74e0232535c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(loss)\n",
    "draw_dot(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe6877f7-148f-48db-ae29-4c2fb5fcad52",
   "metadata": {},
   "outputs": [],
   "source": [
    "[ (p.data, p.grad) for p in n.parameters() ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f341b03-4d31-440d-bf50-5591acee6181",
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in n.parameters():\n",
    "  p.data += -0.01 * p.grad\n",
    "  p.grad = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00380ccd-4578-4c55-9ebc-55165c1c5f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred = [ n(x) for x in xs ]\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c82583b-044a-466a-9a09-13bfde5188ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in n.parameters():\n",
    "  p.data += -0.01 * p.grad\n",
    "  p.grad = 0.0\n",
    "\n",
    "ypred = [ n(x) for x in xs ]\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bee25bbe-af94-4fbf-97df-8ec2f6b7027c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in n.parameters():\n",
    "  p.data += -0.01 * p.grad\n",
    "  p.grad = 0.0\n",
    "\n",
    "ypred = [ n(x) for x in xs ]\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e0bd946-8674-429d-b280-5c46ee37f74f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in n.parameters():\n",
    "  p.data += -0.01 * p.grad\n",
    "  p.grad = 0.0\n",
    "ypred = [ n(x) for x in xs ]\n",
    "loss = sum([(yout - ygt)**2 for ygt, yout in zip(ys, ypred)])\n",
    "loss.backward()\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4a27c39-487c-45ab-b96c-354cdd1bce8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1337)\n",
    "random.seed(1337)\n",
    "\n",
    "xs = [\n",
    "  [2.0, 3.0, -1.0],\n",
    "  [3.0, -1.0, 0.5],\n",
    "  [0.5, 1.0, 1.0],\n",
    "  [1.0, 1.0, -1.0],\n",
    "]\n",
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "z = n(x)\n",
    "n.parameters()\n",
    "print('num params', len(n.parameters()))\n",
    "ys = [ 1.0, -1.0, -1.0, 1.0] # desired targets\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab83a05d-4253-4a7e-9fbb-711edbde92fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in range(20):\n",
    "  # forward pass\n",
    "  ypred = [n(x) for x in xs ]\n",
    "  loss = sum((yout-ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "  # backward pass\n",
    "  for p in n.parameters():\n",
    "    p.grad = 0.0\n",
    "  loss.backward()\n",
    "  # update\n",
    "  for p in n.parameters():\n",
    "    p.data += -0.01 *p.grad\n",
    "  print(k, loss.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "735eb73e-3145-4222-9857-d0c55dc1a117",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}