Added sigmoid activation, support for disabling grad

src/activation_functions/relu.py

@@ -6,5 +6,5 @@ def forward(self, input):
 
     def _build_backward_function(self, input, out):
         def _backward():
-            input.grad += (out.value > 0) * out.grad
+            input.grad += (out.value > 0) * out.grad if out.requires_grad else 0
         return _backward

src/activation_functions/sigmoid.py

@@ -0,0 +1,14 @@
+from activation_functions.activation import ActivationFunction
+import math
+
+class SigmoidActivation(ActivationFunction):
+    def _simgoid(self, x):
+        return 1 / (1 + math.exp(-x))
+
+    def forward(self, input):
+        return self._simgoid(input.value)
+
+    def _build_backward_function(self, input, out):
+        def _backward():
+            input.grad += out.grad * out.value * (1 - out.value) if out.requires_grad else 0
+        return _backward

src/ops/basic_ops.py

@@ -8,8 +8,8 @@ def forward(self, a, b):
 
     def _build_backward_function(self, a, b, out):
         def _backward():
-            a.grad += out.grad
-            b.grad += out.grad
+            a.grad += out.grad  if a.requires_grad else 0
+            b.grad += out.grad  if b.requires_grad else 0
         return _backward
 
 
@@ -19,8 +19,8 @@ def forward(self, a, b):
 
     def _build_backward_function(self, a, b, out):
         def _backward():
-            a.grad += b.value * out.grad
-            b.grad += a.value * out.grad
+            a.grad += b.value * out.grad if a.requires_grad else 0
+            b.grad += a.value * out.grad if b.requires_grad else 0
         return _backward
 
 
@@ -30,8 +30,8 @@ def forward(self, a, b):
 
     def _build_backward_function(self, a, b, out):
         def _backward():
-            a.grad += out.grad
-            b.grad -= out.grad
+            a.grad += out.grad  if a.requires_grad else 0
+            b.grad -= out.grad  if b.requires_grad else 0
         return _backward
 
 class PowOperation(Operation):
@@ -40,8 +40,8 @@ def forward(self, a, b):
 
     def _build_backward_function(self, a, b, out):
         def _backward():
-            a.grad += b.value * (a.value ** (b.value - 1)) * out.grad
-            b.grad += (a.value ** b.value) * math.log(a.value) * out.grad
+            a.grad += b.value * (a.value ** (b.value - 1)) * out.grad  if a.requires_grad else 0
+            b.grad += (a.value ** b.value) * math.log(a.value) * out.grad  if b.requires_grad else 0
         return _backward
 
 class DivOperation(Operation):
@@ -50,7 +50,7 @@ def forward(self, a, b):
 
     def _build_backward_function(self, a, b, out):
         def _backward():
-            a.grad += (1/b.value) * out.grad
-            b.grad -= (a.value / (b.value **2)) * out.grad
+            a.grad += (1/b.value) * out.grad  if a.requires_grad else 0
+            b.grad -= (a.value / (b.value **2)) * out.grad  if b.requires_grad else 0
         return _backward
 

src/variable/variable.py

@@ -1,13 +1,14 @@
 from ops.basic_ops import *
 
 class Variable:
-    def __init__(self, value, _children=(), _op=None, _name=None) -> None:
+    def __init__(self, value, _children=(), _op=None, _name=None, requires_grad=True) -> None:
         self.value = value
         self.grad = 0
         self._op = _op
         self._prev = set(_children)
         self._backward = lambda: None
         self._name = _name
+        self.requires_grad = requires_grad
 
     def __add__(self, other):
         return self._apply(AddOperation(), other)