32
32
33
33
singa_dtype = {"float16" : tensor .float16 , "float32" : tensor .float32 }
34
34
35
+
35
36
#### self-defined loss begin
36
37
37
38
### from autograd.py
@@ -62,11 +63,13 @@ def backward(self, dy=1.0):
62
63
dx *= dy
63
64
return dx
64
65
66
+
65
67
def se_loss (x ):
66
68
# assert x.shape == t.shape, "input and target shape different: %s, %s" % (
67
69
# x.shape, t.shape)
68
70
return SumError ()(x )[0 ]
69
71
72
+
70
73
### from layer.py
71
74
class SumErrorLayer (Layer ):
72
75
"""
@@ -79,6 +82,7 @@ def __init__(self):
79
82
def forward (self , x ):
80
83
return se_loss (x )
81
84
85
+
82
86
#### self-defined loss end
83
87
84
88
class MSMLP (model .Model ):
@@ -92,7 +96,6 @@ def __init__(self, data_size=10, perceptron_size=100, num_classes=10):
92
96
self .linear1 = layer .Linear (perceptron_size )
93
97
self .linear2 = layer .Linear (num_classes )
94
98
self .softmax_cross_entropy = layer .SoftMaxCrossEntropy ()
95
-
96
99
self .sum_error = SumErrorLayer ()
97
100
98
101
def forward (self , inputs ):
@@ -101,12 +104,24 @@ def forward(self, inputs):
101
104
y = self .linear2 (y )
102
105
return y
103
106
104
- def train_one_batch (self , x , y , synflow_flag , dist_option , spars ):
107
+ def train_one_batch (self , x , y , dist_option , spars , synflow_flag ):
108
+ # print ("in train_one_batch")
105
109
out = self .forward (x )
106
- loss = self .softmax_cross_entropy (out , y )
110
+ # print ("train_one_batch x.data: \n", x.data)
111
+ # print ("train_one_batch y.data: \n", y.data)
112
+ # print ("train_one_batch out.data: \n", out.data)
113
+ if synflow_flag :
114
+ # print ("sum_error")
115
+ loss = self .sum_error (out )
116
+ else : # normal training
117
+ # print ("softmax_cross_entropy")
118
+ loss = self .softmax_cross_entropy (out , y )
119
+ # print ("train_one_batch loss.data: \n", loss.data)
107
120
108
121
if dist_option == 'plain' :
122
+ # print ("before pn_p_g_list = self.optimizer(loss)")
109
123
pn_p_g_list = self .optimizer (loss )
124
+ # print ("after pn_p_g_list = self.optimizer(loss)")
110
125
elif dist_option == 'half' :
111
126
self .optimizer .backward_and_update_half (loss )
112
127
elif dist_option == 'partialUpdate' :
@@ -119,17 +134,24 @@ def train_one_batch(self, x, y, synflow_flag, dist_option, spars):
119
134
self .optimizer .backward_and_sparse_update (loss ,
120
135
topK = False ,
121
136
spars = spars )
137
+ # print ("len(pn_p_g_list): \n", len(pn_p_g_list))
138
+ # print ("len(pn_p_g_list[0]): \n", len(pn_p_g_list[0]))
139
+ # print ("pn_p_g_list[0][0]: \n", pn_p_g_list[0][0])
140
+ # print ("pn_p_g_list[0][1].data: \n", pn_p_g_list[0][1].data)
141
+ # print ("pn_p_g_list[0][2].data: \n", pn_p_g_list[0][2].data)
122
142
return pn_p_g_list , out , loss
143
+ # return pn_p_g_list[0], pn_p_g_list[1], pn_p_g_list[2], out, loss
123
144
124
145
def set_optimizer (self , optimizer ):
125
146
self .optimizer = optimizer
126
147
127
148
128
149
def create_model (pretrained = False , ** kwargs ):
129
150
"""Constructs a CNN model.
151
+
130
152
Args:
131
153
pretrained (bool): If True, returns a pre-trained model.
132
-
154
+
133
155
Returns:
134
156
The created CNN model.
135
157
"""
@@ -196,4 +218,4 @@ def create_model(pretrained=False, **kwargs):
196
218
out , loss = model (tx , ty , 'fp32' , spars = None )
197
219
198
220
if i % 100 == 0 :
199
- print ("training loss = " , tensor .to_numpy (loss )[0 ])
221
+ print ("training loss = " , tensor .to_numpy (loss )[0 ])
0 commit comments