Add testing framework

Author: pplonski

README.md

@@ -2,7 +2,7 @@
 
 This is a bunch of code to port Keras neural network model into pure C++. Neural network weights and architecture are stored in plain text file and input is presented as `vector<vector<vector<float> > >` in case of image. The code is prepared to support simple Convolutional network (from MNIST example) but can be easily extended. There are implemented only ReLU and Softmax activations.
 
-It is working regardless the Keras backend.
+It is working with the Theano backend - support for Tensorflow will be added soon.
 
 #Usage
 
@@ -16,3 +16,7 @@ It is working regardless the Keras backend.
  2. Dump network to plain text file `python dump_to_simple_cpp.py -a example/my_nn_arch.json -w example/my_nn_weights.h5 -o example/dumped.nnet`.
  3. Compile example `g++ -std=c++11 keras_model.cc example_main.cc` - see code in `example_main.cc`.
  4. Run binary `./a.out` - you shoul get the same output as in step one from Keras.
+
+#Testing
+
+If you want to test dumping for your network, please use `test_run.sh` script. Please provide there your network architecture and weights. 

dump_to_simple_cpp.py

@@ -10,14 +10,13 @@
 parser.add_argument('-a', '--architecture', help="JSON with model architecture", required=True)
 parser.add_argument('-w', '--weights', help="Model weights in HDF5 format", required=True)
 parser.add_argument('-o', '--output', help="Ouput file name", required=True)
-
+parser.add_argument('-v', '--verbose', help="Verbose", required=False)
 args = parser.parse_args()
 
 print 'Read architecture from', args.architecture
 print 'Read weights from', args.weights
 print 'Writing to', args.output
 
-
 arch = open(args.architecture).read()
 model = model_from_json(arch)
 model.load_weights(args.weights)
@@ -29,10 +28,12 @@
 
     layers = []
     for ind, l in enumerate(arch["config"]):
-        print ind, l
+        if args.verbose:
+            print ind, l
         fout.write('layer ' + str(ind) + ' ' + l['class_name'] + '\n')
 
-        print str(ind), l['class_name']
+        if args.verbose:
+            print str(ind), l['class_name']
         layers += [l['class_name']]
         if l['class_name'] == 'Convolution2D':
             #fout.write(str(l['config']['nb_filter']) + ' ' + str(l['config']['nb_col']) + ' ' + str(l['config']['nb_row']) + ' ')
@@ -42,7 +43,8 @@
             #fout.write('\n')
 
             W = model.layers[ind].get_weights()[0]
-            print W.shape
+            if args.verbose:
+                print W.shape
             fout.write(str(W.shape[0]) + ' ' + str(W.shape[1]) + ' ' + str(W.shape[2]) + ' ' + str(W.shape[3]) + ' ' + l['config']['border_mode'] + '\n')
 
             for i in range(W.shape[0]):
@@ -60,7 +62,8 @@
         if l['class_name'] == 'Dense':
             #fout.write(str(l['config']['output_dim']) + '\n')
             W = model.layers[ind].get_weights()[0]
-            print W.shape
+            if args.verbose:
+                print W.shape
             fout.write(str(W.shape[0]) + ' ' + str(W.shape[1]) + '\n')
 
 

example_main.cc

@@ -21,7 +21,7 @@ int main() {
   DataChunk *sample = new DataChunk2D();
   sample->read_from_file("./example/sample_mnist.dat");
   std::cout << sample->get_3d().size() << std::endl;
-  KerasModel m("./example/dumped.nnet");
+  KerasModel m("./example/dumped.nnet", true);
   m.compute_output(sample);
   delete sample;
 

keras_model.cc

@@ -44,8 +44,8 @@ void keras::LayerConv2D::load_weights(std::ifstream &fin) {
   fin >> m_kernels_cnt >> m_depth >> m_rows >> m_cols >> m_border_mode;
   if (m_border_mode == "[") { m_border_mode = "valid"; skip = true; }
 
-  cout << "LayerConv2D " << m_kernels_cnt << "x" << m_depth << "x" << m_rows <<
-            "x" << m_cols << " border_mode " << m_border_mode << endl;
+  //cout << "LayerConv2D " << m_kernels_cnt << "x" << m_depth << "x" << m_rows <<
+  //            "x" << m_cols << " border_mode " << m_border_mode << endl;
   // reading kernel weights
   for(int k = 0; k < m_kernels_cnt; ++k) {
     vector<vector<vector<float> > > tmp_depths;
@@ -78,12 +78,12 @@ void keras::LayerConv2D::load_weights(std::ifstream &fin) {
 
 void keras::LayerActivation::load_weights(std::ifstream &fin) {
   fin >> m_activation_type;
-  cout << "Activation type " << m_activation_type << endl;
+  //cout << "Activation type " << m_activation_type << endl;
 }
 
 void keras::LayerMaxPooling::load_weights(std::ifstream &fin) {
   fin >> m_pool_x >> m_pool_y;
-  cout << "MaxPooling " << m_pool_x << "x" << m_pool_y << endl;
+  //cout << "MaxPooling " << m_pool_x << "x" << m_pool_y << endl;
 }
 
 void keras::LayerDense::load_weights(std::ifstream &fin) {
@@ -100,18 +100,19 @@ void keras::LayerDense::load_weights(std::ifstream &fin) {
     fin >> tmp_char; // for ']'
     m_weights.push_back(tmp_n);
   }
-  cout << "weights " << m_weights.size() << endl;
+  //cout << "weights " << m_weights.size() << endl;
   fin >> tmp_char; // for '['
   for(int n = 0; n < m_neurons; ++n) {
     fin >> tmp_float;
     m_bias.push_back(tmp_float);
   }
   fin >> tmp_char; // for ']'
-  cout << "bias " << m_bias.size() << endl;
+  //cout << "bias " << m_bias.size() << endl;
 
 }
 
-keras::KerasModel::KerasModel(const string &input_fname) {
+keras::KerasModel::KerasModel(const string &input_fname, bool verbose)
+                                                       : m_verbose(verbose) {
   load_weights(input_fname);
 }
 
@@ -361,28 +362,25 @@ std::vector<float> keras::KerasModel::compute_output(keras::DataChunk *dc) {
     inp = out;
   }
 
-  cout << "Output: ";
-  out->show_values();
-
   std::vector<float> flat_out = out->get_1d();
   delete out;
 
   return flat_out;
 }
 
 void keras::KerasModel::load_weights(const string &input_fname) {
-  cout << "Reading model from " << input_fname << endl;
+  if(m_verbose) cout << "Reading model from " << input_fname << endl;
   ifstream fin(input_fname.c_str());
   string layer_type = "";
   string tmp_str = "";
   int tmp_int = 0;
 
   fin >> tmp_str >> m_layers_cnt;
-  cout << "Layers " << m_layers_cnt << endl;
+  if(m_verbose) cout << "Layers " << m_layers_cnt << endl;
 
   for(int layer = 0; layer < m_layers_cnt; ++layer) { // iterate over layers
     fin >> tmp_str >> tmp_int >> layer_type;
-    cout << "Layer " << tmp_int << " " << layer_type << endl;
+    if(m_verbose) cout << "Layer " << tmp_int << " " << layer_type << endl;
 
     Layer *l = 0L;
     if(layer_type == "Convolution2D") {

keras_model.h

@@ -191,7 +191,7 @@ class keras::LayerDense : public Layer {
 
 class keras::KerasModel {
 public:
-  KerasModel(const std::string &input_fname);
+  KerasModel(const std::string &input_fname, bool verbose);
   ~KerasModel();
   std::vector<float> compute_output(keras::DataChunk *dc);
 
@@ -204,6 +204,7 @@ class keras::KerasModel {
   void load_weights(const std::string &input_fname);
   int m_layers_cnt; // number of layers
   std::vector<Layer *> m_layers; // container with layers
+  bool m_verbose;
 
 };
 

test_compare.py

@@ -0,0 +1,40 @@
+import numpy as np
+np.random.seed(1336)
+import json
+import argparse
+import sys
+
+def get_numbers_from_file(fname):
+    r = []
+    with open(fname) as fin:
+        for line in fin:
+            tmp = line.split()
+            r = [float(t) for t in tmp]
+    return np.array(r)
+
+np.set_printoptions(threshold=np.inf)
+parser = argparse.ArgumentParser(description='This is a simple script compare predictions from keras and keras2cpp.')
+
+parser.add_argument('-k', '--keras_response', help="Response from Keras (test_run_cnn.py)", required=True)
+parser.add_argument('-c', '--keras2cpp_response', help="Response from Keras2cpp (test_run_cnn.cc)", required=True)
+args = parser.parse_args()
+
+
+keras_output     = get_numbers_from_file(args.keras_response)
+keras2cpp_output = get_numbers_from_file(args.keras2cpp_response)
+
+if len(keras_output) != len(keras2cpp_output):
+    print "Different output dimensions"
+    sys.exit(1)
+
+sub = np.sum(np.abs(keras_output - keras2cpp_output))
+
+if sub < 1e-6:
+    print 'Test: [DONE]'
+    print 'Dump is working correctly.'
+    sys.exit(0)
+else:
+    print 'Test: [ERROR]'
+    print 'The output from Keras and Keras2cpp are different.'
+    print 'Difference value:', sub
+    sys.exit(1)

test_run.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+echo 'Test for CNN dumping'
+
+# Parameters
+INPUT_ARCH="../../keras2cpp_nets/th/52c55ce6-201b-4ce4-ad53-68b86d79804f.cnn.architecture.json"
+INPUT_WEIGHTS="../../keras2cpp_nets/th/52c55ce6-201b-4ce4-ad53-68b86d79804f.cnn.model.h5"
+
+DUMPED_CNN="test_cnn.dumped"
+DATA_SAMPLE="test_random_input.dat"
+KERAS_OUTPUT="test_keras_output.dat"
+KERAS2CPP_OUTPUT="test_keras2cpp_output.dat"
+TEST_BIN="test_bin"
+
+echo 'Test, step 1'
+echo 'Dump network into plain text file' $DUMPED_CNN
+python dump_to_simple_cpp.py -a $INPUT_ARCH -w $INPUT_WEIGHTS -o $DUMPED_CNN
+
+echo 'Test, step 2'
+echo 'Generate random input sample and save in' $DATA_SAMPLE
+echo 'Compute ouput on generated sample with Keras and store predictions for comparison'
+python test_run_cnn.py -a $INPUT_ARCH -w $INPUT_WEIGHTS -d $DATA_SAMPLE -o $KERAS_OUTPUT
+
+echo 'Test, step 3'
+echo 'Compile keras2cpp code'
+g++ -std=c++11 test_run_cnn.cc keras_model.cc -o $TEST_BIN
+echo 'Run predictions with dumped network and random data sample from step 2'
+./$TEST_BIN $DUMPED_CNN $DATA_SAMPLE $KERAS2CPP_OUTPUT
+
+echo 'Test, step 4'
+echo 'Compare Keras and Keras2cpp outputs'
+python test_compare.py --keras_response $KERAS_OUTPUT --keras2cpp_response $KERAS2CPP_OUTPUT
+
+# Clean
+echo 'Cleaning after test'
+rm $DUMPED_CNN
+rm $DATA_SAMPLE
+rm $KERAS_OUTPUT
+rm $KERAS2CPP_OUTPUT
+rm $TEST_BIN
+# used only if you log hidden layers output in test_run_cnn.py file
+#rm test_layer_*.output

test_run_cnn.cc

@@ -0,0 +1,41 @@
+#include "keras_model.h"
+
+#include <iostream>
+
+using namespace std;
+using namespace keras;
+
+
+
+int main(int argc, char *argv[]) {
+
+  if(argc != 4) {
+    cout << "Wrong input, going to exit." << endl;
+    cout << "There should be arguments: dumped_cnn_file input_sample output_file." << endl;
+    return -1;
+  }
+  string dumped_cnn = argv[1];
+  string input_data = argv[2];
+  string response_file = argv[3];
+
+  cout << "Testing network from " << dumped_cnn << " on data from " << input_data << endl;
+
+  // Input data sample
+  DataChunk *sample = new DataChunk2D();
+  sample->read_from_file(input_data);
+
+  // Construct network
+  KerasModel m(dumped_cnn, false);
+  std::vector<float> response = m.compute_output(sample);
+
+  // clean sample
+  delete sample;
+
+  // save response into file
+  ofstream fout(response_file);
+  for(unsigned int i = 0; i < response.size(); i++) {
+      fout << response[i] << " ";
+  }
+  fout.close();
+  return 0;
+}

test_run_cnn.py

@@ -0,0 +1,82 @@
+import numpy as np
+np.random.seed(1336)
+from keras.models import Sequential, model_from_json
+import json
+import argparse
+from keras import backend as K
+
+np.set_printoptions(threshold=np.inf)
+parser = argparse.ArgumentParser(description='This is a simple script to run Keras model from saved architecture and weights.\
+                                              This script also creates a input data sample for c++ run_net.')
+
+parser.add_argument('-a', '--architecture', help="JSON with model architecture", required=True)
+parser.add_argument('-w', '--weights', help="Model weights in HDF5 format", required=True)
+parser.add_argument('-d', '--data_sample', help="File where to write random data sample", required=True)
+parser.add_argument('-o', '--output', help="File where to write network outpu", required=True)
+parser.add_argument('-v', '--verbose', help="Verbose", required=False)
+args = parser.parse_args()
+
+print 'Verbose', args.verbose
+print 'Read architecture from', args.architecture
+print 'Read weights from', args.weights
+
+arch = open(args.architecture).read()
+model = model_from_json(arch)
+model.load_weights(args.weights)
+model.compile(loss='categorical_crossentropy', optimizer='adadelta')
+arch = json.loads(arch)
+print 'There are', str(len(model.layers)), 'layers in your network, is it good?'
+print 'I think yes :)'
+
+
+first_layer = arch["config"][0]["config"]
+input_shape = first_layer["batch_input_shape"]
+print "Input shape of your network", input_shape
+
+print "Generate random input for testing purposes"
+random_input = np.random.rand(1, input_shape[1], input_shape[2], input_shape[3])
+print "Random input shape", random_input.shape
+response = model.predict(random_input)[0]
+if args.verbose:
+    print '-'*50
+    print 'Prediction from Keras'
+    print response
+    print '-'*50
+# save response to the file
+with open(args.output, "w") as fin:
+    fin.write(' '.join([str(r) for r in response]))
+# store one sample in text file
+# this code is working for input_shape[1] == 1
+if input_shape[1] != 1:
+    print '-'*50
+    print 'Sorry but below code can be a buggy for image depth > 1 !!!'
+    print '-'*50
+
+# save random data sample into file
+with open(args.data_sample, "w") as fin:
+    print "Save to", args.data_sample, "sample shape", str(input_shape[1]) + " " + str(input_shape[2]) + " " + str(input_shape[3])
+    fin.write(str(input_shape[1]) + " " + str(input_shape[2]) + " " + str(input_shape[3]) + "\n")
+    a = random_input[0,0]
+    for b in a:
+        fin.write(str(b)+'\n')
+
+# Get layers output (for debuging)
+'''
+for l in xrange(len(model.layers)):
+    with open('test_layer_' + str(l) + '.output', 'w') as fout:
+
+        get_layer_output = K.function([model.layers[0].input, K.learning_phase()],
+                                      [model.layers[l].output])
+        layer_output = get_layer_output([random_input, 0])
+
+        print 'Layer', l, layer_output[0].shape
+        if l > 10:
+            print layer_output
+        fout.write(str(layer_output[0].shape) + '\n')
+        fout.write(str(layer_output) + '\n')
+'''
+#print 'input?'
+#get_layer_output = K.function([model.layers[0].input, K.learning_phase()],
+#                              [model.layers[0].input])
+#layer_output = get_layer_output([random_input, 0])
+#print layer_output