Skip to content

Commit 8ee58b7

Browse files
shohad25shohad25
committed
Add all CRF+NN results and scripts
Ohad
1 parent c5c98ad commit 8ee58b7

9 files changed

+845
-0
lines changed

‎crf_with_nn_-2.py

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
"""
2+
===============================
3+
OCR Letter sequence recognition
4+
===============================
5+
This example illustrates the use of a chain CRF for optical character
6+
recognition. The example is taken from Taskar et al "Max-margin markov random
7+
fields".
8+
9+
Each example consists of a handwritten word, that was presegmented into
10+
characters. Each character is represented as a 16x8 binary image. The task is
11+
to classify the image into one of the 26 characters a-z. The first letter of
12+
every word was ommited as it was capitalized and the task does only consider
13+
small caps letters.
14+
15+
We compare classification using a standard linear SVM that classifies
16+
each letter individually with a chain CRF that can exploit correlations
17+
between neighboring letters (the correlation is particularly strong
18+
as the same words are used during training and testing).
19+
20+
The first figures shows the segmented letters of four words from the test set.
21+
In set are the ground truth (green), the prediction using SVM (blue) and the
22+
prediction using a chain CRF (red).
23+
24+
The second figure shows the pairwise potentials learned by the chain CRF.
25+
The strongest patterns are "y after l" and "n after i".
26+
27+
There are obvious extensions that both methods could benefit from, such as
28+
window features or non-linear kernels. This example is more meant to give a
29+
demonstration of the CRF than to show its superiority.
30+
"""
31+
import numpy as np
32+
import matplotlib.pyplot as plt
33+
import os
34+
from sklearn.svm import LinearSVC
35+
from sklearn.svm import SVC
36+
from common.viewers.imshow import imshow
37+
from pystruct.datasets import load_letters
38+
from pystruct.models import ChainCRF, GraphCRF
39+
from pystruct.learners import FrankWolfeSSVM
40+
from sklearn.linear_model import LinearRegression
41+
from common.utils import get_letters_in_pred_like, arrange_letters_in_pred_like
42+
import cPickle
43+
abc = "abcdefghijklmnopqrstuvwxyz"
44+
45+
# Load data:
46+
letters = load_letters()
47+
X, y, folds = letters['data'], letters['labels'], letters['folds']
48+
49+
# we convert the lists to object arrays, as that makes slicing much more
50+
# convenient
51+
X, y = np.array(X), np.array(y)
52+
X_train, X_test = X[folds == 1], X[folds != 1]
53+
y_train, y_test = y[folds == 1], y[folds != 1]
54+
55+
56+
net_base_path = '/media/ohadsh/sheard/googleDrive/Master/courses/probabilistic_graphical_models/outputs/part_3/training_2016_06_11/'
57+
# Load pre-trained network
58+
train_name = 'train_pred_-2.pkl'
59+
test_name = 'test_pred_-2.pkl'
60+
with open(os.path.join(net_base_path, train_name), 'r') as f:
61+
train_net_pred = cPickle.load(f)
62+
with open(os.path.join(net_base_path, test_name), 'r') as f:
63+
test_net_pred = cPickle.load(f)
64+
65+
# Rearrange data for CRF
66+
nn_predictions_train = arrange_letters_in_pred_like(X_train, train_net_pred, size_of_pred=26)
67+
nn_predictions_test = arrange_letters_in_pred_like(X_test, test_net_pred, size_of_pred=26)
68+
69+
# Train LCCRF
70+
chain_model = ChainCRF(directed=True)
71+
chain_ssvm = FrankWolfeSSVM(model=chain_model, C=.1, max_iter=11)
72+
chain_ssvm.fit(X_train, y_train)
73+
74+
# Train LCCRF+NN
75+
chain_model = ChainCRF(directed=True)
76+
chain_ssvm_nn = FrankWolfeSSVM(model=chain_model, C=.1, max_iter=11)
77+
chain_ssvm_nn.fit(nn_predictions_train, y_train)
78+
79+
print("Test score with linear NN: 84.15%")
80+
81+
print("Test score with LCCRF: %f" % chain_ssvm.score(X_test, y_test))
82+
83+
print("Test score with LCCRF+NN: %f" % chain_ssvm_nn.score(nn_predictions_test, y_test))
84+
85+
# plot some word sequenced
86+
n_words = 4
87+
rnd = np.random.RandomState(1)
88+
selected = rnd.randint(len(y_test), size=n_words)
89+
max_word_len = max([len(y_) for y_ in y_test[selected]])
90+
fig, axes = plt.subplots(n_words, max_word_len, figsize=(10, 10))
91+
fig.subplots_adjust(wspace=0)
92+
fig.text(0.2, 0.05, 'GT', color="#00AA00", size=25)
93+
fig.text(0.4, 0.05, 'NN', color="#5555FF", size=25)
94+
fig.text(0.6, 0.05, 'LCCRF', color="#FF5555", size=25)
95+
fig.text(0.8, 0.05, 'LCCRF+NN', color="#FFD700", size=25)
96+
97+
fig.text(0.05, 0.5, 'Word', color="#000000", size=25)
98+
fig.text(0.5, 0.95, 'Letters', color="#000000", size=25)
99+
100+
with open(os.path.join(net_base_path, 'test_pred_-1_normed.pkl'), 'r') as f:
101+
test_net_pred_last = cPickle.load(f)
102+
test_net_pred_last = arrange_letters_in_pred_like(X_test, test_net_pred_last, size_of_pred=26)
103+
104+
for ind, axes_row in zip(selected, axes):
105+
y_pred_nn = test_net_pred_last[ind].argmax(axis=1)
106+
y_pred_chain = chain_ssvm.predict([X_test[ind]])[0]
107+
y_pred_chain_nn = chain_ssvm_nn.predict([nn_predictions_test[ind]])[0]
108+
109+
for i, (a, image, y_true, y_nn, y_chain, y_chain_nn) in enumerate(
110+
zip(axes_row, X_test[ind], y_test[ind], y_pred_nn, y_pred_chain, y_pred_chain_nn)):
111+
a.matshow(image.reshape(16, 8), cmap=plt.cm.Greys)
112+
a.text(0, 3, abc[y_true], color="#00AA00", size=25) # Green
113+
a.text(0, 14, abc[y_nn], color="#5555FF", size=25) # Blue
114+
a.text(5, 14, abc[y_chain], color="#FF5555", size=25) # Red
115+
a.text(5, 3, abc[y_chain_nn], color="#FFD700", size=25) # Yellow
116+
a.set_xticks(())
117+
a.set_yticks(())
118+
for ii in range(i + 1, max_word_len):
119+
axes_row[ii].set_visible(False)
120+
121+
w = chain_ssvm_nn.w[26 * 128:].reshape(26, 26)
122+
w_prob = np.exp(w) / sum(np.exp(w))
123+
124+
fig, ax = plt.subplots(nrows=1, ncols=2)
125+
ax[0].set_title('Transition parameters of LCCRF+NN.', fontsize=30)
126+
127+
plt.sca(ax[0])
128+
plt.xticks(np.arange(26), abc, fontsize=20)
129+
plt.yticks(np.arange(26), abc, fontsize=20)
130+
imshow(w, ax=ax[0], fig=fig, colormap='rainbow')
131+
132+
ax[1].set_title('Transition Probability of LCCRF+NN.', fontsize=30)
133+
plt.sca(ax[1])
134+
plt.xticks(np.arange(26), abc, fontsize=20)
135+
plt.yticks(np.arange(26), abc, fontsize=20)
136+
imshow(w_prob, ax=ax[1], fig=fig, colormap='rainbow', block=True)
137+
138+

‎crf_with_nn_-3.py

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
"""
2+
===============================
3+
OCR Letter sequence recognition
4+
===============================
5+
This example illustrates the use of a chain CRF for optical character
6+
recognition. The example is taken from Taskar et al "Max-margin markov random
7+
fields".
8+
9+
Each example consists of a handwritten word, that was presegmented into
10+
characters. Each character is represented as a 16x8 binary image. The task is
11+
to classify the image into one of the 26 characters a-z. The first letter of
12+
every word was ommited as it was capitalized and the task does only consider
13+
small caps letters.
14+
15+
We compare classification using a standard linear SVM that classifies
16+
each letter individually with a chain CRF that can exploit correlations
17+
between neighboring letters (the correlation is particularly strong
18+
as the same words are used during training and testing).
19+
20+
The first figures shows the segmented letters of four words from the test set.
21+
In set are the ground truth (green), the prediction using SVM (blue) and the
22+
prediction using a chain CRF (red).
23+
24+
The second figure shows the pairwise potentials learned by the chain CRF.
25+
The strongest patterns are "y after l" and "n after i".
26+
27+
There are obvious extensions that both methods could benefit from, such as
28+
window features or non-linear kernels. This example is more meant to give a
29+
demonstration of the CRF than to show its superiority.
30+
"""
31+
import numpy as np
32+
import matplotlib.pyplot as plt
33+
import os
34+
from sklearn.svm import LinearSVC
35+
from sklearn.svm import SVC
36+
from common.viewers.imshow import imshow
37+
from pystruct.datasets import load_letters
38+
from pystruct.models import ChainCRF, GraphCRF
39+
from pystruct.learners import FrankWolfeSSVM
40+
from sklearn.linear_model import LinearRegression
41+
from common.utils import get_letters_in_pred_like, arrange_letters_in_pred_like
42+
import cPickle
43+
abc = "abcdefghijklmnopqrstuvwxyz"
44+
45+
# Load data:
46+
letters = load_letters()
47+
X, y, folds = letters['data'], letters['labels'], letters['folds']
48+
49+
# we convert the lists to object arrays, as that makes slicing much more
50+
# convenient
51+
X, y = np.array(X), np.array(y)
52+
X_train, X_test = X[folds == 1], X[folds != 1]
53+
y_train, y_test = y[folds == 1], y[folds != 1]
54+
55+
56+
net_base_path = '/media/ohadsh/sheard/googleDrive/Master/courses/probabilistic_graphical_models/outputs/part_3/training_2016_06_11/'
57+
# Load pre-trained network
58+
train_name = 'train_pred_-3.pkl'
59+
test_name = 'test_pred_-3.pkl'
60+
with open(os.path.join(net_base_path, train_name), 'r') as f:
61+
train_net_pred = cPickle.load(f)
62+
with open(os.path.join(net_base_path, test_name), 'r') as f:
63+
test_net_pred = cPickle.load(f)
64+
65+
# Rearrange data for CRF
66+
nn_predictions_train = arrange_letters_in_pred_like(X_train, train_net_pred, size_of_pred=26)
67+
nn_predictions_test = arrange_letters_in_pred_like(X_test, test_net_pred, size_of_pred=26)
68+
69+
# Train LCCRF
70+
chain_model = ChainCRF(directed=True)
71+
chain_ssvm = FrankWolfeSSVM(model=chain_model, C=.1, max_iter=11)
72+
chain_ssvm.fit(X_train, y_train)
73+
74+
# Train LCCRF+NN
75+
chain_model = ChainCRF(directed=True)
76+
chain_ssvm_nn = FrankWolfeSSVM(model=chain_model, C=.1, max_iter=11)
77+
chain_ssvm_nn.fit(nn_predictions_train, y_train)
78+
79+
print("Test score with linear NN: 84.15%")
80+
81+
print("Test score with LCCRF: %f" % chain_ssvm.score(X_test, y_test))
82+
83+
print("Test score with LCCRF+NN: %f" % chain_ssvm_nn.score(nn_predictions_test, y_test))
84+
85+
# plot some word sequenced
86+
n_words = 4
87+
rnd = np.random.RandomState(1)
88+
selected = rnd.randint(len(y_test), size=n_words)
89+
max_word_len = max([len(y_) for y_ in y_test[selected]])
90+
fig, axes = plt.subplots(n_words, max_word_len, figsize=(10, 10))
91+
fig.subplots_adjust(wspace=0)
92+
fig.text(0.2, 0.05, 'GT', color="#00AA00", size=25)
93+
fig.text(0.4, 0.05, 'NN', color="#5555FF", size=25)
94+
fig.text(0.6, 0.05, 'LCCRF', color="#FF5555", size=25)
95+
fig.text(0.8, 0.05, 'LCCRF+NN', color="#FFD700", size=25)
96+
97+
fig.text(0.05, 0.5, 'Word', color="#000000", size=25)
98+
fig.text(0.5, 0.95, 'Letters', color="#000000", size=25)
99+
100+
with open(os.path.join(net_base_path, 'test_pred_-1_normed.pkl'), 'r') as f:
101+
test_net_pred_last = cPickle.load(f)
102+
test_net_pred_last = arrange_letters_in_pred_like(X_test, test_net_pred_last, size_of_pred=26)
103+
104+
for ind, axes_row in zip(selected, axes):
105+
y_pred_nn = test_net_pred_last[ind].argmax(axis=1)
106+
y_pred_chain = chain_ssvm.predict([X_test[ind]])[0]
107+
y_pred_chain_nn = chain_ssvm_nn.predict([nn_predictions_test[ind]])[0]
108+
109+
for i, (a, image, y_true, y_nn, y_chain, y_chain_nn) in enumerate(
110+
zip(axes_row, X_test[ind], y_test[ind], y_pred_nn, y_pred_chain, y_pred_chain_nn)):
111+
a.matshow(image.reshape(16, 8), cmap=plt.cm.Greys)
112+
a.text(0, 3, abc[y_true], color="#00AA00", size=25) # Green
113+
a.text(0, 14, abc[y_nn], color="#5555FF", size=25) # Blue
114+
a.text(5, 14, abc[y_chain], color="#FF5555", size=25) # Red
115+
a.text(5, 3, abc[y_chain_nn], color="#FFD700", size=25) # Yellow
116+
a.set_xticks(())
117+
a.set_yticks(())
118+
for ii in range(i + 1, max_word_len):
119+
axes_row[ii].set_visible(False)
120+
121+
w = chain_ssvm_nn.w[26 * 128:].reshape(26, 26)
122+
w_prob = np.exp(w) / sum(np.exp(w))
123+
124+
fig, ax = plt.subplots(nrows=1, ncols=2)
125+
ax[0].set_title('Transition parameters of LCCRF+NN.', fontsize=30)
126+
127+
plt.sca(ax[0])
128+
plt.xticks(np.arange(26), abc, fontsize=20)
129+
plt.yticks(np.arange(26), abc, fontsize=20)
130+
imshow(w, ax=ax[0], fig=fig, colormap='rainbow')
131+
132+
ax[1].set_title('Transition Probability of LCCRF+NN.', fontsize=30)
133+
plt.sca(ax[1])
134+
plt.xticks(np.arange(26), abc, fontsize=20)
135+
plt.yticks(np.arange(26), abc, fontsize=20)
136+
imshow(w_prob, ax=ax[1], fig=fig, colormap='rainbow', block=True)
137+
138+

0 commit comments

Comments
 (0)