Skip to content

Commit 952bf69

Browse files
WafaaTDina Suehiro Jones
authored and
Dina Suehiro Jones
committed
Add: FastRCNN (ResNet50) FP32 inference benchmark scripts (#20)
* add fastrcnn resnet50 inference benchmark script with no checkpoint file attached. * changes in README, fixing style in model_init for code review comments. * minor changes * minor changes and NCF model init update. * minor changes in README * fix style * fix base_benchmark_util. * update squeezenet model_init
1 parent 509dc6d commit 952bf69

File tree

22 files changed

+1659
-45
lines changed

22 files changed

+1659
-45
lines changed

__init__.py

Whitespace-only changes.

benchmarks/README.md

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ Training and inference scripts with intel optimized MKL
1414
* [FP32](image_recognition/tensorflow/squeezenet/README.md#fp32-inference-instructions)
1515
* Object Detection
1616
* Tensorflow
17+
* Fast R-CNN (ResNet50)
18+
* Inference
19+
* [FP32](object_detection/tensorflow/fastrcnn/README.md#fp32-inference-instructions)
1720
* SSD-Mobilenet
1821
* Inference
1922
* [FP32](object_detection/tensorflow/ssd-mobilenet/README.md#fp32-inference-instructions)

benchmarks/common/base_benchmark_util.py

+13-35
Original file line numberDiff line numberDiff line change
@@ -175,44 +175,22 @@ def initialize_model(self, args, unknown_args):
175175
model_initializer = None
176176
model_init_file = None
177177
if args.model_name: # not empty
178-
current_path = os.path.dirname(os.path.dirname(
179-
os.path.realpath(__file__)))
178+
current_path = os.path.dirname(
179+
os.path.dirname(os.path.realpath(__file__)))
180180

181181
# find the path to the model_init.py file
182182
filename = "{}.py".format(self.MODEL_INITIALIZER)
183-
print("current path: {}".format(current_path))
184-
search_path = os.path.join(current_path, "*", args.framework,
185-
args.model_name, args.mode,
186-
args.platform, filename)
187-
print("search path: {}".format(search_path))
188-
matches = glob.glob(search_path)
189-
190-
if len(matches) > 1:
191-
# we should never get more than one match
192-
raise ValueError("Found multiple model_init.py files for "
193-
"{} {} {} {}".format(
194-
args.framework, args.model_name,
195-
args.platform, args.mode))
196-
elif len(matches) == 0:
197-
raise ValueError("No model_init.py was found for {} {} {} "
198-
"{}".format(args.framework, args.model_name,
199-
args.platform, args.mode))
200-
201-
model_init_file = matches[0]
202-
203-
print ("Using model init: {}".format(model_init_file))
204-
if os.path.exists(model_init_file):
205-
dir_list = model_init_file.split("/")
206-
framework_index = dir_list.index(args.framework)
207-
usecase = dir_list[framework_index - 1]
208-
209-
package = ".".join([usecase, args.framework, args.model_name,
210-
args.mode, args.platform])
211-
model_init_module = __import__(package + "." +
212-
self.MODEL_INITIALIZER,
213-
fromlist=['*'])
214-
model_initializer = model_init_module.ModelInitializer(
215-
args, unknown_args, self._platform_util)
183+
model_init_file = os.path.join(current_path, args.use_case,
184+
args.framework, args.model_name,
185+
args.mode, args.platform,
186+
filename)
187+
package = ".".join([args.use_case, args.framework,
188+
args.model_name, args.mode, args.platform])
189+
model_init_module = __import__(package + "." +
190+
self.MODEL_INITIALIZER,
191+
fromlist=['*'])
192+
model_initializer = model_init_module.ModelInitializer(
193+
args, unknown_args, self._platform_util)
216194

217195
if model_initializer is None:
218196
raise ImportError("Unable to locate {}.".format(model_init_file))

benchmarks/common/tensorflow/run_tf_benchmark.py

+5
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ def main(self):
4242
nargs='?',
4343
dest="intelai_models")
4444

45+
arg_parser.add_argument("--use-case",
46+
help="The corresponding use case of the given model ",
47+
nargs='?',
48+
dest="use_case")
49+
4550
# checkpoint directory location
4651
arg_parser.add_argument('-c', "--checkpoint",
4752
help='Specify the location of trained model '

benchmarks/common/tensorflow/start.sh

+39-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#
2121

2222
echo 'Running with parameters:'
23+
echo " USE_CASE: ${USE_CASE}"
2324
echo " FRAMEWORK: ${FRAMEWORK}"
2425
echo " WORKSPACE: ${WORKSPACE}"
2526
echo " DATASET_LOCATION: ${DATASET_LOCATION}"
@@ -88,6 +89,7 @@ function run_model() {
8889
# basic run command with commonly used args
8990
CMD="python ${RUN_SCRIPT_PATH} \
9091
--framework=${FRAMEWORK} \
92+
--use-case=${USE_CASE} \
9193
--model-name=${MODEL_NAME} \
9294
--platform=${PLATFORM} \
9395
--mode=${MODE} \
@@ -114,6 +116,40 @@ function install_protoc() {
114116

115117
}
116118

119+
# Fast R-CNN (ResNet50) model
120+
function fastrcnn() {
121+
if [ ${MODE} == "inference" ] && [ ${PLATFORM} == "fp32" ]; then
122+
if [[ -z "${config_file}" ]]; then
123+
echo "Fast R-CNN requires -- config_file arg to be defined"
124+
exit 1
125+
fi
126+
# install dependencies
127+
pip install -r "${MOUNT_BENCHMARK}/object_detection/tensorflow/fastrcnn/requirements.txt"
128+
original_dir=$(pwd)
129+
cd "${MOUNT_EXTERNAL_MODELS_SOURCE}/research"
130+
# install protoc v3.3.0, if necessary, then compile protoc files
131+
install_protoc "https://github.com/google/protobuf/releases/download/v3.3.0/protoc-3.3.0-linux-x86_64.zip"
132+
echo "Compiling protoc files"
133+
./bin/protoc object_detection/protos/*.proto --python_out=.
134+
135+
export PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim
136+
# install cocoapi
137+
cd ${MOUNT_EXTERNAL_MODELS_SOURCE}/cocoapi/PythonAPI
138+
echo "Installing COCO API"
139+
make
140+
cp -r pycocotools ${MOUNT_EXTERNAL_MODELS_SOURCE}/research/
141+
export PYTHONPATH=${PYTHONPATH}:${MOUNT_EXTERNAL_MODELS_SOURCE}
142+
143+
cd $original_dir
144+
CMD="${CMD} --checkpoint=${CHECKPOINT_DIRECTORY} \
145+
--config_file=${config_file}"
146+
147+
PYTHONPATH=${PYTHONPATH} CMD=${CMD} run_model
148+
else
149+
echo "MODE:${MODE} and PLATFORM=${PLATFORM} not supported"
150+
fi
151+
}
152+
117153
# NCF model
118154
function ncf() {
119155
# For nfc, if dataset location is empty, script downloads dataset at given location.
@@ -232,7 +268,9 @@ LOGFILE=${LOG_OUTPUT}/benchmark_${MODEL_NAME}_${MODE}_${PLATFORM}.log
232268
echo 'Log output location: ${LOGFILE}'
233269

234270
MODEL_NAME=$(echo ${MODEL_NAME} | tr 'A-Z' 'a-z')
235-
if [ ${MODEL_NAME} == "ncf" ]; then
271+
if [ ${MODEL_NAME} == "fastrcnn" ]; then
272+
fastrcnn
273+
elif [ ${MODEL_NAME} == "ncf" ]; then
236274
ncf
237275
elif [ ${MODEL_NAME} == "resnet50" ]; then
238276
resnet50

benchmarks/image_recognition/tensorflow/squeezenet/inference/fp32/model_init.py

-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ def __init__(self, args, custom_args, platform_util):
2323
ncores = self.args.num_intra_threads
2424

2525
script_path = os.path.join(self.args.intelai_models,
26-
"image_recognition", args.framework,
27-
args.model_name, args.platform,
2826
"train_squeezenet.py")
2927

3028
self.command = ("taskset -c {:.0f}-{:.0f} python {} "

benchmarks/launch_benchmark.py

+37-4
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,38 @@ def run_docker_container(self, args):
124124
variables to start running the benchmarking job.
125125
"""
126126
benchmark_scripts = os.getcwd()
127-
intelai_models = os.path.join(benchmark_scripts, os.pardir, "models")
127+
if args.model_name: # not empty
128+
# find the custom model path
129+
print("current path: {}".format(benchmark_scripts))
130+
search_path = os.path.join(
131+
benchmark_scripts, os.pardir, "models", "*",
132+
args.framework, args.model_name, args.mode, args.platform)
133+
print("search path: {}".format(search_path))
134+
matches = glob.glob(search_path)
135+
if len(matches) > 1:
136+
# we should never get more than one match
137+
raise ValueError("Found multiple model locations for {} {} {}"
138+
.format(args.framework,
139+
args.model_name,
140+
args.platform))
141+
elif len(matches) == 0:
142+
raise ValueError("No model was found for {} {} {}"
143+
.format(args.framework,
144+
args.model_name,
145+
args.platform))
146+
147+
intelai_models = matches[0]
148+
print ("Using the custom model in : {}".format(intelai_models))
149+
if os.path.exists(intelai_models):
150+
dir_list = intelai_models.split("/")
151+
framework_index = dir_list.index(args.framework)
152+
use_case = str(dir_list[framework_index - 1])
153+
else:
154+
use_case = None
155+
else:
156+
use_case = None
157+
intelai_models = os.path.join(benchmark_scripts, os.pardir, "models")
158+
128159
mount_benchmark = "/workspace/benchmarks"
129160
mount_external_models_source = "/workspace/models"
130161
mount_intelai_models = "/workspace/intelai_models"
@@ -151,6 +182,7 @@ def run_docker_container(self, args):
151182
"--env MOUNT_BENCHMARK={} "
152183
"--env MOUNT_EXTERNAL_MODELS_SOURCE={} "
153184
"--env MOUNT_INTELAI_MODELS_SOURCE={} "
185+
"--env USE_CASE={} "
154186
"--env FRAMEWORK={} "
155187
"--env NUM_CORES={} "
156188
"--env DATASET_LOCATION=/dataset "
@@ -163,9 +195,10 @@ def run_docker_container(self, args):
163195
args.model_name, args.mode, args.platform,
164196
args.verbose, args.batch_size, workspace,
165197
in_graph_filename, mount_benchmark,
166-
mount_external_models_source, mount_intelai_models,
167-
args.framework, args.num_cores,
168-
args.benchmark_only, args.accuracy_only))
198+
mount_external_models_source,
199+
mount_intelai_models, use_case,
200+
args.framework, args.num_cores, args.benchmark_only,
201+
args.accuracy_only))
169202

170203
# Add custom model args as env vars
171204
for custom_arg in args.model_args:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
# Fast R-CNN (ResNet50)
2+
3+
This document has instructions for how to run FastRCNN for the
4+
following modes/platforms:
5+
* [FP32 inference](#fp32-inference-instructions)
6+
7+
Benchmarking instructions and scripts for the Fast R-CNN ResNet50 model training and inference
8+
other platforms are coming later.
9+
10+
## FP32 Inference Instructions
11+
12+
1. Clone the `tensorflow/models` and `cocoapi` repositories:
13+
14+
```
15+
$ git clone [email protected]:tensorflow/models.git
16+
$ cd models
17+
$ git clone https://github.com/cocodataset/cocoapi.git
18+
19+
```
20+
21+
The TensorFlow models repo will be used for running inference as well as
22+
converting the coco dataset to the TF records format.
23+
24+
2. Download the 2017 validation
25+
[COCO dataset](http://cocodataset.org/#home) and annotations:
26+
27+
```
28+
$ mkdir val
29+
$ cd val
30+
$ wget http://images.cocodataset.org/zips/val2017.zip
31+
$ unzip val2017.zip
32+
$ cd ..
33+
34+
$ mkdir annotations
35+
$ cd annotations
36+
$ wget http://images.cocodataset.org/annotations/annotations_trainval2017.zip
37+
$ unzip annotations_trainval2017.zip
38+
$ cd ..
39+
```
40+
41+
Since we are only using the validation dataset in this example, we will
42+
create an empty directory and empty annotations json file to pass as the
43+
train and test directories in the next step.
44+
45+
```
46+
$ mkdir empty_dir
47+
48+
$ cd annotations
49+
$ echo "{ \"images\": {}, \"categories\": {}}" > empty.json
50+
$ cd ..
51+
```
52+
53+
3. Now that you have the raw COCO dataset, we need to convert it to the
54+
TF records format in order to use it with the inference script. We will
55+
do this by running the `create_coco_tf_record.py` file in the TensorFlow
56+
models repo.
57+
58+
Follow the steps below to navigate to the proper directory and point the
59+
script to the raw COCO dataset files that you have downloaded in step 2.
60+
The `--output_dir` is the location where the TF record files will be
61+
located after the script has completed.
62+
63+
```
64+
65+
# We are going to use an older version of the conversion script to checkout the git commit
66+
$ cd models
67+
$ git checkout 7a9934df2afdf95be9405b4e9f1f2480d748dc40
68+
69+
$ cd research/object_detection/dataset_tools/
70+
$ python create_coco_tf_record.py --logtostderr \
71+
--train_image_dir="/home/myuser/coco/empty_dir" \
72+
--val_image_dir="/home/myuser/coco/val/val2017" \
73+
--test_image_dir="/home/myuser/coco/empty_dir" \
74+
--train_annotations_file="/home/myuser/coco/annotations/empty.json" \
75+
--val_annotations_file="/home/myuser/coco/annotations/instances_val2017.json" \
76+
--testdev_annotations_file="/home/myuser/coco/annotations/empty.json" \
77+
--output_dir="/home/myuser/coco/output"
78+
79+
$ ll /home/myuser/coco/output
80+
total 1598276
81+
-rw-rw-r--. 1 myuser myuser 0 Nov 2 21:46 coco_testdev.record
82+
-rw-rw-r--. 1 myuser myuser 0 Nov 2 21:46 coco_train.record
83+
-rw-rw-r--. 1 myuser myuser 818336740 Nov 2 21:46 coco_val.record
84+
85+
# Go back to the main models directory and get master code
86+
$ cd /home/myuser/models
87+
$ git checkout master
88+
```
89+
90+
The `coco_val.record` file is what we will use in this inference example.
91+
92+
4. Download the pre-trained model fast_rcnn_resnet50_fp32_coco_pretrained_model.tar.gz.
93+
The pre-trained model includes the checkpoint files and the Fast R-CNN ResNet50 model `pipeline.config`.
94+
Extract and check out its contents as shown:
95+
```
96+
$ wget https://storage.cloud.google.com/intel-optimized-tensorflow/models/fast_rcnn_resnet50_fp32_coco_pretrained_model.tar.gz
97+
$ tar -xzvf fast_rcnn_resnet50_fp32_coco_pretrained_model.tar.gz
98+
$ ls -l fast_rcnn_resnet50_fp32_coco
99+
total 374848
100+
-rw-r--r-- 1 myuser myuser 77 Nov 12 22:33 checkpoint
101+
-rw-r--r-- 1 myuser myuser 176914228 Nov 12 22:33 model.ckpt.data-00000-of-00001
102+
-rw-r--r-- 1 myuser myuser 14460 Nov 12 22:33 model.ckpt.index
103+
-rw-r--r-- 1 myuser myuser 5675175 Nov 12 22:33 model.ckpt.meta
104+
-rwxr--r-- 1 myuser myuser 5056 Nov 12 22:33 mscoco_label_map.pbtxt
105+
-rwxr-xr-x 1 myuser myuser 3244 Nov 12 22:33 pipeline.config
106+
drwxr-xr-x 4 myuser myuser 128 Nov 12 22:30 saved_model
107+
108+
```
109+
Make sure that the `eval_input_reader` section in the `pipeline.config` file has the mounted
110+
`coco_val.record` data and pre-trained model `mscoco_label_map.pbtxt` location.
111+
112+
5. Clone the [intelai/models](https://github.com/intelai/models) repo.
113+
This repo has the launch script for running benchmarking.
114+
115+
```
116+
$ git clone [email protected]:IntelAI/models.git
117+
Cloning into 'models'...
118+
remote: Enumerating objects: 11, done.
119+
remote: Counting objects: 100% (11/11), done.
120+
remote: Compressing objects: 100% (7/7), done.
121+
remote: Total 11 (delta 3), reused 4 (delta 0), pack-reused 0
122+
Receiving objects: 100% (11/11), done.
123+
Resolving deltas: 100% (3/3), done.
124+
```
125+
126+
6. Run the `launch_benchmark.py` script from the intelai/models repo
127+
, with the appropriate parameters including: the
128+
`coco_val.record` data location (from step 3), the pre-trained model
129+
`pipeline.config` file and the checkpoint location (from step 4, and the
130+
location of your `tensorflow/models` clone (from step 1).
131+
132+
```
133+
$ cd /home/myuser/models/benchmarks
134+
135+
$ python launch_benchmark.py \
136+
--data-location /home/myuser/coco/output/ \
137+
--model-source-dir /home/myuser/tensorflow/models \
138+
--model-name fastrcnn \
139+
--framework tensorflow \
140+
--platform fp32 \
141+
--mode inference \
142+
--checkpoint /home/myuser/fast_rcnn_resnet50_fp32_coco \
143+
--docker-image intelaipg/intel-optimized-tensorflow:latest-devel-mkl \
144+
-- config-file=pipeline.config
145+
```
146+
147+
7. The log file is saved to:
148+
models/benchmarks/common/tensorflow/logs/benchmark_fastrcnn_inference.log
149+
150+
The tail of the log output when the benchmarking completes should look
151+
something like this:
152+
153+
```
154+
Time spent : 172.880 seconds.
155+
Time spent per BATCH: 0.173 seconds.
156+
Received these standard args: Namespace(batch_size=-1, checkpoint='/checkpoints', config='/checkpoints/pipeline.config', data_location=/dataset, inference_only=True, num_cores=-1, num_inter_threads=1, num_intra_threads=28, single_socket=True, socket_id=0, verbose=True)
157+
Received these custom args: []
158+
Initialize here.
159+
Run model here. numactl --cpunodebind=0 --membind=0 python object_detection/eval.py --num_inter_threads 1 --num_intra_threads 28 --pipeline_config_path /checkpoints/pipeline.config --checkpoint_dir /checkpoints --eval_dir /tensorflow-models/research/object_detection/log/eval
160+
```
161+

0 commit comments

Comments
 (0)