Profiling Applications

Profiling with Darshan

With Horovod

export DARSHAN_PRELOAD=/soft/perftools/darshan/darshan-3.1.8/lib/libdarshan.so export DXT_ENABLE_IO_TRACE=1

aprun -n 1 -N 1 -j 2 -cc depth -e DXT_ENABLE_IO_TRACE=1 -e LD_PRELOAD=$DARSHAN_PRELOAD python ./test.py

For Non MPI

export DARSHAN_PRELOAD=/soft/perftools/darshan/darshan-3.2.1-nompi/lib/libdarshan.so
export DXT_ENABLE_IO_TRACE=1
export DARSHAN_ENABLE_NONMPI=1

aprun  -n 1 -N 1 -j 2 -cc depth -e DARSHAN_ENABLE_NONMPI=1 -e DXT_ENABLE_IO_TRACE=1 -e LD_PRELOAD=${DARSHAN_PRELOAD} python ./test.py

Trace with Tensorflow (Requirement 2.2)

# Within App run
from datetime import datetime
import random
tf_orig.compat.v1.summary.FileWriterCache.clear()
run = "./logdir/folder_{}".format(random.randint(0,2000))
tf_orig.profiler.experimental.start(run)
# your application code here
tf_orig.profiler.experimental.stop()