Skip to content

Commit ea0de58

Browse files
aliabdolaliJessicaMeixner-NOAAMatthewMasarik-NOAAthesser1
authored
Add SCOTCH library for geographical domain decomposition (#849)
Co-authored-by: JessicaMeixner-NOAA <[email protected]> Co-authored-by: Matthew Masarik <[email protected]> Co-authored-by: Ty Hesser <[email protected]>
1 parent a4fc630 commit ea0de58

29 files changed

+262
-51
lines changed

‎.github/workflows/gnu.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ concurrency:
77
cancel-in-progress: true
88

99
env:
10-
cache_key: gnu5
10+
cache_key: gnu8
1111
CC: gcc-10
1212
FC: gfortran-10
1313
CXX: g++-10

‎cmake/FindSCOTCH.cmake

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
message(STATUS "Searching for PTSCOTCHparmetis library ...")
2+
find_library(ptscotchparmetis_lib NAMES libptscotchparmetisv3.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
3+
find_path(ptscotchparmetis_inc parmetis.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
4+
5+
message(STATUS "Searching for SCOTCH library ...")
6+
find_library(scotch_lib NAMES libscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
7+
find_path(scotch_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
8+
9+
message(STATUS "Searching for PTSCOTCH library ...")
10+
find_library(ptscotch_lib NAMES libptscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
11+
find_path(ptscotch_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
12+
13+
message(STATUS "Searching for SCOTCHerr library ...")
14+
find_library(scotcherr_lib NAMES libscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
15+
find_path(scotcherr_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
16+
17+
message(STATUS "Searching for PTSCOTCHerr library ...")
18+
find_library(ptscotcherr_lib NAMES libptscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
19+
find_path(ptscotcherr_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
20+
21+
add_library(PTSCOTCHparmetis::PTSCOTCHparmetis STATIC IMPORTED)
22+
add_library(SCOTCH::SCOTCH STATIC IMPORTED)
23+
add_library(PTSCOTCH::PTSCOTCH STATIC IMPORTED)
24+
add_library(SCOTCHerr::SCOTCHerr STATIC IMPORTED)
25+
add_library(PTSCOTCHerr::PTSCOTCHerr STATIC IMPORTED)
26+
27+
set_target_properties(SCOTCH::SCOTCH PROPERTIES
28+
IMPORTED_LOCATION "${scotch_lib}"
29+
INTERFACE_INCLUDE_DIRECTORIES "${scotch_inc}")
30+
31+
set_target_properties(SCOTCHerr::SCOTCHerr PROPERTIES
32+
IMPORTED_LOCATION "${scotcherr_lib}"
33+
INTERFACE_INCLUDE_DIRECTORIES "${scotcherr_inc}")
34+
35+
set_target_properties(PTSCOTCH::PTSCOTCH PROPERTIES
36+
IMPORTED_LOCATION "${ptscotch_lib}"
37+
INTERFACE_INCLUDE_DIRECTORIES "${ptscotch_inc}")
38+
39+
set_target_properties(PTSCOTCHerr::PTSCOTCHerr PROPERTIES
40+
IMPORTED_LOCATION "${ptscotcherr_lib}"
41+
INTERFACE_INCLUDE_DIRECTORIES "${ptscotcherr_inc}")
42+
43+
set_target_properties(PTSCOTCHparmetis::PTSCOTCHparmetis PROPERTIES
44+
IMPORTED_LOCATION "${ptscotchparmetis_lib}"
45+
INTERFACE_INCLUDE_DIRECTORIES "${ptscotchparmetis_inc}")
46+
47+
48+
## Interfaces and links
49+
target_link_libraries(PTSCOTCHparmetis::PTSCOTCHparmetis INTERFACE PTSCOTCH::PTSCOTCH PTSCOTCHerr::PTSCOTCHerr SCOTCH::SCOTCH)
50+
51+
52+
## Finalize find_package
53+
include(FindPackageHandleStandardArgs)
54+
55+
find_package_handle_standard_args(
56+
${CMAKE_FIND_PACKAGE_NAME}
57+
REQUIRED_VARS scotch_lib
58+
scotch_inc)
59+
60+
find_package_handle_standard_args(
61+
${CMAKE_FIND_PACKAGE_NAME}
62+
REQUIRED_VARS ptscotch_lib
63+
ptscotch_inc)
64+
65+
find_package_handle_standard_args(
66+
${CMAKE_FIND_PACKAGE_NAME}
67+
REQUIRED_VARS scotcherr_lib
68+
scotcherr_inc)
69+
70+
find_package_handle_standard_args(
71+
${CMAKE_FIND_PACKAGE_NAME}
72+
REQUIRED_VARS ptscotcherr_lib
73+
ptscotcherr_inc)
74+
75+
find_package_handle_standard_args(
76+
${CMAKE_FIND_PACKAGE_NAME}
77+
REQUIRED_VARS ptscotchparmetis_lib
78+
ptscotchparmetis_inc)
79+
80+
message(STATUS "Found SCOTCH: ${scotch_lib}")
81+
message(STATUS "Found PTSCOTCH: ${ptscotch_lib}")
82+
message(STATUS "Found SCOTCHerr: ${scotcherr_lib}")
83+
message(STATUS "Found PTSCOTCHerr: ${ptscotcherr_lib}")
84+
message(STATUS "Found PTSCOTCHparmetis: ${ptscotchparmetis_lib}")

‎model/bin/build_utils.sh

+32-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ check_switches()
4040
dstress s_ice s_is reflection \
4141
wind windx wcor rwind curr currx mgwind mgprop mggse \
4242
subsec tdyn dss0 pdif tide refrx ig rotag nnt mprf \
43-
cou oasis agcm ogcm igcm trknc setup pdlib memck uost rstwind b4b
43+
cou oasis agcm ogcm igcm trknc setup pdlib ddlib memck uost rstwind b4b
4444
do
4545

4646
# 1.a.1 Group switches by category
@@ -291,6 +291,12 @@ check_switches()
291291
ID='use pdlib'
292292
TS='PDLIB'
293293
OK='PDLIB' ;;
294+
295+
#sort:ddlib:
296+
ddlib ) TY='upto1'
297+
ID='domain decomposition library'
298+
OK='METIS SCOTCH' ;;
299+
294300
#sort:memck:
295301
memck ) TY='upto1'
296302
ID='check memory use'
@@ -428,6 +434,7 @@ check_switches()
428434
igcm ) igcm=$sw ;;
429435
trknc ) trknc=$sw ;;
430436
pdlib ) pdlib=$sw ;;
437+
ddlib ) ddlib=$sw ;;
431438
memck ) memck=$sw ;;
432439
setup ) setup=$sw ;;
433440
uost ) uost=$sw ;;
@@ -547,6 +554,30 @@ check_switches()
547554
echo ' ' ; exit 17
548555
fi
549556

557+
if [ "$pdlib" = 'PDLIB' ] && [ "$ddlib" != 'SCOTCH' ]
558+
then
559+
if [ "$ddlib" != 'METIS' ]
560+
then
561+
echo ' '
562+
echo " *** For PDLIB, we need either SCOTCH or METIS, not both."
563+
echo ' ' ; exit 18
564+
fi
565+
fi
566+
567+
568+
if [ "$ddlib" = 'METIS' ] && [ "$pdlib" != 'PDLIB' ]
569+
then
570+
echo ' '
571+
echo " *** For METIS, we need to have PDLIB as well."
572+
echo ' ' ; exit 19
573+
fi
574+
575+
if [ "$ddlib" = 'SCOTCH' ] && [ "$pdlib" != 'PDLIB' ]
576+
then
577+
echo ' '
578+
echo " *** For SCOTCH, we need to have PDLIB as well."
579+
echo ' ' ; exit 20
580+
fi
550581

551582
} #end of check_switches
552583

‎model/bin/link.tmpl

+15-1
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,23 @@
117117
libs="$libs `$NETCDF_CONFIG --flibs` `$NETCDF_CONFIG --libs`"
118118
fi
119119

120+
# scotch library
121+
if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
122+
if [ "$scotch_mod" = 'yes' ] ; then
123+
if [ -z "$(env | grep SCOTCH_PATH)" ] ; then
124+
echo ''
125+
echo "[ERROR] SCOTCH_PATH is not defined"
126+
exit 1
127+
fi
128+
echo "link with scotch"
129+
libs="$libs $SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a $SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
130+
fi
131+
fi
132+
133+
120134
# parmetis library
121135
if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
122-
if [ "$pdlib_mod" = 'yes' ] ; then
136+
if [ "$metis_mod" = 'yes' ] ; then
123137
if [ -z "$(env | grep METIS_PATH)" ] ; then
124138
echo ''
125139
echo "[ERROR] METIS_PATH is not defined"

‎model/bin/switch_Ifremer2_pdlib

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
NOGRB PDLIB SCRIP SCRIPNC TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 MLIM TR0 BS0 IC2 IS2 REF1 IG1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
1+
NOGRB PDLIB METIS SCRIP SCRIPNC TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 MLIM TR0 BS0 IC2 IS2 REF1 IG1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

‎model/bin/switch_USACE_1

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
NOGRB TRKNC DIST MPI MLIM PR3 UQ FLX1 PDLIB LN1 ST1 STAB0 NL1 BT4 DB0 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
1+
NOGRB TRKNC DIST MPI MLIM PR3 UQ FLX1 PDLIB SCOTCH LN1 ST1 STAB0 NL1 BT4 DB0 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

‎model/bin/switch_USACE_2

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
NOGRB PDLIB TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT1 DB1 MLIM TR0 BS0 WNT1 WNX1 IC0 IS0 REF0 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
1+
NOGRB PDLIB SCOTCH TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT1 DB1 MLIM TR0 BS0 WNT1 WNX1 IC0 IS0 REF0 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

‎model/bin/switch_ite_pdlib

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
NOGRB PDLIB TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IC0 IS0 REF1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
1+
NOGRB PDLIB SCOTCH TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IC0 IS0 REF1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

‎model/bin/switch_ugdev2

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
1+
NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB SCOTCH LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

‎model/bin/w3_automake

+2
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,8 @@
305305
./sort_switch -s -r switch
306306
sed -e 's/DIST/SHRD/g' \
307307
-e 's/PDLIB //g' \
308+
-e 's/METIS //g' \
309+
-e 's/SCOTCH //g' \
308310
-e 's/OMPG //g' \
309311
-e 's/PDLIB //g' \
310312
-e 's/OMPH //g' \

‎model/bin/w3_make

+16-1
Original file line numberDiff line numberDiff line change
@@ -498,13 +498,28 @@ EOF
498498
export oasis_mod
499499

500500
pdlib_mod=no
501+
metis_mod=no
502+
scotch_mod=no
503+
501504
esmfpdlib=''
502505
if [ -n "`grep PDLIB $switch_file`" ]
503506
then
504507
export pdlib_mod='yes'
505-
esmfpdlib="$METIS_PATH/lib/libparmetis.a $METIS_PATH/lib/libmetis.a"
508+
if [ -n "`grep METIS $switch_file`" ]
509+
then
510+
export metis_mod='yes'
511+
esmfpdlib="$METIS_PATH/lib/libparmetis.a $METIS_PATH/lib/libmetis.a"
512+
fi
513+
if [ -n "`grep SCOTCH $switch_file`" ]
514+
then
515+
export scotch_mod='yes'
516+
esmfpdlib="$SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a \
517+
$SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
518+
fi
506519
fi
507520
export pdlib_mod
521+
export metis_mod
522+
export scotch_mod
508523

509524
# NCEP GRIB
510525
export ncep_grib_compile="no"

‎model/ci/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# Continuous Integration
22
WW3 uses Github Actions for continuous integration.
33

4-
This directory contains a Spack environment file to build library dependencies such as NetCDF, ESMF, ParMETIS, and g2.
4+
This directory contains a Spack environment file to build library dependencies such as NetCDF, ESMF, ParMETIS, SCOTCH and g2.

‎model/ci/spack_gnu.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ spack:
66
specs:
77
- [email protected]~shared
88
- [email protected]~shared
9+
- [email protected]+mpi+metis~shared
910
- [email protected]~dap
1011
- [email protected]
1112
- [email protected]
@@ -15,4 +16,4 @@ spack:
1516
- [email protected]~pio~pnetcdf~xerces
1617
view: true
1718
concretizer:
18-
unify: true
19+
unify: when_possible

‎model/ci/spack_intel.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ spack:
77
specs:
88
- [email protected]~shared
99
- [email protected]~shared
10+
- [email protected]+mpi+metis~shared
1011
- [email protected]~dap
1112
- [email protected]
1213
- [email protected]
@@ -17,4 +18,4 @@ spack:
1718
- intel-oneapi-mpi %intel
1819
view: true
1920
concretizer:
20-
unify: true
21+
unify: when_possible

‎model/src/CMakeLists.txt

+8
Original file line numberDiff line numberDiff line change
@@ -157,9 +157,17 @@ endif()
157157

158158
# Handle PDLIB, SCRIP, SCRIPNC build files directly instead of through configuration file
159159
if("PDLIB" IN_LIST switches)
160+
if("SCOTCH" IN_LIST switches)
161+
find_package(SCOTCH REQUIRED)
162+
target_sources(ww3_lib PRIVATE ${pdlib_src})
163+
target_link_libraries(ww3_lib PUBLIC PTSCOTCHparmetis::PTSCOTCHparmetis)
164+
elseif("METIS" IN_LIST switches)
160165
find_package(ParMETIS REQUIRED)
161166
target_sources(ww3_lib PRIVATE ${pdlib_src})
162167
target_link_libraries(ww3_lib PUBLIC ParMETIS::ParMETIS)
168+
else()
169+
message(FATAL_ERROR "PDLIB requires METIS or SCOTCH library for domain decomposition")
170+
endif()
163171
endif()
164172

165173
if("SCRIP" IN_LIST switches)

‎model/src/PDLIB/yowpdlibmain.F90

+13
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,7 @@ subroutine runParmetis(MNP)
436436
! parmetis need single precision
437437
real(4), allocatable :: xyz(:), tpwgts(:), ubvec(:)
438438
integer :: IP_glob, itmp
439+
integer :: ref
439440
logical :: lexist = .false.
440441

441442
! Node to domain mapping.
@@ -627,12 +628,24 @@ subroutine runParmetis(MNP)
627628
endif
628629

629630
!if(debugParmetis) write(710+myrank,*) "Run ParMETIS now..."
631+
#ifdef W3_SCOTCH
632+
call SCOTCH_ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, &
633+
vwgt, & !vwgt - ignore weights
634+
adjwgt, & ! adjwgt - ignore weights
635+
wgtflag, &
636+
numflag,ndims,xyz,ncon,nparts,tpwgts,ubvec,options, &
637+
edgecut,part, comm,ref)
638+
#endif
639+
640+
#ifdef W3_METIS
630641
call ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, &
631642
vwgt, & !vwgt - ignore weights
632643
adjwgt, & ! adjwgt - ignore weights
633644
wgtflag, &
634645
numflag,ndims,xyz,ncon,nparts,tpwgts,ubvec,options, &
635646
edgecut,part, comm)
647+
#endif
648+
636649

637650
CALL REAL_MPI_BARRIER_PDLIB(comm, "runParmetis, step 9")
638651

‎model/src/cmake/switches.json

+16-1
Original file line numberDiff line numberDiff line change
@@ -749,10 +749,25 @@
749749
"name": "pdlib",
750750
"num_switches": "upto1",
751751
"description": "use pdlib",
752-
"valid-options": [
752+
"valid-options":[
753753
{
754754
"name": "PDLIB",
755755
"requires": ["MPI"]
756+
}
757+
]
758+
},
759+
{
760+
"name": "ddlib",
761+
"num_switches": "upto1",
762+
"description": "domain decomposition library",
763+
"valid-options": [
764+
{
765+
"name": "METIS",
766+
"requires": ["PDLIB"]
767+
},
768+
{
769+
"name": "SCOTCH",
770+
"requires": ["PDLIB"]
756771
}
757772
]
758773
},

‎regtests/bin/matrix.base

+1
Original file line numberDiff line numberDiff line change
@@ -2047,6 +2047,7 @@
20472047
if [ "$pdlib" = 'y' ]
20482048
then
20492049
echo "$rtst -s MPI -s PDLIB -w work_b -g b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
2050+
echo "$rtst -s MPI -s PDLIB_METIS -w work_b_metis -g b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
20502051
echo "$rtst -s MPI -s PDLIB -w work_mb -m grdset_b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
20512052
fi
20522053
fi

0 commit comments

Comments
 (0)