Only rank 0 count attribute write size:

Attribute write is collective. However, only one process will actually write the attribute.
In HDF5 and ADIOS2 driver, when counting application write size, attirbute write should only be count on one process.
We count attribute write size at rank 0.
Attribute read size is still counted on all processes.

Author: khou2020

src/drivers/e3sm_io_driver_adios2.cpp

@@ -90,12 +90,15 @@ e3sm_io_driver_adios2::~e3sm_io_driver_adios2 () {
 
 int e3sm_io_driver_adios2::create (std::string path, MPI_Comm comm, MPI_Info info, int *fid) {
     int nerrs = 0;
+    int err;
     adios2_error aerr;
     adios2_file *fp;
     char ng[32];
 
     fp       = new adios2_file ();
     fp->path = std::string (path);
+    err      = MPI_Comm_rank (comm, &(fp->rank));
+    CHECK_MPIERR
 
     fp->adp = adios2_init (comm, "");
     CHECK_APTR (fp->adp)
@@ -134,12 +137,15 @@ err_out:;
 
 int e3sm_io_driver_adios2::open (std::string path, MPI_Comm comm, MPI_Info info, int *fid) {
     int nerrs = 0;
+    int err;
     adios2_error aerr;
     adios2_file *fp;
     adios2_step_status stat;
 
     fp       = new adios2_file ();
     fp->path = std::string (path);
+    err      = MPI_Comm_rank (comm, &(fp->rank));
+    CHECK_MPIERR
 
     fp->adp = adios2_init (comm, "");
     CHECK_APTR (fp->adp)
@@ -176,8 +182,8 @@ int e3sm_io_driver_adios2::close (int fid) {
     adios2_bool result;
 
     if (fp->ep) {
-        //aerr = adios2_end_step (fp->ep);
-        //CHECK_AERR
+        // aerr = adios2_end_step (fp->ep);
+        // CHECK_AERR
 
         aerr = adios2_close (fp->ep);
         CHECK_AERR
@@ -532,8 +538,10 @@ int e3sm_io_driver_adios2::put_att (
         CHECK_APTR (aid)
     }
 
-    esize = adios2_type_size (atype);
-    fp->putsize += size * esize;
+    if (fp->rank == 0) {
+        esize = adios2_type_size (atype);
+        fp->putsize += size * esize;
+    }
 
 err_out:;
     return nerrs;

src/drivers/e3sm_io_driver_adios2.hpp

@@ -56,6 +56,7 @@ class e3sm_io_driver_adios2 : public e3sm_io_driver {
         adios2_operator *op;
         MPI_Offset putsize = 0;
         MPI_Offset getsize = 0;
+        int rank;
     } adios2_file;
     std::vector<adios2_file *> files;
     double tsel, twrite, tread, text;

src/drivers/e3sm_io_driver_hdf5.cpp

@@ -130,6 +130,7 @@ e3sm_io_driver_hdf5::~e3sm_io_driver_hdf5 () {
 
 int e3sm_io_driver_hdf5::create (std::string path, MPI_Comm comm, MPI_Info info, int *fid) {
     int nerrs = 0;
+    int err;
     herr_t herr;
     hid_t faplid;
     hdf5_file *fp;
@@ -138,6 +139,9 @@ int e3sm_io_driver_hdf5::create (std::string path, MPI_Comm comm, MPI_Info info,
 
     fp = new hdf5_file (*this);
 
+    err = MPI_Comm_rank (comm, &(fp->rank));
+    CHECK_MPIERR
+
     faplid = H5Pcreate (H5P_FILE_ACCESS);
     CHECK_HID (faplid)
     herr = H5Pset_fapl_mpio (faplid, comm, info);
@@ -166,6 +170,7 @@ err_out:;
 
 int e3sm_io_driver_hdf5::open (std::string path, MPI_Comm comm, MPI_Info info, int *fid) {
     int nerrs = 0;
+    int err;
     herr_t herr;
     hid_t faplid;
     hdf5_file *fp;
@@ -174,6 +179,9 @@ int e3sm_io_driver_hdf5::open (std::string path, MPI_Comm comm, MPI_Info info, i
 
     fp = new hdf5_file (*this);
 
+    err = MPI_Comm_rank (comm, &(fp->rank));
+    CHECK_MPIERR
+
     faplid = H5Pcreate (H5P_FILE_ACCESS);
     CHECK_HID (faplid)
     herr = H5Pset_fapl_mpio (faplid, comm, info);
@@ -528,9 +536,11 @@ int e3sm_io_driver_hdf5::put_att (
 
     E3SM_IO_TIMER_STOP (E3SM_IO_TIMER_HDF5)
 
-    err = MPI_Type_size (type, &esize);
-    CHECK_MPIERR
-    fp->putsize += asize * esize;
+    if (fp->rank == 0) {
+        err = MPI_Type_size (type, &esize);
+        CHECK_MPIERR
+        fp->putsize += asize * esize;
+    }
 
 err_out:;
     if (asid >= 0) H5Sclose (asid);

src/drivers/e3sm_io_driver_hdf5.hpp

@@ -34,6 +34,7 @@ class e3sm_io_driver_hdf5 : public e3sm_io_driver {
         MPI_Offset recsize = 0;
         MPI_Offset putsize = 0;
         MPI_Offset getsize = 0;
+        int rank;
 
 #ifndef HDF5_HAVE_DWRITE_MULTI
         typedef struct H5D_rw_multi_t {