Moved UpdateIntermediateVariables into kernel.

Author: hctorres

swm_AMReX/Make.package

@@ -1,3 +1,3 @@
 CEXE_sources += main.cpp swm_mini_app_utils.cpp
-CEXE_headers += swm_mini_app_utils.h
+CEXE_headers += swm_mini_app_utils.h swm_mini_app_kernels.h
 

swm_AMReX/README.md

@@ -3,8 +3,8 @@
 Based on the NCAR SWM mini-app found [here](https://github.com/NCAR/SWM). 
 
 ## Prerequisites
- - g++ (a version with support for c++20)
-    - Other compilers should also work fine but I have been using gcc/g++ for testing so far.
+ - A compiler for C, C++, and Fortran 
+    - Just about any compilers should work but I have been using gcc/g++/gfortran for testing so far.
  - make
  - [AMReX](https://github.com/AMReX-Codes/amrex)
  - [yt](https://yt-project.org/) (Only needed if you want to run the postprocessing script [plot_with_yt.py](plot_with_yt.py))

swm_AMReX/main.cpp

@@ -133,15 +133,13 @@ int main (int argc, char* argv[])
     Copy(p, p_old);
 
     // Constants used in time stepping loop
-    const double fsdx = 4.0/dx;
-    const double fsdy = 4.0/dy;
     double tdt = dt;
     const double alpha = 0.001; 
 
     for (int time_step = 1; time_step <= n_time_steps; ++time_step)
     {
         // Sets: cu, cv, h, z
-        UpdateIntermediateVariables(fsdx, fsdy, geom,
+        UpdateIntermediateVariables(dx, dy, geom,
                                      p, u, v,
                                      cu, cv, h, z);
 
@@ -161,10 +159,6 @@ int main (int argc, char* argv[])
         // Sets: p, u, v
         UpdateVariables(geom, u_new, v_new, p_new, u, v, p);
 
-        if (time_step == 0) {
-            tdt = tdt + tdt;
-        }
-
         time = time + dt;
 
         // Write a plotfile of the current data (plot_interval was defined in the inputs file)
@@ -173,6 +167,10 @@ int main (int argc, char* argv[])
             WriteOutput(psi, p, u, v, geom, time, time_step, output_values);
         }
 
+        if (time_step == 0) {
+            tdt = tdt + tdt;
+        }
+
     }
 
     //amrex::Print() << "Final: " << std::endl;

swm_AMReX/swm_mini_app_kernels.h

@@ -0,0 +1,22 @@
+#ifndef SWM_MINI_APP_KERNELS_H_
+#define SWM_MINI_APP_KERNELS_H_
+
+#include <AMReX.H>
+
+AMREX_GPU_DEVICE AMREX_FORCE_INLINE
+void UpdateIntermediateVariablesKernel( const int i, const int j, const int k, const double fsdx, const double fsdy,
+                                        const amrex::Array4<amrex::Real const>& p,
+                                        const amrex::Array4<amrex::Real const>& u,
+                                        const amrex::Array4<amrex::Real const>& v,
+                                        const amrex::Array4<amrex::Real>& cu,
+                                        const amrex::Array4<amrex::Real>& cv,
+                                        const amrex::Array4<amrex::Real>& h,
+                                        const amrex::Array4<amrex::Real>& z)
+{
+            cu(i,j,k) = 0.5*(p(i,j,k) + p(i+1,j,k))*u(i,j,k);
+            cv(i,j,k) = 0.5*(p(i,j,k) + p(i,j+1,k))*v(i,j,k);
+            z(i,j,k) = (fsdx*(v(i+1,j,k)-v(i,j,k)) + fsdy*(u(i,j+1,k)-u(i,j,k)))/(p(i,j,k)+p(i+1,j,k)+p(i,j+1,k)+p(i+1,j+1,k));
+            h(i,j,k) = p(i,j,k) + 0.25*(u(i-1,j,k)*u(i-1,j,k) + u(i,j,k)*u(i,j,k) + v(i,j-1,k)*v(i,j-1,k) + v(i,j,k)*v(i,j,k));
+}
+
+#endif // SWM_MINI_APP_KERNELS_H_

swm_AMReX/swm_mini_app_utils.cpp

@@ -8,7 +8,7 @@
 #include <AMReX_Array.H>
 
 #include "swm_mini_app_utils.h"
-
+#include "swm_mini_app_kernels.h"
 
 void ParseInput(int & nx, int & ny,
                 amrex::Real & dx, amrex::Real & dy,
@@ -320,10 +320,13 @@ void Copy(const amrex::MultiFab & src, amrex::MultiFab & dest)
     return;
 }
 
-void UpdateIntermediateVariables(amrex::Real fsdx, amrex::Real fsdy, const amrex::Geometry& geom,
+void UpdateIntermediateVariables(amrex::Real dx, amrex::Real dy, const amrex::Geometry& geom,
                                  const amrex::MultiFab& p, const amrex::MultiFab& u, const amrex::MultiFab& v,
                                  amrex::MultiFab& cu, amrex::MultiFab& cv, amrex::MultiFab& h, amrex::MultiFab& z)
 {
+    const double fsdx = 4.0/dx;
+    const double fsdy = 4.0/dy;
+
     for (amrex::MFIter mfi(p); mfi.isValid(); ++mfi)
     {
         const amrex::Box& bx = mfi.validbox();
@@ -341,10 +344,9 @@ void UpdateIntermediateVariables(amrex::Real fsdx, amrex::Real fsdy, const amrex
 
         amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k)
         {
-            cu_array(i,j,k) = 0.5*(p_array(i,j,k) + p_array(i+1,j,k))*u_array(i,j,k);
-            cv_array(i,j,k) = 0.5*(p_array(i,j,k) + p_array(i,j+1,k))*v_array(i,j,k);
-            z_array(i,j,k) = (fsdx*(v_array(i+1,j,k)-v_array(i,j,k)) + fsdy*(u_array(i,j+1,k)-u_array(i,j,k)))/(p_array(i,j,k)+p_array(i+1,j,k)+p_array(i,j+1,k)+p_array(i+1,j+1,k));
-            h_array(i,j,k) = p_array(i,j,k) + 0.25*(u_array(i-1,j,k)*u_array(i-1,j,k) + u_array(i,j,k)*u_array(i,j,k) + v_array(i,j-1,k)*v_array(i,j-1,k) + v_array(i,j,k)*v_array(i,j,k));
+            UpdateIntermediateVariablesKernel(i, j, k, fsdx, fsdy,
+                                              p_array, u_array, v_array,
+                                              cu_array, cv_array, h_array, z_array);
         });
     }
 

swm_AMReX/swm_mini_app_utils.h

@@ -50,7 +50,7 @@ amrex::MultiFab CreateMultiFab(const amrex::MultiFab & mf);
 
 void Copy(const amrex::MultiFab & src, amrex::MultiFab & dest);
 
-void UpdateIntermediateVariables(amrex::Real fsdx, amrex::Real fsdy, const amrex::Geometry& geom,
+void UpdateIntermediateVariables(amrex::Real dx, amrex::Real dy, const amrex::Geometry& geom,
                                  const amrex::MultiFab& p, const amrex::MultiFab& u, const amrex::MultiFab& v,
                                  amrex::MultiFab& cu, amrex::MultiFab& cv, amrex::MultiFab& h, amrex::MultiFab& z);