From 0eb41e0e17c90817b2f1ae523ac87ff4b93804d0 Mon Sep 17 00:00:00 2001
From: Arturo Vargas <artv3.89@gmail.com>
Date: Mon, 23 May 2022 09:13:07 -0700
Subject: [PATCH] update hypre + use non-conforming meshes in ex1.cpp

---
 machine-configs/lassen.sh |  6 ++++--
 package-builders/hypre.sh | 16 ++++++++++++++++
 tests/mfem_ex1/ex1.cpp    |  3 +++
 tests/mfem_ex1/ex1.sh     |  4 ++--
 4 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/machine-configs/lassen.sh b/machine-configs/lassen.sh
index 6df7b5a..9bc4d80 100644
--- a/machine-configs/lassen.sh
+++ b/machine-configs/lassen.sh
@@ -21,7 +21,7 @@ function setup_xlc()
    MPIF77=mpixlf
    mpi_info_flag="-qversion=verbose"
 
-   CFLAGS="-O2 -qmaxmem=-1 -qarch=auto -qcache=auto -qtune=auto"
+   CFLAGS="-O2"
    # CFLAGS="-O3 -qarch=auto -qcache=auto -qhot -qtune=auto"
    # CFLAGS+=" -qipa=threads"
 
@@ -44,10 +44,12 @@ function setup_gcc()
 
    MPICC=mpigcc
    MPICXX=mpig++
+   which $MPICXX
+   $MPICXX --version
    MPIF77=mpigfortran
    mpi_info_flag="--version"
 
-   CFLAGS="-O3 -mcpu=native -mtune=native"
+   CFLAGS="-O3"
    FFLAGS="$CFLAGS"
    TEST_EXTRA_CFLAGS=""
    # TEST_EXTRA_CFLAGS+=" -std=c++11 -fdump-tree-optimized-blocks"
diff --git a/package-builders/hypre.sh b/package-builders/hypre.sh
index 5bd03ae..797beaf 100644
--- a/package-builders/hypre.sh
+++ b/package-builders/hypre.sh
@@ -73,19 +73,35 @@ function hypre_build()
       my_cflags="-g"
    fi
    echo "Building $pkg, sending output to ${pkg_bld_dir}_build.log ..." && {
+   local CUDA_MAKE_OPTS=()
+   if [[ -n "$CUDA_ENABLED" ]]; then
+      CUDA_MAKE_OPTS=(
+         "--with-cuda"
+         "--with-gpu-arch=70"
+         "--enable-device-memory-pool"
+      #   "CUCC=$cuda_home/bin/nvcc"
+      )
+      xcompiler="-Xcompiler="
+      optim_flags="$cxx11_flag $xcompiler\"$CFLAGS\""
+   else
+      echo "${magenta}INFO: Building $pkg without CUDA ...${none}"
+   fi
       cd "$pkg_bld_dir/src" && \
       if [[ -e config/Makefile.config ]]; then
          make distclean
       fi && \
+      CUCC="$cuda_home/bin/nvcc -ccbin=$MPICXX" \
       ./configure \
          CC="$MPICC" \
          CXX="$MPICXX" \
          CFLAGS="$my_cflags" \
          CXXFLAGS="$my_cflags" \
          $HYPRE_EXTRA_CONFIG \
+         "${CUDA_MAKE_OPTS[@]}" \
          ${hypre_big_int:+--enable-bigint} \
          ${hypre_mixed_int:+--enable-mixedint} \
          ${hypre_debug:+--enable-debug} \
+         --with-MPI \
          --disable-fortran \
          --without-fei && \
       make -j $num_proc_build
diff --git a/tests/mfem_ex1/ex1.cpp b/tests/mfem_ex1/ex1.cpp
index ed825f1..068fc2a 100644
--- a/tests/mfem_ex1/ex1.cpp
+++ b/tests/mfem_ex1/ex1.cpp
@@ -81,6 +81,9 @@ int main(int argc, char *argv[])
    Array<int> nxyz;
    Mesh *mesh = make_mesh(myid, num_procs, dim, level, par_ref_levels, nxyz);
 
+   //Go through NC mesh machinery
+   mesh->EnsureNCMesh();
+
    // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
diff --git a/tests/mfem_ex1/ex1.sh b/tests/mfem_ex1/ex1.sh
index 3686a9c..96a87dc 100644
--- a/tests/mfem_ex1/ex1.sh
+++ b/tests/mfem_ex1/ex1.sh
@@ -39,7 +39,7 @@ function run_tests()
    # 'max_dofs_proc' can be set on the command line
    local l_max_dofs_proc=${max_dofs_proc:-4200000}
    # 'mfem_devs' can be set on the command line
-   local l_mfem_devs=(${mfem_devs:-cpu})
+   local l_mfem_devs=(${mfem_devs:-cuda})
    local dim=3
    local args=
    for dev in ${l_mfem_devs[@]}; do
@@ -92,6 +92,6 @@ libceed_branch=${libceed_branch:-master}
 # Only with HIP
 #packages="hip metis hypre mfem-hip"
 
-packages=${packages:-metis hypre mfem}
+packages=${packages:-metis cuda hypre mfem}
 
 test_required_packages=${packages}