From ed54da1c316e27880085ef4c7093609b5154df92 Mon Sep 17 00:00:00 2001
From: liuyangzhuan <liuyangzhuan@gmail.com>
Date: Fri, 27 Oct 2023 15:23:06 -0400
Subject: [PATCH] change #include hipblas.h to #include <hipblas/hipblas.h>

---
 SRC/gpu_wrapper.h                             |  2 +-
 ...batch_script_mpi_runit_frontier_gcc_hip.sh | 27 ++++++++++++++-----
 ...un_cmake_build_frontier_gnu_gpu_hip_amd.sh |  4 +--
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/SRC/gpu_wrapper.h b/SRC/gpu_wrapper.h
index e7ec2d8d..4ac86e85 100644
--- a/SRC/gpu_wrapper.h
+++ b/SRC/gpu_wrapper.h
@@ -105,7 +105,7 @@ at the top-level directory.
 
 #include "hip/hip_runtime_api.h"
 #include "hip/hip_runtime.h"
-#include "hipblas.h"
+#include <hipblas/hipblas.h>
 
 // #include "roctracer_ext.h"    // need to pass the include dir directly to HIP_HIPCC_FLAGS
 // // roctx header file
diff --git a/example_scripts/batch_script_mpi_runit_frontier_gcc_hip.sh b/example_scripts/batch_script_mpi_runit_frontier_gcc_hip.sh
index ba20f64a..cc50db1b 100644
--- a/example_scripts/batch_script_mpi_runit_frontier_gcc_hip.sh
+++ b/example_scripts/batch_script_mpi_runit_frontier_gcc_hip.sh
@@ -1,4 +1,12 @@
 #!/bin/bash
+#SBATCH -A csc289
+#SBATCH -J superlu_test
+#SBATCH -o %x-%j.out
+#SBATCH -t 00:20:00
+#SBATCH -p batch
+#SBATCH -N 2
+
+
 # Bash script to submit many files to Cori/Edison/Queue
 EXIT_SUCCESS=0
 EXIT_HOST=1
@@ -35,7 +43,10 @@ FILE=$FILE_DIR/$FILE_NAME
 CORES_PER_NODE=64
 
 
-
+export SUPERLU_NUM_GPU_STREAMS=1
+# export SUPERLU_BIND_MPI_GPU=1
+export SUPERLU_ACC_OFFLOAD=0
+export MAX_BUFFER_SIZE=500000000 
 #nprows=(6 12 24)
 #npcols=(6 12 24)
 
@@ -74,8 +85,8 @@ CORES_PER_NODE=64
 #npcols=(12 144 1)
 
 
-nprows=(2 4 8)
-npcols=(2 4 8)
+nprows=(4)
+npcols=(4)
  
 for ((i = 0; i < ${#npcols[@]}; i++)); do
 NROW=${nprows[i]}
@@ -90,8 +101,7 @@ then
   NODE_VAL=`expr $NODE_VAL + 1`
 fi
 
-export MAX_BUFFER_SIZE=500000000 
-export NUM_GPU_STREAMS=1
+
 for NTH in 1 
 do
 OMP_NUM_THREADS=$NTH
@@ -121,7 +131,9 @@ OMP_NUM_THREADS=$NTH
   # for MAT in HTS/boyd1.mtx
   # for MAT in HTS/rajat16.mtx
   # for MAT in big.rua
-  for MAT in Ga19As19H42.mtx
+  # for MAT in Geo_1438.mtx
+  # for MAT in Ga19As19H42.mtx
+  for MAT in s2D9pt2048.rua 
   # for MAT in matrix121.dat matrix211.dat tdr190k.dat tdr455k.dat nlpkkt80.mtx torso3.mtx helm2d03.mtx  
   # for MAT in tdr190k.dat Ga19As19H42.mtx
  # for MAT in torso3.mtx hvdc2.mtx matrix121.dat nlpkkt80.mtx helm2d03.mtx
@@ -137,7 +149,8 @@ OMP_NUM_THREADS=$NTH
     mkdir -p $MAT
     #srun -n $CORE_VAL -c $NTH --cpu_bind=cores /opt/rocm/bin/rocprof --hsa-trace --hip-trace $FILE -c $NCOL -r $NROW $INPUT_DIR/$MAT | tee ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_${OMP_NUM_THREADS}_mrhs
     #srun -n $CORE_VAL -c $NTH --cpu_bind=cores /opt/rocm/bin/rocprof --hsa-trace --roctx-trace $FILE -c $NCOL -r $NROW $INPUT_DIR/$MAT | tee ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_${OMP_NUM_THREADS}_mrhs
-    srun -n $CORE_VAL -c $NTH --cpu_bind=cores $FILE -c $NCOL -r $NROW $INPUT_DIR/$MAT | tee ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_${OMP_NUM_THREADS}_mrhs
+    # srun -n $CORE_VAL -c $NTH --gpu-bind=closest --ntasks-per-gpu=1 --gpus $CORE_VAL --gpus-per-node=8 $FILE -c $NCOL -r $NROW $INPUT_DIR/$MAT | tee ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_${OMP_NUM_THREADS}_mrhs
+    srun -N ${SLURM_NNODES} --ntasks-per-node=8 --gpus-per-task=1 --gpu-bind=closest $FILE -c $NCOL -r $NROW $INPUT_DIR/$MAT | tee ./$MAT/SLU.o_mpi_${NROW}x${NCOL}_${OMP_NUM_THREADS}_mrhs
     # Add final line (srun line) to temporary slurm script
 
   done
diff --git a/example_scripts/run_cmake_build_frontier_gnu_gpu_hip_amd.sh b/example_scripts/run_cmake_build_frontier_gnu_gpu_hip_amd.sh
index 6a13ac78..e6ac85b3 100644
--- a/example_scripts/run_cmake_build_frontier_gnu_gpu_hip_amd.sh
+++ b/example_scripts/run_cmake_build_frontier_gnu_gpu_hip_amd.sh
@@ -35,8 +35,8 @@ cmake .. \
 	-DCMAKE_C_COMPILER=cc \
 	-DCMAKE_CXX_COMPILER=CC \
 	-DCMAKE_INSTALL_PREFIX=. \
-	-DTPL_BLAS_LIBRARIES="${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu_81_mp.so" \
-	-DTPL_LAPACK_LIBRARIES="${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu_81_mp.so" \
+	-DTPL_BLAS_LIBRARIES="${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu_82_mp.so" \
+	-DTPL_LAPACK_LIBRARIES="${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_gnu_82_mp.so" \
 	-DCMAKE_BUILD_TYPE=Release \
 	-DTPL_ENABLE_HIPLIB=TRUE \
 	-DHIP_HIPCC_FLAGS="--amdgpu-target=gfx90a -I${CRAY_MPICH_DIR}/include" \