resolved merge conflicts; however, gcc with GPU3DVERSION=1 will still…

… crash, nvidia is working correctly

CMakeLists.txt

@@ -288,18 +288,21 @@ if (TPL_ENABLE_CUDALIB)   ## want to use cuda
     set(HAVE_CUDA TRUE)
     # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DHAVE_CUDA")
     # set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -DHAVE_CUDA")
+
+    #--------------------- NVSHMEM libraries ---------------------
+    if (TPL_ENABLE_NVSHMEM)   ## want to use nvshmem
+        set(NVSHMEM_LIB ${TPL_NVSHMEM_LIBRARIES}) 
+        message("-- Enabled support for NVSHMEM='${NVSHMEM_LIB}'")
+        set(HAVE_NVSHMEM TRUE)
+    endif()
+
   else()
     message("-- CUDA libraries not found.")
   endif()
 endif()
 
 
-#--------------------- NVSHMEM libraries ---------------------
-if (TPL_ENABLE_NVSHMEM)   ## want to use nvshmem
-    set(NVSHMEM_LIB ${TPL_NVSHMEM_LIBRARIES}) 
-    message("-- Enabled support for NVSHMEM='${NVSHMEM_LIB}'")
-    set(HAVE_NVSHMEM TRUE)
-endif()
+
 
 #--------------------- HIP libraries ---------------------
 if (TPL_ENABLE_HIPLIB)   ## want to use hip

Dockerfile

@@ -0,0 +1,14 @@
+# FROM ubuntu:16.04
+# FROM debian:stable
+# FROM ubuntu:18.04
+FROM nvidia/cuda:11.4.0-base-ubuntu20.04
+
+WORKDIR /app
+RUN apt-get update
+RUN apt-get install git -y
+RUN git clone https://github.com/xiaoyeli/superlu_dist.git
+WORKDIR superlu_dist
+RUN git fetch
+RUN git pull
+RUN git checkout gpu_trisolve_new
+# RUN bash config_cleanlinux.sh  # not sure why parmetis build fails, build superlu inside docker manually instead

EXAMPLE/Makefile

@@ -32,7 +32,7 @@ include ../make.inc
 
 SEXM	= psdrive.o screate_matrix.o dcreate_matrix.o screate_A_x_b.o \
 	  psgssvx_tracking.o psgsrfs_tracking.o 
-	## psutil.o sdistribute.o
+	#	psutil.o sdistribute.o
 # psgssvx_d2.o psgsrfs_d2.o sutil_dist.o psgsequb.o
 
 DEXM	= pddrive.o dcreate_matrix.o 

EXAMPLE/pddrive.c

@@ -119,6 +119,8 @@ int main(int argc, char *argv[])
     options.IterRefine = NOREFINE;
     options.DiagInv           = YES;
 #if 0
+    options.ParSymbFact       = YES;
+    options.ColPerm           = PARMETIS;
     options.RowPerm = LargeDiag_HWPM;
     options.IterRefine = NOREFINE;
     options.ColPerm = NATURAL;

EXAMPLE/pddrive1.c

@@ -206,15 +206,14 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        2. NOW SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
        RIGHT-HAND SIDE,  WE WILL USE THE EXISTING L AND U FACTORS IN
        LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
        ------------------------------------------------------------*/
     options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     nrhs = 1;
     pdgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,
@@ -232,15 +231,14 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        3. SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
        NUMBER OF RIGHT-HAND SIDES,  WE WILL USE THE EXISTING L AND U
        FACTORS IN LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
        ------------------------------------------------------------*/
     options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     nrhs = 3;
 
@@ -269,11 +267,11 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        DEALLOCATE STORAGE.
        ------------------------------------------------------------*/
+    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A);
     dScalePermstructFree(&ScalePermstruct);   
     dDestroy_LU(n, &grid, &LUstruct);

EXAMPLE/pddrive2.c

@@ -201,7 +201,6 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A.  */ 
     dDestroy_LU(n, &grid, &LUstruct); /* Deallocate storage associated with 
 					the L and U matrices.  */
@@ -228,7 +227,7 @@ int main(int argc, char *argv[])
     dcreate_matrix_perturbed_postfix(&A, nrhs, &b1, &ldb,
                                   &xtrue1, &ldx, fp, postfix, &grid);
 
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     /* Solve the linear system. */
     pdgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,

EXAMPLE/pddrive3.c

@@ -216,7 +216,6 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A.  */
     SUPERLU_FREE(b);                 /* Free storage of right-hand side.    */
 
@@ -227,7 +226,7 @@ int main(int argc, char *argv[])
        NUMERICAL VALUES AS THAT IN A PREVIOUS SYSTEM.
        ------------------------------------------------------------*/
     options.Fact = SamePattern_SameRowPerm;
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     /* Set up the local A in NR_loc format */
 

EXAMPLE/pddrive3d.c

@@ -363,6 +363,9 @@ main (int argc, char *argv[])
     options.DiagInv           = YES;
     // options.ParSymbFact       = YES;
     // options.ColPerm           = PARMETIS;
+    options.Algo3d = YES;
+	options.DiagInv = YES;
+    options.ReplaceTinyPivot  = YES;    
 #if 0
     options.ReplaceTinyPivot = YES;
     options.RowPerm = NOROWPERM;
@@ -425,11 +428,12 @@ main (int argc, char *argv[])
        DEALLOCATE STORAGE.
        ------------------------------------------------------------ */
 
-    dDestroy_LU (n, &(grid.grid2d), &LUstruct);
+    // dDestroy_LU (n, &(grid.grid2d), &LUstruct);
     if ( grid.zscp.Iam == 0 ) { // process layer 0
 	    PStatPrint (&options, &stat, &(grid.grid2d)); /* Print 2D statistics.*/
     }
     dSolveFinalize (&options, &SOLVEstruct);
+    dDestroy_LU (n, &(grid.grid2d), &LUstruct);
 
     dDestroy_A3d_gathered_on_2d(&SOLVEstruct, &grid);
 

EXAMPLE/psdrive1.c

@@ -207,15 +207,14 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        2. NOW SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
        RIGHT-HAND SIDE,  WE WILL USE THE EXISTING L AND U FACTORS IN
        LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
        ------------------------------------------------------------*/
     options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     nrhs = 1;
     psgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,
@@ -233,15 +232,14 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        3. SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
        NUMBER OF RIGHT-HAND SIDES,  WE WILL USE THE EXISTING L AND U
        FACTORS IN LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
        ------------------------------------------------------------*/
     options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     nrhs = 3;
 
@@ -270,11 +268,11 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        DEALLOCATE STORAGE.
        ------------------------------------------------------------*/
+    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A);
     sScalePermstructFree(&ScalePermstruct);   
     sDestroy_LU(n, &grid, &LUstruct);

EXAMPLE/psdrive2.c

@@ -202,7 +202,6 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A.  */ 
     sDestroy_LU(n, &grid, &LUstruct); /* Deallocate storage associated with 
 					the L and U matrices.  */
@@ -229,7 +228,7 @@ int main(int argc, char *argv[])
     screate_matrix_perturbed_postfix(&A, nrhs, &b1, &ldb,
                                   &xtrue1, &ldx, fp, postfix, &grid);
 
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     /* Solve the linear system. */
     psgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,

EXAMPLE/psdrive3.c

@@ -217,7 +217,6 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A.  */
     SUPERLU_FREE(b);                 /* Free storage of right-hand side.    */
 
@@ -228,7 +227,7 @@ int main(int argc, char *argv[])
        NUMERICAL VALUES AS THAT IN A PREVIOUS SYSTEM.
        ------------------------------------------------------------*/
     options.Fact = SamePattern_SameRowPerm;
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     /* Set up the local A in NR_loc format */
 

EXAMPLE/pzdrive1.c

@@ -205,15 +205,14 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        2. NOW SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
        RIGHT-HAND SIDE,  WE WILL USE THE EXISTING L AND U FACTORS IN
        LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
        ------------------------------------------------------------*/
     options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     nrhs = 1;
     pzgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,
@@ -231,15 +230,14 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        3. SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
        NUMBER OF RIGHT-HAND SIDES,  WE WILL USE THE EXISTING L AND U
        FACTORS IN LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
        ------------------------------------------------------------*/
     options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     nrhs = 3;
 
@@ -268,11 +266,11 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
 
     /* ------------------------------------------------------------
        DEALLOCATE STORAGE.
        ------------------------------------------------------------*/
+    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A);
     zScalePermstructFree(&ScalePermstruct);   
     zDestroy_LU(n, &grid, &LUstruct);

EXAMPLE/pzdrive2.c

@@ -200,7 +200,6 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A.  */ 
     zDestroy_LU(n, &grid, &LUstruct); /* Deallocate storage associated with 
 					the L and U matrices.  */
@@ -227,7 +226,7 @@ int main(int argc, char *argv[])
     zcreate_matrix_perturbed_postfix(&A, nrhs, &b1, &ldb,
                                   &xtrue1, &ldx, fp, postfix, &grid);
 
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     /* Solve the linear system. */
     pzgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,

EXAMPLE/pzdrive3.c

@@ -215,7 +215,6 @@ int main(int argc, char *argv[])
     }
 
     PStatPrint(&options, &stat, &grid);        /* Print the statistics. */
-    PStatFree(&stat);
     Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A.  */
     SUPERLU_FREE(b);                 /* Free storage of right-hand side.    */
 
@@ -226,7 +225,7 @@ int main(int argc, char *argv[])
        NUMERICAL VALUES AS THAT IN A PREVIOUS SYSTEM.
        ------------------------------------------------------------*/
     options.Fact = SamePattern_SameRowPerm;
-    PStatInit(&stat); /* Initialize the statistics variables. */
+    PStatClear(&stat); /* clear the statistics variables. */
 
     /* Set up the local A in NR_loc format */
 

PATCH/parmetis/CMakeLists.txt

@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 2.8)
+project(ParMETIS)
+
+set(GKLIB_PATH METIS/GKlib CACHE PATH "path to GKlib")
+set(METIS_PATH METIS CACHE PATH "path to METIS")
+
+# Symlink ./metis to wherever metis is. This allows files to be
+# included from metis/libmetis/.
+execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${METIS_PATH} metis)
+
+# Search for MPI.
+# GK commented this out as it seems to be creating problems
+# include(FindMPI)
+# if(NOT MPI_FOUND)
+#   message(FATAL_ERROR "mpi is not found")
+# endif()
+# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${MPI_COMPILE_FLAGS}")
+
+# Prepare libraries.
+if(SHARED)
+  set(ParMETIS_LIBRARY_TYPE SHARED)
+  set(METIS_LIBRARY_TYPE SHARED)
+else()
+  set(ParMETIS_LIBRARY_TYPE STATIC)
+  set(METIS_LIBRARY_TYPE STATIC)
+endif()
+
+include(${GKLIB_PATH}/GKlibSystem.cmake)
+
+# List of paths that the compiler will search for header files.
+# i.e., the -I equivalent
+include_directories(include)
+include_directories(${MPI_INCLUDE_PATH})
+include_directories(${GKLIB_PATH})
+include_directories(${METIS_PATH}/include)
+
+# List of directories that cmake will look for CMakeLists.txt
+add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis)
+add_subdirectory(include)
+add_subdirectory(libparmetis)
+add_subdirectory(programs)
+
+# This is for testing during development and is not being distributed
+#add_subdirectory(test)

SRC/CMakeLists.txt

@@ -109,6 +109,10 @@ endif()
     double/pdsymbfact_distdata.c
     double/ddistribute.c
     double/pddistribute.c
+    double/pddistribute3d.c
+    double/d3DPartition.c
+    double/distCheckArray.c
+    double/pddistribute-aux3d.c
     double/pdgstrf.c
     double/dstatic_schedule.c
     double/pdgstrf2.c
@@ -142,6 +146,7 @@ if (TPL_ENABLE_CUDALIB)
   TRF3dV100/pdgstrf3d_summit.cpp
     TRF3dV100/dsparseTreeFactor_summit.cpp
     TRF3dV100/dsparseTreeFactorGPU.cpp
+    TRF3dV100/anc25d-GPU.cpp 
     TRF3dV100/lupanels.cpp
     TRF3dV100/lupanels_comm3d.cpp
     TRF3dV100/lupanelsComm3dGPU.cpp