Mitigate S3 test interference + Unlimited Dimensions in NCZarr

.github/workflows/run_tests_win_mingw.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           msystem: MINGW64
           update: true
-          install: git mingw-w64-x86_64-toolchain automake libtool autoconf make cmake mingw-w64-x86_64-hdf5 unzip mingw-w64-x86_64-libxml2
+          install: git mingw-w64-x86_64-toolchain automake libtool autoconf make cmake mingw-w64-x86_64-hdf5 unzip mingw-w64-x86_64-libxml2 mingw-w64-x86_64-zlib
 
 ###
 # Configure and build

CMakeLists.txt

@@ -61,17 +61,17 @@ ENDIF()
 
 # Define some Platforms
 if(osname MATCHES "CYGWIN.*")
-SET(ISCYGWIN yes)
+  SET(ISCYGWIN yes)
 endif()
-if(osname MATCHES "Darwin.*")
-SET(ISOSX yes)
+  if(osname MATCHES "Darwin.*")
+  SET(ISOSX yes)
 endif()
 if(MSVC)
-SET(ISMSVC yes)
+  SET(ISMSVC yes)
 endif()
 if(osname MATCHES "MINGW.*" OR osname MATCHES "MSYS.*")
-SET(ISMINGW yes)
-SET(MINGW yes)
+  SET(ISMINGW yes)
+  SET(MINGW yes)
 endif()
 
 ###
@@ -210,9 +210,6 @@ SET(EXTRA_DEPS "")
 ENABLE_TESTING()
 INCLUDE(CTest)
 
-# Copy the CTest customization file into binary directory, as required.
-FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-
 # Set Memory test program for non-MSVC based builds.
 # Assume valgrind for now.
 IF((NOT MSVC) AND (NOT MINGW) AND (NOT ISCYGWIN))
@@ -1327,24 +1324,31 @@ ENDIF()
 IF(ENABLE_S3)
   IF(NOT ENABLE_S3_INTERNAL)
     # See if aws-s3-sdk is available
-    find_package(AWSSDK REQUIRED COMPONENTS s3;core)
+    find_package(AWSSDK REQUIRED COMPONENTS s3;transfer)
     IF(AWSSDK_FOUND)
-      SET(service s3;core)
-      AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES)
       SET(ENABLE_S3_AWS ON CACHE BOOL "S3 AWS" FORCE)
-    ELSE()
+      INCLUDE_DIRECTORIES(${AWSSDK_INCLUDE_DIR})
+    ELSE(AWSSDK_FOUND)
       SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
-    ENDIF()
-  ENDIF()
+    ENDIF(AWSSDK_FOUND)
+  ELSE(NOT ENABLE_S3_INTERNAL)
+    # Find crypto libraries required with testing with the internal s3 api.
+    #FIND_LIBRARY(SSL_LIB NAMES ssl openssl)
+    find_package(OpenSSL REQUIRED)
+    IF(NOT OpenSSL_FOUND)
+      MESSAGE(FATAL_ERROR "Can't find an ssl library, required by S3_INTERNAL")
+    ENDIF(NOT OpenSSL_FOUND)
+
+    #find_package(Crypto REQUIRED)
+    #IF(NOT CRYPTO_LIB)
+    #  MESSAGE(FATAL_ERROR "Can't find a crypto library, required by S3_INTERNAL")
+    #ENDIF(NOT CRYPTO_LIB)
+
+  ENDIF(NOT ENABLE_S3_INTERNAL)
 ELSE()
   SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
 ENDIF()
 
-# Unless/until we get aws-sdk-cpp working for Windows, force use of internal
-IF(ENABLE_S3 AND MSVC)
-  SET(ENABLE_S3_INTERNAL ON CACHE BOOL "S3 Intern" FORCE)
-ENDIF()
-
 IF(ENABLE_S3)
   IF(NOT ENABLE_S3_AWS AND NOT ENABLE_S3_INTERNAL)
     message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO")
@@ -1357,10 +1361,10 @@ IF(ENABLE_S3)
 ENDIF()
 
 IF(NOT ENABLE_S3)
-IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
-  message(WARNING "S3 support is disabled => WITH_S3_TESTING=OFF")
-  SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
-ENDIF()
+  IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
+    message(WARNING "S3 support is disabled => WITH_S3_TESTING=OFF")
+    SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE)
+  ENDIF()
 ENDIF()
 
 OPTION(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON)
@@ -1791,6 +1795,7 @@ IF(MSVC)
 CHECK_INCLUDE_FILE("io.h"      HAVE_IO_H)
 ENDIF(MSVC)
 CHECK_INCLUDE_FILE("stdlib.h"  HAVE_STDLIB_H)
+CHECK_INCLUDE_FILE("ctype.h"   HAVE_CTYPE_H)
 CHECK_INCLUDE_FILE("stdarg.h"    HAVE_STDARG_H)
 CHECK_INCLUDE_FILE("strings.h"   HAVE_STRINGS_H)
 CHECK_INCLUDE_FILE("signal.h"    HAVE_SIGNAL_H)
@@ -1891,6 +1896,10 @@ CHECK_TYPE_SIZE("uintptr_t" SIZEOF_UINTPTR_T)
 IF(SIZEOF_UINTPTR_T)
   SET(HAVE_UINTPTR_T TRUE)
 ENDIF(SIZEOF_UINTPTR_T)
+CHECK_TYPE_SIZE("mode_t" SIZEOF_MODE_T)
+IF(SIZEOF_MODE_T)
+  SET(HAVE_MODE_T TRUE)
+ENDIF(SIZEOF_MODE_T)
 
 # __int64 is used on Windows for large file support.
 CHECK_TYPE_SIZE("__int64"   SIZEOF___INT_64)
@@ -1953,16 +1962,16 @@ ENDIF()
 
 # Check to see if MAP_ANONYMOUS is defined.
 IF(MSVC)
-MESSAGE(WARNING "mmap not supported under visual studio: disabling MMAP support.")
-SET(ENABLE_MMAP OFF)
-ELSE()
-CHECK_C_SOURCE_COMPILES("
-#include <sys/mman.h>
-int main() {int x = MAP_ANONYMOUS;}" HAVE_MAPANON)
-IF(NOT HAVE_MMAP OR NOT HAVE_MAPANON)
-  MESSAGE(WARNING "mmap or MAP_ANONYMOUS not found: disabling MMAP support.")
+  MESSAGE(WARNING "mmap not supported under visual studio: disabling MMAP support.")
   SET(ENABLE_MMAP OFF)
-ENDIF()
+ELSE()
+  CHECK_C_SOURCE_COMPILES("
+  #include <sys/mman.h>
+  int main() {int x = MAP_ANONYMOUS;}" HAVE_MAPANON)
+  IF(NOT HAVE_MMAP OR NOT HAVE_MAPANON)
+    MESSAGE(WARNING "mmap or MAP_ANONYMOUS not found: disabling MMAP support.")
+    SET(ENABLE_MMAP OFF)
+  ENDIF()
 ENDIF()
 
 IF(ENABLE_MMAP)
@@ -2136,6 +2145,7 @@ MACRO(build_bin_test_no_prefix F)
   ENDIF()
 ENDMACRO()
 
+# Build a test and add it to the test list.
 MACRO(add_bin_test prefix F)
   ADD_EXECUTABLE(${prefix}_${F} ${F}.c ${ARGN})
   TARGET_LINK_LIBRARIES(${prefix}_${F}
@@ -2541,7 +2551,9 @@ SET(host_vendor "${osname}")
 SET(host_os "${osrel}")
 SET(abs_top_builddir "${CMAKE_CURRENT_BINARY_DIR}")
 SET(abs_top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}")
-
+STRING(RANDOM LENGTH 3 ALPHABET "0123456789" PLATFORMUID)
+MATH(EXPR PLATFORMUID "${PLATFORMUID} + 1" OUTPUT_FORMAT DECIMAL)
+
 SET(CC_VERSION "${CMAKE_C_COMPILER}")
 
 # Build *FLAGS for libnetcdf.settings.
@@ -2589,26 +2601,43 @@ is_enabled(HAVE_BZ2 HAS_BZ2)
 is_enabled(ENABLE_REMOTE_FUNCTIONALITY DO_REMOTE_FUNCTIONALITY)
 
 if(ENABLE_S3_INTERNAL)
-SET(WHICH_S3_SDK "internal")
-SET(NC_WHICH_S3_SDK "internal")
+  SET(WHICH_S3_SDK "internal")
+  SET(NC_WHICH_S3_SDK "internal")
 elseif(ENABLE_S3_AWS)
-SET(WHICH_S3_SDK "aws-sdk-cpp")
-SET(NC_WHICH_S3_SDK "aws-sdk-cpp")
+  SET(WHICH_S3_SDK "aws-sdk-cpp")
+  SET(NC_WHICH_S3_SDK "aws-sdk-cpp")
 else()
-SET(WHICH_S3_SDK "none")
-SET(NC_WHICH_S3_SDK "none")
+  SET(WHICH_S3_SDK "none")
+  SET(NC_WHICH_S3_SDK "none")
 endif()
 
 if(WITH_S3_TESTING STREQUAL PUBLIC)
-SET(DO_S3_TESTING "public")
+SET(ENABLE_S3_TESTING "public")
 elseif(WITH_S3_TESTING)
-SET(DO_S3_TESTING "yes")
+SET(ENABLE_S3_TESTING "yes")
+SET(ENABLE_S3_TESTALL "yes")
 elseif(NOT WITH_S3_TESTING)
-SET(DO_S3_TESTING "no")
+SET(ENABLE_S3_TESTING "no")
 else()
-SET(DO_S3_TESTING "no")
+SET(ENABLE_S3_TESTING "no")
+endif()
+
+# The Unidata testing S3 bucket
+# WARNING: this must match the value in configure.ac
+SET(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
+
+# The working S3 path tree within the Unidata bucket.
+# WARNING: this must match the value in configure.ac
+SET(S3TESTSUBTREE "netcdf-c" CACHE STRING "Working S3 path.")
+# Build a unique id based on the date
+string(TIMESTAMP TESTUID "%s")
+if(ENABLE_S3_TESTING)
+FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/s3cleanup_${PLATFORMUID}.uids" "${TESTUID}\n")
 endif()
 
+# Copy the CTest customization file into binary directory, as required.
+CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake")
+
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
   "${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
@@ -2653,6 +2682,15 @@ SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
 SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)
 
+####
+# Build s3cleanup.sh and s3gc.sh
+#####
+SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/s3cleanup.in ${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in)
+SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}")
+SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}")
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3cleanup.in ${CMAKE_CURRENT_BINARY_DIR}/s3cleanup.sh @ONLY NEWLINE_STYLE LF)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in ${CMAKE_CURRENT_BINARY_DIR}/s3gc.sh @ONLY NEWLINE_STYLE LF)
+
 #####
 # Build and copy nc_test4/findplugin.sh to various places
 #####
@@ -2679,6 +2717,13 @@ IF(ENABLE_TESTS)
     # Build dap4_test/pingurl4.c
     #####
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ncdap_test/pingurl.c ${CMAKE_CURRENT_BINARY_DIR}/dap4_test/pingurl4.c @ONLY NEWLINE_STYLE LF)
+
+    #####
+    # Build CTestCustom.cmake to cleanup S3 after tests are done.
+    #####
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake NEWLINE_STYLE LF)
+
+
 ENDIF()
 
 if(DEFINED ENV{LIB_FUZZING_ENGINE})

CTestCustom.cmake → CTestCustom.cmake.in

@@ -12,3 +12,10 @@
 ## for more information.
 
 set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "500")
+
+IF(HAVE_BASH)
+IF(ENABLE_S3_TESTING)
+# Assume run in top-level CMAKE_BINARY_DIR
+set(CTEST_CUSTOM_POST_TEST "bash ${CMAKE_BINARY_DIR}/s3cleanup.sh")
+ENDIF()
+ENDIF()

Makefile.am

@@ -31,7 +31,7 @@ AM_DISTCHECK_CONFIGURE_FLAGS = --without-plugin-dir
 EXTRA_DIST = README.md COPYRIGHT INSTALL.md test_prog.c lib_flags.am	\
 cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in		\
 cmake_uninstall.cmake.in FixBundle.cmake.in nc-config.cmake.in		\
-RELEASE_NOTES.md CTestCustom.cmake CTestConfig.cmake.in			\
+RELEASE_NOTES.md CTestConfig.cmake.in CTestCustom.cmake.in		\
 libnetcdf.settings.in netCDFConfig.cmake.in CMakeInstallation.cmake	\
 test-driver-verbose test_common.in fuzz
 
@@ -116,10 +116,8 @@ endif
 
 # Optionally build test plugins
 if ENABLE_PLUGINS
-if ENABLE_FILTER_TESTING
 PLUGIN_DIR = plugins
 endif
-endif
 
 # If benchmarks were turned on, build and run a bunch more tests.
 if BUILD_BENCHMARKS
@@ -209,7 +207,17 @@ install-data-hook:
 
 # Create the VERSION file after the build
 # in case it is being used by packagers
+# Also track the S3 cleanup id
 all-local: liblib/libnetcdf.la
 	echo ${PACKAGE_VERSION} > VERSION
+if ENABLE_S3_TESTALL
+	echo "@TESTUID@" >> ${abs_top_builddir}/s3cleanup_@[email protected]
+endif
+
+if ENABLE_S3_TESTALL
+distclean-local:
+	rm -f ${abs_top_builddir}/s3cleanup_@[email protected]
+endif
+
 # Remove the VERSION file
 CLEANFILES = VERSION

RELEASE_NOTES.md

@@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.3 - TBD
 
+* Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
+* Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
 * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737).
 * Fix default parameters for caching of NCZarr. See [Github #2734](https://github.com/Unidata/netcdf-c/pull/2734).
 * Introducing configure-time options to disable various filters, even if the required libraries are available on the system, in support of [GitHub #2712](https://github.com/Unidata/netcdf-c/pull/2712). 

config.h.cmake.in

@@ -163,6 +163,12 @@ are set when opening a binary file on Windows. */
 /* if true, enable S3 testing*/
 #cmakedefine WITH_S3_TESTING "PUBLIC"
 
+/* S3 Test Bucket */
+#define S3TESTBUCKET "${S3TESTBUCKET}"
+
+/* S3 Working subtree path prefix*/
+#define S3TESTSUBTREE "${S3TESTSUBTREE}"
+
 /* if true, run extra tests which may not work yet */
 #cmakedefine EXTRA_TESTS 1
 
@@ -337,6 +343,9 @@ are set when opening a binary file on Windows. */
 /* Define to 1 if you have the `snprintf' function. */
 #cmakedefine HAVE_SNPRINTF 1
 
+/* Define to 1 if the system has the type `mode_t'. */
+#cmakedefine HAVE_MODE_T 1
+
 /* Define to 1 if the system has the type `ssize_t'. */
 #cmakedefine HAVE_SSIZE_T 1
 
@@ -358,6 +367,9 @@ are set when opening a binary file on Windows. */
 /* Define to 1 if you have the <stdlib.h> header file. */
 #cmakedefine HAVE_STDLIB_H 1
 
+/* Define to 1 if you have the <ctype.h> header file. */
+#cmakedefine HAVE_CTYPE_H 1
+
 /* Define to 1 if you have the <strings.h> header file. */
 #cmakedefine HAVE_STRINGS_H 1