Extend the netcdf API to support programmatic changes to the plugin s…

…earch path

re: Unidata#2753

As suggested by Ed Hartnett, This PR extends the netcdf.h API to support programmatic control over the search path used to locate plugins.

I created several different APIs, but finally settled on the following
API as being the simplest possible. It has the disadvantage that
it requires use of a global lock (not implemented) if used
in a threaded environment.

Specifically, note that modifying the plugin paths must be done "atomically".
That is, in a multi-threaded environment, it is important that the sequence of actions involved in setting up the plugin paths must be done by a single processor or in some other way as to guarantee that two or more processors are not simultaneously accessing the plugin path read/write operations.

As an example, assume there exists a mutex lock called PLUGINLOCK.
Then any processor accessing the plugin paths should operate as follows:
````
lock(PLUGINLOCK);
nc_plugin_path_read(...);
<rebuild plugin path>
nc_plugin_path_write(...);
unlock(PLUGINLOCK);
````

The API proposed in this PR looks like this (from netcdf-c/include/netcdf_filter.h).

* ````int nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs);````

    This function returns the current sequence of directories in the internal plugin path list. Since this function does not modify the plugin path, it can be called at any time.

    The arguments are as follows:
    - _formatx_ specify which dispatch implementation to read: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR.
    - _ndirsp_ return the number of dirs in the internal path list
    - _dirs_ memory for storing the sequence of directies in the internal path list.

    In practice, this function needs to be called twice. The first time with npaths not NULL and pathlist set to NULL to get the size of the path list. The second time with pathlist not NULL to get the actual sequence of paths.

* ````int nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs);````

    This function empties the current internal path sequence and replaces it with the sequence of directories argument. Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths.

    The arguments are as follows:
    - _formatx_ specify which dispatch implementation to write: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR or 0 (zero).
    - _ndirs_ length of the dirs argument
    - _dirs_ a vector of directory path string used to overwrite the current internal path list

    If the value zero is used for the formatx argument, then the value being written is applied to all implemention: currently NC_FORMATX_NC_HDF5 and NC_FORMATX_NCZARR.

In addition, two other API functions are defined.
````
int nc_plugin_path_initialize(void);
int nc_plugin_path_finalize(void);
````
As a rule, the initialize and finalize functions do not need to be explicitly called by the user because they are called as part of *nc_initialize()/nc_finalize()*.

In addition to the above changes, add a plugin path testcase:
unit_tests/run_pluginpaths.sh+tst_pluginpaths.c.

## Misc. Changes
1. Added a version number for the formatx dispatcher.
2. Setup a per-dispatcher global state mechanism.
3. Add some path manipulation utilities to netcf_aux.h
4. Fix the construction of netcdf_json.h as a BUILT_SOURCE.
5. Fix some minor bugs in netcdf_json.h
6. Fix the construction of netcdf_proplist.h as a BUILT_SOURCE.

CMakeLists.txt

@@ -78,6 +78,10 @@ add_library(netCDF::netcdf ALIAS netcdf)
 # configure.ac.
 set(NC_DISPATCH_VERSION 5)
 
+# Version of the plugin path dispatch table. This must match the value in
+# configure.ac.
+set(NC_PLUGINPATH_DISPATCH_VERSION 1)
+
 # Get system configuration, Use it to determine osname, os release, cpu. These
 # will be used when committing to CDash.
 find_program(UNAME NAMES uname)
@@ -869,6 +873,7 @@ if(NETCDF_ENABLE_TESTS)
   ###
   option(NETCDF_ENABLE_BENCHMARKS "Run benchmark Tests." OFF)
 
+  set(BUILD_BENCHMARKS ${NETCDF_ENABLE_BENCHMARKS} CACHE BOOL "alias for NETCDF_ENABLE_BENCHMARKS")
 
   ###
   # End known-failures.

RELEASE_NOTES.md

@@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.3 - TBD
 
+* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????) for more information.
+
 ## Known Issue
 
 * Parallel operation using `mpich 4.2.0` (the default on `Ubuntu 24.04`) results in 'unexpected results' when running `nc_test4/run_par_test.sh`.  This can be fixed by removing `mpich` and associated libraries and development packages and installing `mpich 4.2.2` by hand, or by using `openmpi` provided via `apt`.

cmake/dependencies.cmake

@@ -226,12 +226,13 @@ if(USE_HDF5)
 endif(USE_HDF5)
 
 ################################
-# Curl Libraryies
+# Curl Libraries
 # Only needed for DAP (DAP2 or DAP4)
-# and NCZARR with S3 Support
+# and NCZARR S3 support
+# and byterange support
 ################################
 
-if( (NETCDF_ENABLE_DAP AND (NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT)) OR (NETCDF_ENABLE_NCZARR AND NETCDF_ENABLENCZARR_S3))
+if( NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT OR NETCDF_ENABLE_NCZARR_S3)
 
   # See if we have libcurl
   find_package(CURL)

config.h.cmake.in

@@ -493,9 +493,12 @@ with zip */
 /* Add extra properties to _NCProperties attribute */
 #cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA}
 
-/* Idspatch table version */
+/* Version for dispatch table version */
 #cmakedefine NC_DISPATCH_VERSION ${NC_DISPATCH_VERSION}
 
+/* Version for plugin path dispatch table version */
+#cmakedefine NC_PLUGINPATH_DISPATCH_VERSION ${NC_PLUGINPATH_DISPATCH_VERSION}
+
 /* no IEEE float on this platform */
 #cmakedefine NO_IEEE_FLOAT 1
 

configure.ac

@@ -1153,7 +1153,7 @@ AC_ARG_ENABLE([benchmarks],
                               are timed. We use these tests to check netCDF performance.])])
 test "x$enable_benchmarks" = xyes || enable_benchmarks=no
 AC_MSG_RESULT($enable_benchmarks)
-if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then
+if test "x$enable_hdf5" = xno -a "x$enable_benchmarks" = xyes; then
 AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.])
 fi
 AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes])
@@ -1633,6 +1633,7 @@ has_hdf5_ros3=no
 
 if test "x$enable_hdf5" = xyes; then
 
+   AC_DEFINE([NETCDF_ENABLE_HDF5], [1], [if true, use HDF5])
    AC_DEFINE([USE_HDF5], [1], [if true, use HDF5])
    AC_DEFINE([H5_USE_16_API], [1], [use HDF5 1.6 API])
 
@@ -2258,6 +2259,13 @@ AX_SET_META([NC_HAS_BZ2],[$have_bz2],[yes])
 AC_SUBST([NC_DISPATCH_VERSION], [5])
 AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
 
+# This is the version of the plugin path dispatch table.
+# If the dispatch table is changed, this should be incremented.
+# If this is changed, make sure the value in
+# CMakeLists.txt also changes to match.
+AC_SUBST([NC_PLUGINPATH_DISPATCH_VERSION], [1])
+AC_DEFINE_UNQUOTED([NC_PLUGINPATH_DISPATCH_VERSION], [${NC_PLUGINPATH_DISPATCH_VERSION}], [Plugin Path Dispatch table version.])
+
 #####
 # End netcdf_meta.h definitions.
 #####

include/Makefile.am

@@ -23,7 +23,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h		\
 ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h	\
 ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h	\
 hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h          \
-ncjson.h ncxml.h ncs3sdk.h
+ncjson.h ncxml.h ncs3sdk.h ncplugins.h
 
 if USE_DAP
 noinst_HEADERS += ncdap.h

include/hdf5internal.h

@@ -196,6 +196,11 @@ struct NC_HDF5_Filter {
     unsigned int* params;  /**< Params for arbitrary filter. */
 };
 
+/* The type for the NC_FORMATX_NC_HDF5 Global State Object */
+typedef struct GlobalHDF5 { /* libhdf5 dispatcher specific parameters */
+    int placeholder; /* ensure a non-empty struct */
+} GlobalHDF5;
+
 int NC4_hdf5_filter_initialize(void);
 int NC4_hdf5_filter_finalize(void);
 int NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id);

include/nc4internal.h

@@ -107,7 +107,10 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
 /* Forward declarations. */
 struct NC_GRP_INFO;
 struct NC_TYPE_INFO;
+
+/* Opaque */
 struct NCRCinfo;
+struct NC_PluginPathDispatch;
 
 /**
  * This struct provides indexed Access to Meta-data objects. See the
@@ -461,15 +464,10 @@ extern int nc_get_alignment(int* thresholdp, int* alignmentp);
 /* Begin to collect global state info in one place (more to do) */
 
 typedef struct NCglobalstate {
-    int initialized;
     char* tempdir; /* track a usable temp dir */
     char* home; /* track $HOME */
     char* cwd; /* track getcwd */
     struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
-    struct GlobalZarr { /* Zarr specific parameters */
-	char dimension_separator;
-	int default_zarrformat;
-    } zarr;
     struct GlobalAWS { /* AWS S3 specific parameters/defaults */
 	char* default_region;
 	char* config_file;
@@ -483,6 +481,12 @@ typedef struct NCglobalstate {
 	int alignment;
     } alignment;
     struct ChunkCache chunkcache;
+    /* Global dispatcher and states specific to each dispatcher
+       and indexed by NC_FORMATX */
+    struct FormatXGlobal {
+        void* state[NC_FORMATX_COUNT]; /* type is opaque (like e.g. file_info_format field) */
+        const struct NC_PluginPathDispatch** pluginapi; /*[NC_FORMATX_COUNT];*/
+    } formatxstate;
 } NCglobalstate;
 
 extern struct NCglobalstate* NC_getglobalstate(void);

include/ncdispatch.h

@@ -35,7 +35,7 @@
 
 /* Given a filename, check its magic number */
 /* Change magic number size from 4 to 8 to be more precise for HDF5 */
-#define MAGIC_NUMBER_LEN ((size_t)8)
+#define MAGIC_NUMBER_LEN ((unsigned long long)8)
 #define MAGIC_HDF5_FILE 1
 #define MAGIC_HDF4_FILE 2
 #define MAGIC_CDF1_FILE 1 /* std classic format */

include/ncplugins.h

@@ -0,0 +1,46 @@
+/*
+Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+See COPYRIGHT for license information.
+*/
+
+/*
+Common functionality for plugin paths/
+For internal use only.
+*/
+
+#ifndef NCPLUGINS_H
+#define NCPLUGINS_H
+
+/* Opaque */
+struct NClist;
+
+/* Define the plugin path management dispatch table */
+
+typedef struct NC_PluginPathDispatch {
+    int model; /* one of the NC_FORMATX #'s */
+    int dispatch_version;
+    int (*initialize)(void** statep, const struct NClist* initialpaths);
+    int (*finalize)(void** statep);
+    int (*read)(void* state, size_t* ndirsp, char** dirs);
+    int (*write)(void* state, size_t ndirs, char** const dirs);
+} NC_PluginPathDispatch;
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* Known Plugin Dispatchers */
+#ifdef USE_HDF5
+EXTERNL NC_PluginPathDispatch* NC4_hdf5_pluginpathtable;
+#endif
+#ifdef NETCDF_ENABLE_NCZARR
+EXTERNL NC_PluginPathDispatch* NCZ_pluginpathtable;
+#endif
+
+/* See the file netcdf_aux.h for plugin-related utility functions */
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /*NCPLUGINS_H*/

include/netcdf.h

@@ -226,10 +226,13 @@ Use this in mode flags for both nc_create() and nc_open(). */
 #define NC_FORMATX_NCZARR    (10)
 #define NC_FORMATX_UNDEFINED (0)
 
-  /* To avoid breaking compatibility (such as in the python library),
+/* Define the legal NC_FORMATX indices i.e. 0<=index<NC_FORMATX_COUNT */
+#define NC_FORMATX_COUNT ((NC_FORMATX_NCZARR)+1)
+
+/* To avoid breaking compatibility (such as in the python library),
    we need to retain the NC_FORMAT_xxx format as well. This may come
-  out eventually, as the NC_FORMATX is more clear that it's an extended
-  format specifier.*/
+   out eventually, as the NC_FORMATX is more clear that it's an extended
+   format specifier.*/
 
 #define NC_FORMAT_NC3       NC_FORMATX_NC3 /**< \deprecated As of 4.4.0, use NC_FORMATX_NC3 */
 #define NC_FORMAT_NC_HDF5   NC_FORMATX_NC_HDF5 /**< \deprecated As of 4.4.0, use NC_FORMATX_NC_HDF5 */

include/netcdf_aux.h

@@ -96,6 +96,70 @@ EXTERNL int ncaux_abort_compound(void* tag);
 EXTERNL int ncaux_add_field(void* tag,  const char *name, nc_type field_type,
 			   int ndims, const int* dimsizes);
 
+/**************************************************/
+/* Path-list Utilities */
+
+/**
+Parse a string into a sequence of path directories.
+
+The pathlist argument has the following syntax:
+    paths := <empty> | dirlist
+    dirlist := dir | dirlist separator dir
+    separator := ';' | ':'
+    dir := <OS specific directory path>
+
+@param pathlist a string encoding a list of directories
+@param sep  one of ';' | ':' | '\0' where '\0' means use the platform's default separator.
+@param ndirsp return the number of directories in dirsp
+@param dirsp return a vector of strings representing the directories parsed from pathlist; caller frees
+@return ::NC_NOERR
+
+Note that this function is called twice: first time to get the number of directories
+and second to get the directories.
+
+Author: Dennis Heimbigner
+*/
+
+EXTERNL int ncaux_plugin_path_parse(const char* pathlist, char sep, size_t* ndirsp, char** dirs);
+
+/**
+Concatenate a vector of directories with the separator between.
+This is more-or-less the inverse of the ncaux_plugin_path_parse function
+
+The resulting string has following syntax:
+    paths := <empty> | dirlist
+    dirlist := dir | dirlist separator dir
+    separator := ';' | ':'
+    dir := <OS specific directory path>
+    
+@param ndirs the number of directories
+@param dirsp the directory vector to concatenate
+@param sep one of ';', ':', or '\0'
+@param catlen length of the cat arg including a nul terminator
+@param cat user provided space for holding the concatenation; nul termination guaranteed if catlen > 0.
+@return ::NC_NOERR
+@return ::NC_EINVAL for illegal arguments
+
+Note that this function is called twice: first time to get the expected size of
+the concatenated string and second to get the contents of the concatenation.
+
+Author: Dennis Heimbigner
+*/
+
+EXTERNL int ncaux_plugin_path_tostring(size_t ndirs, char** const dirs, char sep, size_t* catlen, char* cat);
+
+
+/*
+Reclaim a char** object possibly produced by ncaux_plugin_parse function.
+
+@param veclen the number of entries in vec
+@param vec    a char** vectore
+@return ::NC_NOERR
+@return ::NC_EINVAL for illegal arguments
+*/
+
+EXTERNL int ncaux_plugin_path_freestringvec(size_t veclen, char** vec);
+
 #if defined(__cplusplus)
 }
 #endif

include/netcdf_filter.h

@@ -12,8 +12,8 @@
 #ifndef NETCDF_FILTER_H
 #define NETCDF_FILTER_H 1
 
-/* API for libdispatch/dfilter.c
-*/
+/**************************************************/
+/* API for libdispatch/dfilter.c */
 
 /* Must match values in <H5Zpublic.h> */
 #ifndef H5Z_FILTER_DEFLATE
@@ -110,9 +110,65 @@ EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *leve
 EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle);
 EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep);
 
+/* Filter path query/set */
+EXTERNL int nc_filter_path_query(int id);
+
+/**************************************************/
+/* API for libdispatch/dplugin.c */
+
+/* Plugin path functions */
+
+/**
+ * This function is called as part of nc_initialize.
+ * Its purpose is to initialize the plugin paths state.
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+*/
+
+EXTERNL int nc_plugin_path_initialize(void);
+
+/**
+ * This function is called as part of nc_finalize()
+ * Its purpose is to clean-up plugin path state.
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+*/
+
+EXTERNL int nc_plugin_path_finalize(void);
+
+/**
+ * Return the current sequence of directories in the internal plugin path list.
+ * Since this function does not modify the plugin path, it can be called at any time.
+ * @param formatx specify which dispatch implementatio to read: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR.
+ * @param ndirsp return the number of dirs in the internal path list
+ * @param dirs memory for storing the sequence of directies in the internal path list.
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+ *
+ * As a rule, this function needs to be called twice.
+ * The first time with npaths not NULL and pathlist set to NULL
+ *     to get the size of the path list.
+ * The second time with pathlist not NULL to get the actual sequence of paths.
+*/
+
+EXTERNL int nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs);
+
+/**
+ * Empty the current internal path sequence
+ * and replace with the sequence of directories argument.
+ *
+ * Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths.
+ * @param formatx specify which dispatch implementation to write: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR.
+ * @param ndirs length of the dirs argument
+ * @param dirs to overwrite the current internal path list
+ * @return NC_NOERR
+ * @author Dennis Heimbigner
+*/
+
+EXTERNL int nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs);
+
 #if defined(__cplusplus)
 }
 #endif
-/**************************************************/
 
 #endif /* NETCDF_FILTER_H */

include/netcdf_json.h

@@ -1086,7 +1086,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value)
     else
         NCJnewstring(NCJ_STRING,value,&jvalue);
     NCJinsert(object,key,jvalue);
-done:
     return NCJTHROW(stat);
 }
 
@@ -1100,7 +1099,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue)
     snprintf(digits,sizeof(digits),"%lld",ivalue);
     NCJnewstring(NCJ_STRING,digits,&jvalue);
     NCJinsert(object,key,jvalue);
-done:
     return NCJTHROW(stat);
 }
 
@@ -1305,6 +1303,8 @@ netcdf_supresswarnings(void)
     ignore = (void*)NCJparse;
     ignore = (void*)NCJdump;
     ignore = (void*)NCJtotext;
+    ignore = (void*)NCJinsertstring;
+    ignore = (void*)NCJinsertint;
     ignore = ignore;
 }
 #endif /*NETCDF_JSON_H*/